From e4c2ad4627b71d280603f721234ec8990f3aa6bf Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Fri, 7 Aug 2026 19:23:34 -0700
Subject: [PATCH 001/105] fix(anthropic): buffer streamed responses carrying
 server-fulfilled tools so retrieval tool calls never reach the client

---
 .../compression_interception/handler.py       |   4 +-
 litellm/integrations/custom_logger.py         |   4 +-
 .../messages/agentic_streaming_iterator.py    |  59 ++++++
 litellm/llms/custom_httpx/llm_http_handler.py |  23 +++
 .../guardrail_hooks/headroom/headroom.py      |   1 +
 .../test_agentic_streaming_iterator.py        | 178 ++++++++++++++++++
 .../custom_httpx/test_llm_http_handler.py     |  71 +++++++
 ui/litellm-dashboard/src/lib/http/schema.d.ts |   7 +
 8 files changed, 345 insertions(+), 2 deletions(-)

diff --git a/litellm/integrations/compression_interception/handler.py b/litellm/integrations/compression_interception/handler.py
index 7ea60053e6f..76720682101 100644
--- a/litellm/integrations/compression_interception/handler.py
+++ b/litellm/integrations/compression_interception/handler.py
@@ -7,7 +7,7 @@ litellm_content_retrieve tool calls server-side via the typed agentic loop plan.
 
 import time
 import uuid
-from typing import Any, Final, cast
+from typing import Any, ClassVar, Final, cast
 
 from litellm._logging import verbose_logger
 from litellm.compression import compress
@@ -72,6 +72,8 @@ class CompressionInterceptionLogger(CustomLogger):
     4. Build typed rerun plan with tool_result blocks from the compressed cache.
     """
 
+    server_fulfilled_tool_names: ClassVar[frozenset[str]] = frozenset({LITELLM_CONTENT_RETRIEVE_TOOL_NAME})
+
     def __init__(
         self,
         enabled: bool = True,
diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py
index a0c78674ac8..60af4063f84 100644
--- a/litellm/integrations/custom_logger.py
+++ b/litellm/integrations/custom_logger.py
@@ -3,7 +3,7 @@
 import re
 import traceback
 from collections.abc import AsyncGenerator
-from typing import TYPE_CHECKING, Any, Final, Optional
+from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional
 
 from pydantic import BaseModel
 
@@ -60,6 +60,8 @@ _BASE64_INLINE_PATTERN: Final = re.compile(
 
 class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callback#callback-class
     # Class variables or attributes
+    server_fulfilled_tool_names: ClassVar[frozenset[str]] = frozenset()
+
     def __init__(
         self,
         turn_off_message_logging: bool = False,
diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py
index 5c4fa4700c0..0699595821e 100644
--- a/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py
+++ b/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py
@@ -6,14 +6,27 @@ yields every chunk to the caller (preserving real streaming), collects
 all bytes, and on stream exhaustion rebuilds the full Anthropic response
 to run through agentic completion hooks. If an agentic hook fires, the
 follow-up response is chained as Phase 2 of the same iterator.
+
+In hold-back mode (``hold_back=True``), chunks are buffered instead of
+yielded live, with SSE ping events emitted while the upstream message is
+in flight. On exhaustion the hooks run first: if a follow-up response
+replaces the message, only the follow-up is yielded and the buffered
+message is dropped; otherwise the buffer is replayed verbatim. This is
+required for server-fulfilled tools (e.g. ``headroom_retrieve``), whose
+tool_use blocks must never reach a client that cannot execute them.
 """
 
+import asyncio
+import contextlib
 import json
 from collections.abc import AsyncIterator
 from typing import Any, Final, cast
 
 from litellm._logging import verbose_logger
 
+PING_SSE_BYTES: Final = b'event: ping\ndata: {"type": "ping"}\n\n'
+HOLD_BACK_PING_INTERVAL_SECONDS: Final = 15.0
+
 # ---------------------------------------------------------------------------
 # SSE parsing helpers (module-level to keep the class lean)
 # ---------------------------------------------------------------------------
@@ -156,6 +169,8 @@ class AgenticAnthropicStreamingIterator:
         logging_obj: Any,
         custom_llm_provider: str,
         kwargs: dict,
+        hold_back: bool = False,
+        ping_interval_seconds: float = HOLD_BACK_PING_INTERVAL_SECONDS,
     ):
         self._inner = completion_stream.__aiter__()
         self._http_handler = http_handler
@@ -166,16 +181,23 @@ class AgenticAnthropicStreamingIterator:
         self._logging_obj = logging_obj
         self._custom_llm_provider = custom_llm_provider
         self._kwargs = kwargs
+        self._hold_back = hold_back
+        self._ping_interval_seconds = ping_interval_seconds
 
         self._collected_bytes: list[bytes] = []
         self._stream_exhausted = False
         self._hook_processing_done = False
         self._follow_up_iterator: AsyncIterator | None = None
+        self._drain_task: asyncio.Task | None = None
+        self._replay_index = 0
 
     def __aiter__(self):
         return self
 
     async def __anext__(self) -> bytes:
+        if self._hold_back:
+            return await self._anext_held_back()
+
         # Phase 1: yield from upstream, collect bytes
         if not self._stream_exhausted:
             try:
@@ -194,11 +216,48 @@ class AgenticAnthropicStreamingIterator:
 
         raise StopAsyncIteration
 
+    async def _drain_upstream(self) -> None:
+        try:
+            while True:
+                self._collected_bytes.append(await self._inner.__anext__())
+        except StopAsyncIteration:
+            return
+
+    async def _anext_held_back(self) -> bytes:
+        if self._drain_task is None:
+            self._drain_task = asyncio.create_task(self._drain_upstream())
+            return PING_SSE_BYTES
+
+        while not self._stream_exhausted:
+            try:
+                await asyncio.wait_for(asyncio.shield(self._drain_task), timeout=self._ping_interval_seconds)
+            except asyncio.TimeoutError:
+                return PING_SSE_BYTES
+            self._stream_exhausted = True
+            await self._process_agentic_hooks()
+
+        if self._follow_up_iterator is not None:
+            return await self._follow_up_iterator.__anext__()
+
+        if self._replay_index < len(self._collected_bytes):
+            chunk: Final = self._collected_bytes[self._replay_index]
+            self._replay_index += 1
+            return chunk
+
+        raise StopAsyncIteration
+
     async def aclose(self) -> None:
         from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
             aclose_if_supported,
         )
 
+        if self._drain_task is not None and self._drain_task.done():
+            if not self._drain_task.cancelled():
+                self._drain_task.exception()
+        elif self._drain_task is not None:
+            self._drain_task.cancel()
+            with contextlib.suppress(asyncio.CancelledError):
+                await self._drain_task
         await aclose_if_supported(self._inner)
         await aclose_if_supported(self._follow_up_iterator)
 
diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py
index a58397c9184..913cedcfa55 100644
--- a/litellm/llms/custom_httpx/llm_http_handler.py
+++ b/litellm/llms/custom_httpx/llm_http_handler.py
@@ -2189,6 +2189,10 @@ class BaseLLMHTTPHandler:
                 logging_obj=logging_obj,
                 custom_llm_provider=custom_llm_provider,
                 kwargs={**kwargs, "api_key": api_key} if api_key else kwargs,
+                hold_back=self._should_hold_back_stream(
+                    logging_obj=logging_obj,
+                    tools=anthropic_messages_optional_request_params.get("tools"),
+                ),
             )
             return AnthropicMessagesStreamingResponse(
                 completion_stream=initial_response,
@@ -5033,6 +5037,25 @@ class BaseLLMHTTPHandler:
                 return True
         return False
 
+    @staticmethod
+    def _should_hold_back_stream(logging_obj: LiteLLMLoggingObj, tools: object) -> bool:
+        """
+        True when the request carries a tool that a registered callback fulfills
+        server-side (e.g. ``headroom_retrieve``). The model's tool_use for such a
+        tool must never reach the client, which cannot execute it: the agentic
+        loop replaces the whole message with a follow-up response, so the stream
+        is buffered (with ping keepalives) instead of forwarded live.
+        """
+        if not isinstance(tools, list) or not tools:
+            return False
+        from litellm.litellm_core_utils.prompt_templates.factory import has_tool_with_name
+
+        return any(
+            has_tool_with_name(tools, name)
+            for cb in _custom_logger_callbacks(logging_obj)
+            for name in getattr(cb, "server_fulfilled_tool_names", frozenset())
+        )
+
     @staticmethod
     def _check_agentic_loop_safety(
         tool_calls: object,
diff --git a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py
index 8bfd5cca58a..84c6b220e62 100644
--- a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py
+++ b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py
@@ -339,6 +339,7 @@ def _build_responses_followup_items(
 
 class HeadroomGuardrail(CustomGuardrail):
     records_own_guardrail_information: ClassVar[bool] = True
+    server_fulfilled_tool_names: ClassVar[frozenset[str]] = frozenset({HEADROOM_RETRIEVE_TOOL_NAME})
 
     @classmethod
     def get_supported_event_hooks(cls) -> list[GuardrailEventHooks]:
diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_agentic_streaming_iterator.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_agentic_streaming_iterator.py
index b9bda07336f..a6b071bbab5 100644
--- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_agentic_streaming_iterator.py
+++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_agentic_streaming_iterator.py
@@ -2,6 +2,7 @@
 Tests for AgenticAnthropicStreamingIterator and SSE rebuild helpers.
 """
 
+import asyncio
 import json
 import os
 import sys
@@ -13,6 +14,7 @@ import pytest
 sys.path.insert(0, os.path.abspath("../../../../.."))
 
 from litellm.llms.anthropic.experimental_pass_through.messages.agentic_streaming_iterator import (
+    PING_SSE_BYTES,
     AgenticAnthropicStreamingIterator,
     _handle_content_block_delta,
     _handle_content_block_start,
@@ -230,6 +232,51 @@ class MockAsyncStream:
         return chunk
 
 
+class MockSlowAsyncStream(MockAsyncStream):
+    """Async iterator that sleeps before every chunk."""
+
+    def __init__(self, chunks: List[bytes], delay_seconds: float):
+        super().__init__(chunks)
+        self._delay_seconds = delay_seconds
+
+    async def __anext__(self) -> bytes:
+        await asyncio.sleep(self._delay_seconds)
+        return await super().__anext__()
+
+
+class MockFailingAsyncStream(MockAsyncStream):
+    """Async iterator that raises after yielding its chunks."""
+
+    def __init__(self, chunks: List[bytes], error: Exception):
+        super().__init__(chunks)
+        self._error = error
+
+    async def __anext__(self) -> bytes:
+        if self._idx >= len(self._chunks):
+            raise self._error
+        return await super().__anext__()
+
+
+def _build_hold_back_iterator(
+    stream: MockAsyncStream,
+    mock_handler: MagicMock,
+    ping_interval_seconds: float = 15.0,
+) -> AgenticAnthropicStreamingIterator:
+    return AgenticAnthropicStreamingIterator(
+        completion_stream=stream,
+        http_handler=mock_handler,
+        model="claude-sonnet-4-20250514",
+        messages=[],
+        anthropic_messages_provider_config=MagicMock(),
+        anthropic_messages_optional_request_params={},
+        logging_obj=MagicMock(),
+        custom_llm_provider="anthropic",
+        kwargs={},
+        hold_back=True,
+        ping_interval_seconds=ping_interval_seconds,
+    )
+
+
 # ---------------------------------------------------------------------------
 # Tests for _parse_sse_events
 # ---------------------------------------------------------------------------
@@ -790,3 +837,134 @@ class TestAgenticStreamingIteratorErrorHandling:
 
         call_kwargs = mock_handler._call_agentic_completion_hooks.call_args
         assert call_kwargs.kwargs["stream"] is True
+
+
+class TestAgenticStreamingIteratorHoldBack:
+    @pytest.mark.asyncio
+    async def test_should_not_leak_intercepted_message_when_follow_up_fires(self):
+        """The buffered tool_use message must be dropped: only pings and follow-up bytes reach the client."""
+        phase1_chunks = _build_tool_use_stream()
+        phase2_chunks = [b"follow-up-chunk-1", b"follow-up-chunk-2"]
+
+        mock_handler = MagicMock()
+        mock_handler._call_agentic_completion_hooks = AsyncMock(return_value=MockAsyncStream(phase2_chunks))
+
+        iterator = _build_hold_back_iterator(MockAsyncStream(phase1_chunks), mock_handler)
+
+        collected = []
+        async for chunk in iterator:
+            collected.append(chunk)
+
+        non_ping = [c for c in collected if c != PING_SSE_BYTES]
+        assert non_ping == phase2_chunks
+        assert b"litellm_content_retrieve" not in b"".join(collected)
+        assert collected[0] == PING_SSE_BYTES
+
+    @pytest.mark.asyncio
+    async def test_should_replay_buffer_verbatim_when_no_hook_fires(self):
+        """Without interception the buffered message is replayed byte-identical after the pings."""
+        chunks = _build_simple_text_stream()
+
+        mock_handler = MagicMock()
+        mock_handler._call_agentic_completion_hooks = AsyncMock(return_value=None)
+
+        iterator = _build_hold_back_iterator(MockAsyncStream(chunks), mock_handler)
+
+        collected = []
+        async for chunk in iterator:
+            collected.append(chunk)
+
+        assert [c for c in collected if c != PING_SSE_BYTES] == chunks
+        mock_handler._call_agentic_completion_hooks.assert_awaited_once()
+
+    @pytest.mark.asyncio
+    async def test_should_emit_pings_while_upstream_is_slow(self):
+        """Pings keep the client connection alive while the upstream message is buffered."""
+        chunks = _build_simple_text_stream()
+
+        mock_handler = MagicMock()
+        mock_handler._call_agentic_completion_hooks = AsyncMock(return_value=None)
+
+        iterator = _build_hold_back_iterator(
+            MockSlowAsyncStream(chunks, delay_seconds=0.05),
+            mock_handler,
+            ping_interval_seconds=0.02,
+        )
+
+        collected = []
+        async for chunk in iterator:
+            collected.append(chunk)
+
+        assert collected.count(PING_SSE_BYTES) >= 2
+        assert [c for c in collected if c != PING_SSE_BYTES] == chunks
+
+    @pytest.mark.asyncio
+    async def test_should_propagate_upstream_error_instead_of_partial_message(self):
+        """An upstream failure surfaces as an error; the client never receives a truncated message."""
+        chunks = _build_simple_text_stream()[:2]
+
+        mock_handler = MagicMock()
+        mock_handler._call_agentic_completion_hooks = AsyncMock(return_value=None)
+
+        iterator = _build_hold_back_iterator(
+            MockFailingAsyncStream(chunks, RuntimeError("upstream died")),
+            mock_handler,
+        )
+
+        collected = []
+        with pytest.raises(RuntimeError, match="upstream died"):
+            async for chunk in iterator:
+                collected.append(chunk)
+
+        assert all(c == PING_SSE_BYTES for c in collected)
+        mock_handler._call_agentic_completion_hooks.assert_not_awaited()
+
+    @pytest.mark.asyncio
+    async def test_should_replay_buffer_when_hook_processing_errors(self):
+        """A hook crash degrades to replaying the original message rather than dropping it."""
+        chunks = _build_tool_use_stream()
+
+        mock_handler = MagicMock()
+        mock_handler._call_agentic_completion_hooks = AsyncMock(side_effect=RuntimeError("hook exploded"))
+
+        mock_logging = MagicMock()
+        mock_logging.litellm_call_id = "test_call_holdback"
+
+        iterator = AgenticAnthropicStreamingIterator(
+            completion_stream=MockAsyncStream(chunks),
+            http_handler=mock_handler,
+            model="claude-sonnet-4-20250514",
+            messages=[],
+            anthropic_messages_provider_config=MagicMock(),
+            anthropic_messages_optional_request_params={},
+            logging_obj=mock_logging,
+            custom_llm_provider="anthropic",
+            kwargs={},
+            hold_back=True,
+        )
+
+        collected = []
+        async for chunk in iterator:
+            collected.append(chunk)
+
+        assert [c for c in collected if c != PING_SSE_BYTES] == chunks
+
+    @pytest.mark.asyncio
+    async def test_aclose_cancels_drain_task(self):
+        """Closing the iterator mid-buffer must cancel the background drain task."""
+        chunks = _build_simple_text_stream()
+
+        mock_handler = MagicMock()
+        mock_handler._call_agentic_completion_hooks = AsyncMock(return_value=None)
+
+        iterator = _build_hold_back_iterator(
+            MockSlowAsyncStream(chunks, delay_seconds=5.0),
+            mock_handler,
+        )
+
+        first = await iterator.__anext__()
+        assert first == PING_SSE_BYTES
+        assert iterator._drain_task is not None
+
+        await iterator.aclose()
+        assert iterator._drain_task.cancelled()
diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py
index fddd8d09dfc..6c2727e7da2 100644
--- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py
+++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py
@@ -2071,3 +2071,74 @@ async def test_anthropic_invalid_thinking_signature_retry_resigns_bedrock_reques
     retry_authorization = posts[1]["headers"]["Authorization"]
     assert retry_authorization.startswith("AWS4-HMAC-SHA256")
     assert retry_authorization != first_attempt_headers["Authorization"]
+
+
+class TestShouldHoldBackStream:
+    """_should_hold_back_stream gates the buffered (non-leaking) streaming mode
+    for server-fulfilled tools like headroom_retrieve."""
+
+    @staticmethod
+    def _logging_obj_with(callbacks):
+        logging_obj = Mock()
+        logging_obj.dynamic_success_callbacks = callbacks
+        return logging_obj
+
+    def test_should_hold_back_when_callback_owns_tool_in_request(self):
+        from litellm.integrations.custom_logger import CustomLogger
+
+        class RetrievalCallback(CustomLogger):
+            server_fulfilled_tool_names = frozenset({"headroom_retrieve"})
+
+        tools = [
+            {"name": "Bash", "input_schema": {"type": "object"}},
+            {"name": "headroom_retrieve", "input_schema": {"type": "object"}},
+        ]
+        assert (
+            BaseLLMHTTPHandler._should_hold_back_stream(
+                logging_obj=self._logging_obj_with([RetrievalCallback()]), tools=tools
+            )
+            is True
+        )
+
+    def test_should_stream_live_when_tool_absent_from_request(self):
+        from litellm.integrations.custom_logger import CustomLogger
+
+        class RetrievalCallback(CustomLogger):
+            server_fulfilled_tool_names = frozenset({"headroom_retrieve"})
+
+        tools = [{"name": "Bash", "input_schema": {"type": "object"}}]
+        assert (
+            BaseLLMHTTPHandler._should_hold_back_stream(
+                logging_obj=self._logging_obj_with([RetrievalCallback()]), tools=tools
+            )
+            is False
+        )
+
+    def test_should_stream_live_when_no_callback_declares_tool_names(self):
+        from litellm.integrations.custom_logger import CustomLogger
+
+        tools = [{"name": "headroom_retrieve", "input_schema": {"type": "object"}}]
+        assert (
+            BaseLLMHTTPHandler._should_hold_back_stream(
+                logging_obj=self._logging_obj_with([CustomLogger()]), tools=tools
+            )
+            is False
+        )
+
+    def test_should_stream_live_without_tools(self):
+        assert BaseLLMHTTPHandler._should_hold_back_stream(logging_obj=self._logging_obj_with([]), tools=None) is False
+
+    def test_interception_callbacks_declare_their_retrieval_tools(self):
+        from litellm.integrations.compression_interception.handler import (
+            LITELLM_CONTENT_RETRIEVE_TOOL_NAME,
+            CompressionInterceptionLogger,
+        )
+        from litellm.proxy.guardrails.guardrail_hooks.headroom.headroom import (
+            HEADROOM_RETRIEVE_TOOL_NAME,
+            HeadroomGuardrail,
+        )
+
+        assert HeadroomGuardrail.server_fulfilled_tool_names == frozenset({HEADROOM_RETRIEVE_TOOL_NAME})
+        assert CompressionInterceptionLogger.server_fulfilled_tool_names == frozenset(
+            {LITELLM_CONTENT_RETRIEVE_TOOL_NAME}
+        )
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index f1660e77ad9..8e950874a10 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -21391,6 +21391,13 @@ export interface components {
              * @description What the routed traffic actually cost
              */
             spend: number;
+            /**
+             * Tier Turns
+             * @description Turns per tier, keyed by the tier name the routing decision recorded at request time (never re-derived at read time, since the tier-to-model mapping is mutable config). Tier names are scoped to this group's router_type and are not comparable across types: a complexity router reports 'simple'/'medium'/'complex'/'reasoning', a quality router reports its numeric quality tier, and an adaptive router records no tier at all. Turns no tier served (the classifier fell back to default_model) are absent rather than pooled under a sentinel key, so the values may sum to less than turns
+             */
+            tier_turns?: {
+                [key: string]: number;
+            };
             /** Turns */
             turns: number;
         };

From f994068a7338b4bb54fa7a53d76fb009dd3e9f6a Mon Sep 17 00:00:00 2001
From: mateo <mateo@berri.ai>
Date: Sat, 8 Aug 2026 07:30:24 +0000
Subject: [PATCH 002/105] fix(anthropic): keep pinging during agentic hooks and
 fail instead of replaying server-fulfilled tool_use

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../messages/agentic_streaming_iterator.py    |  87 ++++++++++++---
 litellm/llms/custom_httpx/llm_http_handler.py |  29 ++---
 .../test_agentic_streaming_iterator.py        | 105 +++++++++++++++---
 .../custom_httpx/test_llm_http_handler.py     |  28 ++---
 4 files changed, 190 insertions(+), 59 deletions(-)

diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py
index 0699595821e..4cf348dda9e 100644
--- a/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py
+++ b/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py
@@ -9,11 +9,13 @@ follow-up response is chained as Phase 2 of the same iterator.
 
 In hold-back mode (``hold_back=True``), chunks are buffered instead of
 yielded live, with SSE ping events emitted while the upstream message is
-in flight. On exhaustion the hooks run first: if a follow-up response
-replaces the message, only the follow-up is yielded and the buffered
-message is dropped; otherwise the buffer is replayed verbatim. This is
-required for server-fulfilled tools (e.g. ``headroom_retrieve``), whose
-tool_use blocks must never reach a client that cannot execute them.
+in flight and while the agentic hooks run. On exhaustion the hooks run
+first: if a follow-up response replaces the message, only the follow-up
+is yielded and the buffered message is dropped; otherwise the buffer is
+replayed verbatim, unless it holds a tool_use for a server-fulfilled tool
+(e.g. ``headroom_retrieve``), in which case an SSE ``error`` event is
+emitted because such a block must never reach a client that cannot
+execute it.
 """
 
 import asyncio
@@ -26,6 +28,11 @@ from litellm._logging import verbose_logger
 
 PING_SSE_BYTES: Final = b'event: ping\ndata: {"type": "ping"}\n\n'
 HOLD_BACK_PING_INTERVAL_SECONDS: Final = 15.0
+SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES: Final = (
+    b"event: error\n"
+    b'data: {"type": "error", "error": {"type": "api_error", "message": '
+    b'"Server-side tool retrieval failed, so this turn could not be completed. Please retry."}}\n\n'
+)
 
 # ---------------------------------------------------------------------------
 # SSE parsing helpers (module-level to keep the class lean)
@@ -170,6 +177,7 @@ class AgenticAnthropicStreamingIterator:
         custom_llm_provider: str,
         kwargs: dict,
         hold_back: bool = False,
+        server_fulfilled_tool_names: frozenset[str] = frozenset(),
         ping_interval_seconds: float = HOLD_BACK_PING_INTERVAL_SECONDS,
     ):
         self._inner = completion_stream.__aiter__()
@@ -182,6 +190,7 @@ class AgenticAnthropicStreamingIterator:
         self._custom_llm_provider = custom_llm_provider
         self._kwargs = kwargs
         self._hold_back = hold_back
+        self._server_fulfilled_tool_names = server_fulfilled_tool_names
         self._ping_interval_seconds = ping_interval_seconds
 
         self._collected_bytes: list[bytes] = []
@@ -189,7 +198,9 @@ class AgenticAnthropicStreamingIterator:
         self._hook_processing_done = False
         self._follow_up_iterator: AsyncIterator | None = None
         self._drain_task: asyncio.Task | None = None
+        self._hook_task: asyncio.Task | None = None
         self._replay_index = 0
+        self._error_emitted = False
 
     def __aiter__(self):
         return self
@@ -223,22 +234,42 @@ class AgenticAnthropicStreamingIterator:
         except StopAsyncIteration:
             return
 
+    async def _completed_within_ping_interval(self, task: asyncio.Task) -> bool:
+        try:
+            await asyncio.wait_for(asyncio.shield(task), timeout=self._ping_interval_seconds)
+        except asyncio.TimeoutError:
+            return False
+        return True
+
     async def _anext_held_back(self) -> bytes:
         if self._drain_task is None:
             self._drain_task = asyncio.create_task(self._drain_upstream())
             return PING_SSE_BYTES
 
-        while not self._stream_exhausted:
-            try:
-                await asyncio.wait_for(asyncio.shield(self._drain_task), timeout=self._ping_interval_seconds)
-            except asyncio.TimeoutError:
+        if not self._stream_exhausted:
+            if not await self._completed_within_ping_interval(self._drain_task):
                 return PING_SSE_BYTES
             self._stream_exhausted = True
-            await self._process_agentic_hooks()
+
+        if self._hook_task is None:
+            self._hook_task = asyncio.create_task(self._process_agentic_hooks())
+        if not await self._completed_within_ping_interval(self._hook_task):
+            return PING_SSE_BYTES
 
         if self._follow_up_iterator is not None:
             return await self._follow_up_iterator.__anext__()
 
+        if self._buffer_holds_server_fulfilled_tool_use():
+            if self._error_emitted:
+                raise StopAsyncIteration
+            self._error_emitted = True
+            verbose_logger.error(
+                "AgenticStreamingIterator: hooks did not replace a message containing a server-fulfilled "
+                "tool_use [model=%s]; emitting an SSE error instead of leaking the tool call to the client",
+                self._model,
+            )
+            return SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES
+
         if self._replay_index < len(self._collected_bytes):
             chunk: Final = self._collected_bytes[self._replay_index]
             self._replay_index += 1
@@ -246,18 +277,40 @@ class AgenticAnthropicStreamingIterator:
 
         raise StopAsyncIteration
 
+    def _buffer_holds_server_fulfilled_tool_use(self) -> bool:
+        if not self._server_fulfilled_tool_names:
+            return False
+        started_blocks: Final = (
+            data.get("content_block")
+            for event_type, data in _parse_sse_events(b"".join(self._collected_bytes))
+            if event_type == "content_block_start"
+        )
+        return any(
+            isinstance(block, dict)
+            and block.get("type") == "tool_use"
+            and block.get("name") in self._server_fulfilled_tool_names
+            for block in started_blocks
+        )
+
+    @staticmethod
+    async def _settle_task(task: asyncio.Task | None) -> None:
+        if task is None:
+            return
+        if task.done():
+            if not task.cancelled():
+                task.exception()
+            return
+        task.cancel()
+        with contextlib.suppress(asyncio.CancelledError):
+            await task
+
     async def aclose(self) -> None:
         from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
             aclose_if_supported,
         )
 
-        if self._drain_task is not None and self._drain_task.done():
-            if not self._drain_task.cancelled():
-                self._drain_task.exception()
-        elif self._drain_task is not None:
-            self._drain_task.cancel()
-            with contextlib.suppress(asyncio.CancelledError):
-                await self._drain_task
+        await self._settle_task(self._drain_task)
+        await self._settle_task(self._hook_task)
         await aclose_if_supported(self._inner)
         await aclose_if_supported(self._follow_up_iterator)
 
diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py
index 913cedcfa55..e9b88e45219 100644
--- a/litellm/llms/custom_httpx/llm_http_handler.py
+++ b/litellm/llms/custom_httpx/llm_http_handler.py
@@ -2179,6 +2179,10 @@ class BaseLLMHTTPHandler:
                 AgenticAnthropicStreamingIterator,
             )
 
+            held_back_tool_names: Final = self._server_fulfilled_tools_in_request(
+                logging_obj=logging_obj,
+                tools=anthropic_messages_optional_request_params.get("tools"),
+            )
             initial_response = AgenticAnthropicStreamingIterator(
                 completion_stream=completion_stream,
                 http_handler=self,
@@ -2189,10 +2193,8 @@ class BaseLLMHTTPHandler:
                 logging_obj=logging_obj,
                 custom_llm_provider=custom_llm_provider,
                 kwargs={**kwargs, "api_key": api_key} if api_key else kwargs,
-                hold_back=self._should_hold_back_stream(
-                    logging_obj=logging_obj,
-                    tools=anthropic_messages_optional_request_params.get("tools"),
-                ),
+                hold_back=bool(held_back_tool_names),
+                server_fulfilled_tool_names=held_back_tool_names,
             )
             return AnthropicMessagesStreamingResponse(
                 completion_stream=initial_response,
@@ -5038,22 +5040,23 @@ class BaseLLMHTTPHandler:
         return False
 
     @staticmethod
-    def _should_hold_back_stream(logging_obj: LiteLLMLoggingObj, tools: object) -> bool:
+    def _server_fulfilled_tools_in_request(logging_obj: LiteLLMLoggingObj, tools: object) -> frozenset[str]:
         """
-        True when the request carries a tool that a registered callback fulfills
-        server-side (e.g. ``headroom_retrieve``). The model's tool_use for such a
-        tool must never reach the client, which cannot execute it: the agentic
-        loop replaces the whole message with a follow-up response, so the stream
-        is buffered (with ping keepalives) instead of forwarded live.
+        The request's tools that a registered callback fulfills server-side (e.g.
+        ``headroom_retrieve``). The model's tool_use for such a tool must never
+        reach the client, which cannot execute it: the agentic loop replaces the
+        whole message with a follow-up response, so a stream carrying any of
+        these is buffered (with ping keepalives) instead of forwarded live.
         """
         if not isinstance(tools, list) or not tools:
-            return False
+            return frozenset()
         from litellm.litellm_core_utils.prompt_templates.factory import has_tool_with_name
 
-        return any(
-            has_tool_with_name(tools, name)
+        return frozenset(
+            name
             for cb in _custom_logger_callbacks(logging_obj)
             for name in getattr(cb, "server_fulfilled_tool_names", frozenset())
+            if has_tool_with_name(tools, name)
         )
 
     @staticmethod
diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_agentic_streaming_iterator.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_agentic_streaming_iterator.py
index a6b071bbab5..d59f76232cd 100644
--- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_agentic_streaming_iterator.py
+++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_agentic_streaming_iterator.py
@@ -15,6 +15,7 @@ sys.path.insert(0, os.path.abspath("../../../../.."))
 
 from litellm.llms.anthropic.experimental_pass_through.messages.agentic_streaming_iterator import (
     PING_SSE_BYTES,
+    SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES,
     AgenticAnthropicStreamingIterator,
     _handle_content_block_delta,
     _handle_content_block_start,
@@ -261,6 +262,7 @@ def _build_hold_back_iterator(
     stream: MockAsyncStream,
     mock_handler: MagicMock,
     ping_interval_seconds: float = 15.0,
+    server_fulfilled_tool_names: frozenset = frozenset({"litellm_content_retrieve"}),
 ) -> AgenticAnthropicStreamingIterator:
     return AgenticAnthropicStreamingIterator(
         completion_stream=stream,
@@ -273,6 +275,7 @@ def _build_hold_back_iterator(
         custom_llm_provider="anthropic",
         kwargs={},
         hold_back=True,
+        server_fulfilled_tool_names=server_fulfilled_tool_names,
         ping_interval_seconds=ping_interval_seconds,
     )
 
@@ -920,27 +923,76 @@ class TestAgenticStreamingIteratorHoldBack:
         mock_handler._call_agentic_completion_hooks.assert_not_awaited()
 
     @pytest.mark.asyncio
-    async def test_should_replay_buffer_when_hook_processing_errors(self):
-        """A hook crash degrades to replaying the original message rather than dropping it."""
+    async def test_should_emit_pings_while_hooks_are_slow(self):
+        """Retrieval and follow-up generation can outlast a client's idle timeout, so hooks get keepalives too."""
+        chunks = _build_tool_use_stream()
+        phase2_chunks = [b"follow-up-chunk"]
+
+        async def slow_hooks(**_kwargs):
+            await asyncio.sleep(0.12)
+            return MockAsyncStream(phase2_chunks)
+
+        mock_handler = MagicMock()
+        mock_handler._call_agentic_completion_hooks = AsyncMock(side_effect=slow_hooks)
+
+        iterator = _build_hold_back_iterator(
+            MockAsyncStream(chunks),
+            mock_handler,
+            ping_interval_seconds=0.02,
+        )
+
+        collected = []
+        async for chunk in iterator:
+            collected.append(chunk)
+
+        assert collected.count(PING_SSE_BYTES) >= 4
+        assert [c for c in collected if c != PING_SSE_BYTES] == phase2_chunks
+
+    @pytest.mark.asyncio
+    async def test_should_error_instead_of_replaying_server_fulfilled_tool_use_when_hook_crashes(self):
+        """A hook crash must not replay the buffered retrieval tool_use: that is the unknown-tool bug."""
         chunks = _build_tool_use_stream()
 
         mock_handler = MagicMock()
         mock_handler._call_agentic_completion_hooks = AsyncMock(side_effect=RuntimeError("hook exploded"))
 
-        mock_logging = MagicMock()
-        mock_logging.litellm_call_id = "test_call_holdback"
+        iterator = _build_hold_back_iterator(MockAsyncStream(chunks), mock_handler)
 
-        iterator = AgenticAnthropicStreamingIterator(
-            completion_stream=MockAsyncStream(chunks),
-            http_handler=mock_handler,
-            model="claude-sonnet-4-20250514",
-            messages=[],
-            anthropic_messages_provider_config=MagicMock(),
-            anthropic_messages_optional_request_params={},
-            logging_obj=mock_logging,
-            custom_llm_provider="anthropic",
-            kwargs={},
-            hold_back=True,
+        collected = []
+        async for chunk in iterator:
+            collected.append(chunk)
+
+        assert [c for c in collected if c != PING_SSE_BYTES] == [SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES]
+        assert b"litellm_content_retrieve" not in b"".join(collected)
+
+    @pytest.mark.asyncio
+    async def test_should_error_instead_of_replaying_when_no_hook_fires_on_tool_use(self):
+        """Hooks returning None on a retrieval tool_use is still a leak, so the turn fails loudly."""
+        chunks = _build_tool_use_stream()
+
+        mock_handler = MagicMock()
+        mock_handler._call_agentic_completion_hooks = AsyncMock(return_value=None)
+
+        iterator = _build_hold_back_iterator(MockAsyncStream(chunks), mock_handler)
+
+        collected = []
+        async for chunk in iterator:
+            collected.append(chunk)
+
+        assert [c for c in collected if c != PING_SSE_BYTES] == [SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES]
+
+    @pytest.mark.asyncio
+    async def test_should_replay_client_owned_tool_use_verbatim(self):
+        """Only server-fulfilled tools are withheld: a client's own tool_use still reaches it byte-identical."""
+        chunks = _build_tool_use_stream()
+
+        mock_handler = MagicMock()
+        mock_handler._call_agentic_completion_hooks = AsyncMock(return_value=None)
+
+        iterator = _build_hold_back_iterator(
+            MockAsyncStream(chunks),
+            mock_handler,
+            server_fulfilled_tool_names=frozenset({"headroom_retrieve"}),
         )
 
         collected = []
@@ -968,3 +1020,26 @@ class TestAgenticStreamingIteratorHoldBack:
 
         await iterator.aclose()
         assert iterator._drain_task.cancelled()
+
+    @pytest.mark.asyncio
+    async def test_aclose_cancels_in_flight_hook_task(self):
+        """Closing while hooks are running must not leave the retrieval follow-up task orphaned."""
+        chunks = _build_tool_use_stream()
+
+        async def never_finishing_hooks(**_kwargs):
+            await asyncio.sleep(5.0)
+
+        mock_handler = MagicMock()
+        mock_handler._call_agentic_completion_hooks = AsyncMock(side_effect=never_finishing_hooks)
+
+        iterator = _build_hold_back_iterator(
+            MockAsyncStream(chunks),
+            mock_handler,
+            ping_interval_seconds=0.02,
+        )
+
+        while iterator._hook_task is None:
+            await iterator.__anext__()
+
+        await iterator.aclose()
+        assert iterator._hook_task.cancelled()
diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py
index 6c2727e7da2..798fb4c92e2 100644
--- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py
+++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py
@@ -2073,9 +2073,9 @@ async def test_anthropic_invalid_thinking_signature_retry_resigns_bedrock_reques
     assert retry_authorization != first_attempt_headers["Authorization"]
 
 
-class TestShouldHoldBackStream:
-    """_should_hold_back_stream gates the buffered (non-leaking) streaming mode
-    for server-fulfilled tools like headroom_retrieve."""
+class TestServerFulfilledToolsInRequest:
+    """_server_fulfilled_tools_in_request gates the buffered (non-leaking) streaming
+    mode for server-fulfilled tools like headroom_retrieve."""
 
     @staticmethod
     def _logging_obj_with(callbacks):
@@ -2093,12 +2093,9 @@ class TestShouldHoldBackStream:
             {"name": "Bash", "input_schema": {"type": "object"}},
             {"name": "headroom_retrieve", "input_schema": {"type": "object"}},
         ]
-        assert (
-            BaseLLMHTTPHandler._should_hold_back_stream(
-                logging_obj=self._logging_obj_with([RetrievalCallback()]), tools=tools
-            )
-            is True
-        )
+        assert BaseLLMHTTPHandler._server_fulfilled_tools_in_request(
+            logging_obj=self._logging_obj_with([RetrievalCallback()]), tools=tools
+        ) == frozenset({"headroom_retrieve"})
 
     def test_should_stream_live_when_tool_absent_from_request(self):
         from litellm.integrations.custom_logger import CustomLogger
@@ -2108,10 +2105,10 @@ class TestShouldHoldBackStream:
 
         tools = [{"name": "Bash", "input_schema": {"type": "object"}}]
         assert (
-            BaseLLMHTTPHandler._should_hold_back_stream(
+            BaseLLMHTTPHandler._server_fulfilled_tools_in_request(
                 logging_obj=self._logging_obj_with([RetrievalCallback()]), tools=tools
             )
-            is False
+            == frozenset()
         )
 
     def test_should_stream_live_when_no_callback_declares_tool_names(self):
@@ -2119,14 +2116,17 @@ class TestShouldHoldBackStream:
 
         tools = [{"name": "headroom_retrieve", "input_schema": {"type": "object"}}]
         assert (
-            BaseLLMHTTPHandler._should_hold_back_stream(
+            BaseLLMHTTPHandler._server_fulfilled_tools_in_request(
                 logging_obj=self._logging_obj_with([CustomLogger()]), tools=tools
             )
-            is False
+            == frozenset()
         )
 
     def test_should_stream_live_without_tools(self):
-        assert BaseLLMHTTPHandler._should_hold_back_stream(logging_obj=self._logging_obj_with([]), tools=None) is False
+        assert (
+            BaseLLMHTTPHandler._server_fulfilled_tools_in_request(logging_obj=self._logging_obj_with([]), tools=None)
+            == frozenset()
+        )
 
     def test_interception_callbacks_declare_their_retrieval_tools(self):
         from litellm.integrations.compression_interception.handler import (

From 398e3d214cc97be0531428f3fb506a7cc42e2683 Mon Sep 17 00:00:00 2001
From: mateo <mateo@berri.ai>
Date: Sat, 8 Aug 2026 19:28:40 +0000
Subject: [PATCH 003/105] refactor(anthropic): trim hold-back commentary and
 drop dead rebuilt-content expression

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../messages/agentic_streaming_iterator.py      | 17 ++++-------------
 litellm/llms/custom_httpx/llm_http_handler.py   |  8 +-------
 2 files changed, 5 insertions(+), 20 deletions(-)

diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py
index 4cf348dda9e..a88b148e92c 100644
--- a/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py
+++ b/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py
@@ -7,14 +7,10 @@ all bytes, and on stream exhaustion rebuilds the full Anthropic response
 to run through agentic completion hooks. If an agentic hook fires, the
 follow-up response is chained as Phase 2 of the same iterator.
 
-In hold-back mode (``hold_back=True``), chunks are buffered instead of
-yielded live, with SSE ping events emitted while the upstream message is
-in flight and while the agentic hooks run. On exhaustion the hooks run
-first: if a follow-up response replaces the message, only the follow-up
-is yielded and the buffered message is dropped; otherwise the buffer is
-replayed verbatim, unless it holds a tool_use for a server-fulfilled tool
-(e.g. ``headroom_retrieve``), in which case an SSE ``error`` event is
-emitted because such a block must never reach a client that cannot
+In hold-back mode (``hold_back=True``) chunks are buffered instead of yielded
+live, keepalive pings run until the hooks finish, and then either the follow-up
+replaces the message or the buffer replays, except that a buffered tool_use for
+a server-fulfilled tool fails the turn rather than reaching a client that cannot
 execute it.
 """
 
@@ -329,11 +325,6 @@ class AgenticAnthropicStreamingIterator:
                 verbose_logger.debug("AgenticStreamingIterator: Could not rebuild response from SSE bytes")
                 return
 
-            [
-                (f"{b.get('type')}({b.get('name', '')})" if b.get("type") == "tool_use" else b.get("type"))
-                for b in rebuilt.get("content", [])
-            ]
-
             result: Final = await self._http_handler._call_agentic_completion_hooks(
                 response=rebuilt,
                 model=self._model,
diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py
index e9b88e45219..193a38a5404 100644
--- a/litellm/llms/custom_httpx/llm_http_handler.py
+++ b/litellm/llms/custom_httpx/llm_http_handler.py
@@ -5041,13 +5041,7 @@ class BaseLLMHTTPHandler:
 
     @staticmethod
     def _server_fulfilled_tools_in_request(logging_obj: LiteLLMLoggingObj, tools: object) -> frozenset[str]:
-        """
-        The request's tools that a registered callback fulfills server-side (e.g.
-        ``headroom_retrieve``). The model's tool_use for such a tool must never
-        reach the client, which cannot execute it: the agentic loop replaces the
-        whole message with a follow-up response, so a stream carrying any of
-        these is buffered (with ping keepalives) instead of forwarded live.
-        """
+        """The request's tools that a registered callback fulfills server-side (e.g. ``headroom_retrieve``)."""
         if not isinstance(tools, list) or not tools:
             return frozenset()
         from litellm.litellm_core_utils.prompt_templates.factory import has_tool_with_name

From cbefb1ce5ffbdc90c9e6691206752b40ec9ef6e0 Mon Sep 17 00:00:00 2001
From: mateo <mateo@berri.ai>
Date: Sat, 8 Aug 2026 19:41:42 +0000
Subject: [PATCH 004/105] fix(anthropic): keep pinging while the held-back
 follow-up stream is in flight

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../messages/agentic_streaming_iterator.py    | 27 ++++++++-
 .../test_agentic_streaming_iterator.py        | 59 +++++++++++++++++++
 2 files changed, 83 insertions(+), 3 deletions(-)

diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py
index a88b148e92c..3aaba0b139c 100644
--- a/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py
+++ b/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py
@@ -8,8 +8,8 @@ to run through agentic completion hooks. If an agentic hook fires, the
 follow-up response is chained as Phase 2 of the same iterator.
 
 In hold-back mode (``hold_back=True``) chunks are buffered instead of yielded
-live, keepalive pings run until the hooks finish, and then either the follow-up
-replaces the message or the buffer replays, except that a buffered tool_use for
+live, keepalive pings run whenever no other byte is ready, and then either the
+follow-up replaces the message or the buffer replays, except that a tool_use for
 a server-fulfilled tool fails the turn rather than reaching a client that cannot
 execute it.
 """
@@ -30,6 +30,14 @@ SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES: Final = (
     b'"Server-side tool retrieval failed, so this turn could not be completed. Please retry."}}\n\n'
 )
 
+
+async def _anext_or_none(iterator: AsyncIterator) -> bytes | None:
+    try:
+        return await iterator.__anext__()
+    except StopAsyncIteration:
+        return None
+
+
 # ---------------------------------------------------------------------------
 # SSE parsing helpers (module-level to keep the class lean)
 # ---------------------------------------------------------------------------
@@ -195,6 +203,7 @@ class AgenticAnthropicStreamingIterator:
         self._follow_up_iterator: AsyncIterator | None = None
         self._drain_task: asyncio.Task | None = None
         self._hook_task: asyncio.Task | None = None
+        self._follow_up_chunk_task: asyncio.Task | None = None
         self._replay_index = 0
         self._error_emitted = False
 
@@ -253,7 +262,7 @@ class AgenticAnthropicStreamingIterator:
             return PING_SSE_BYTES
 
         if self._follow_up_iterator is not None:
-            return await self._follow_up_iterator.__anext__()
+            return await self._next_follow_up_chunk(self._follow_up_iterator)
 
         if self._buffer_holds_server_fulfilled_tool_use():
             if self._error_emitted:
@@ -273,6 +282,17 @@ class AgenticAnthropicStreamingIterator:
 
         raise StopAsyncIteration
 
+    async def _next_follow_up_chunk(self, follow_up_iterator: AsyncIterator) -> bytes:
+        if self._follow_up_chunk_task is None:
+            self._follow_up_chunk_task = asyncio.create_task(_anext_or_none(follow_up_iterator))
+        if not await self._completed_within_ping_interval(self._follow_up_chunk_task):
+            return PING_SSE_BYTES
+        chunk: Final = self._follow_up_chunk_task.result()
+        self._follow_up_chunk_task = None
+        if chunk is None:
+            raise StopAsyncIteration
+        return chunk
+
     def _buffer_holds_server_fulfilled_tool_use(self) -> bool:
         if not self._server_fulfilled_tool_names:
             return False
@@ -307,6 +327,7 @@ class AgenticAnthropicStreamingIterator:
 
         await self._settle_task(self._drain_task)
         await self._settle_task(self._hook_task)
+        await self._settle_task(self._follow_up_chunk_task)
         await aclose_if_supported(self._inner)
         await aclose_if_supported(self._follow_up_iterator)
 
diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_agentic_streaming_iterator.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_agentic_streaming_iterator.py
index d59f76232cd..d4aebf099d1 100644
--- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_agentic_streaming_iterator.py
+++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_agentic_streaming_iterator.py
@@ -1001,6 +1001,65 @@ class TestAgenticStreamingIteratorHoldBack:
 
         assert [c for c in collected if c != PING_SSE_BYTES] == chunks
 
+    @pytest.mark.asyncio
+    async def test_should_emit_pings_while_the_follow_up_stream_is_slow(self):
+        """The corrected answer can be slow to generate, so the follow-up stream gets keepalives too."""
+        chunks = _build_tool_use_stream()
+        phase2_chunks = [b"follow-up-chunk-1", b"follow-up-chunk-2"]
+
+        mock_handler = MagicMock()
+        mock_handler._call_agentic_completion_hooks = AsyncMock(
+            return_value=MockSlowAsyncStream(phase2_chunks, delay_seconds=0.06)
+        )
+
+        iterator = _build_hold_back_iterator(
+            MockAsyncStream(chunks),
+            mock_handler,
+            ping_interval_seconds=0.02,
+        )
+
+        collected = []
+        async for chunk in iterator:
+            collected.append(chunk)
+
+        first_follow_up_index = collected.index(phase2_chunks[0])
+        assert collected[first_follow_up_index + 1] == PING_SSE_BYTES
+        assert [c for c in collected if c != PING_SSE_BYTES] == phase2_chunks
+
+    @pytest.mark.asyncio
+    async def test_should_propagate_follow_up_stream_error(self):
+        """A failing follow-up stream surfaces its error instead of hanging on pings forever."""
+        chunks = _build_tool_use_stream()
+
+        mock_handler = MagicMock()
+        mock_handler._call_agentic_completion_hooks = AsyncMock(
+            return_value=MockFailingAsyncStream([b"follow-up-chunk"], RuntimeError("follow-up died"))
+        )
+
+        iterator = _build_hold_back_iterator(MockAsyncStream(chunks), mock_handler, ping_interval_seconds=0.02)
+
+        with pytest.raises(RuntimeError, match="follow-up died"):
+            async for _ in iterator:
+                pass
+
+    @pytest.mark.asyncio
+    async def test_aclose_cancels_in_flight_follow_up_chunk_task(self):
+        """Closing while a follow-up chunk is pending must not orphan that task."""
+        chunks = _build_tool_use_stream()
+
+        mock_handler = MagicMock()
+        mock_handler._call_agentic_completion_hooks = AsyncMock(
+            return_value=MockSlowAsyncStream([b"follow-up-chunk"], delay_seconds=5.0)
+        )
+
+        iterator = _build_hold_back_iterator(MockAsyncStream(chunks), mock_handler, ping_interval_seconds=0.02)
+
+        while iterator._follow_up_chunk_task is None:
+            await iterator.__anext__()
+
+        await iterator.aclose()
+        assert iterator._follow_up_chunk_task.cancelled()
+
     @pytest.mark.asyncio
     async def test_aclose_cancels_drain_task(self):
         """Closing the iterator mid-buffer must cancel the background drain task."""

From bb0bb48da8c3a5fe3557812e79a31c71c608e006 Mon Sep 17 00:00:00 2001
From: mateo <mateo@berri.ai>
Date: Sat, 8 Aug 2026 20:05:45 +0000
Subject: [PATCH 005/105] fix(proxy): do not let held-back keepalive pings
 block the budget reservation refund on client disconnect

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/constants.py                          |  2 ++
 .../messages/agentic_streaming_iterator.py    | 10 +++----
 litellm/proxy/common_request_processing.py    |  6 ++--
 litellm/proxy/common_utils/sse_keepalive.py   |  4 ++-
 .../test_agentic_streaming_iterator.py        | 30 +++++++++----------
 .../proxy/test_budget_reservation.py          | 28 +++++++++++++++++
 6 files changed, 57 insertions(+), 23 deletions(-)

diff --git a/litellm/constants.py b/litellm/constants.py
index 6f0e9e7afe2..9db9fb36b65 100644
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -434,6 +434,8 @@ CONNECTION_ERROR_PATTERNS: Final[list[str]] = [
 ]
 STREAM_SSE_DONE_STRING: Final[str] = "[DONE]"
 STREAM_SSE_DATA_PREFIX: Final[str] = "data: "
+STREAM_SSE_KEEPALIVE_PING_CHUNK: Final[str] = 'event: ping\ndata: {"type": "ping"}\n\n'
+STREAM_SSE_KEEPALIVE_PING_BYTES: Final[bytes] = STREAM_SSE_KEEPALIVE_PING_CHUNK.encode("utf-8")
 ### SPEND TRACKING ###
 DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND: Final = float(
     os.getenv("DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND", 0.001400)
diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py
index 3aaba0b139c..d6f4e51a09a 100644
--- a/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py
+++ b/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py
@@ -21,8 +21,8 @@ from collections.abc import AsyncIterator
 from typing import Any, Final, cast
 
 from litellm._logging import verbose_logger
+from litellm.constants import STREAM_SSE_KEEPALIVE_PING_BYTES
 
-PING_SSE_BYTES: Final = b'event: ping\ndata: {"type": "ping"}\n\n'
 HOLD_BACK_PING_INTERVAL_SECONDS: Final = 15.0
 SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES: Final = (
     b"event: error\n"
@@ -249,17 +249,17 @@ class AgenticAnthropicStreamingIterator:
     async def _anext_held_back(self) -> bytes:
         if self._drain_task is None:
             self._drain_task = asyncio.create_task(self._drain_upstream())
-            return PING_SSE_BYTES
+            return STREAM_SSE_KEEPALIVE_PING_BYTES
 
         if not self._stream_exhausted:
             if not await self._completed_within_ping_interval(self._drain_task):
-                return PING_SSE_BYTES
+                return STREAM_SSE_KEEPALIVE_PING_BYTES
             self._stream_exhausted = True
 
         if self._hook_task is None:
             self._hook_task = asyncio.create_task(self._process_agentic_hooks())
         if not await self._completed_within_ping_interval(self._hook_task):
-            return PING_SSE_BYTES
+            return STREAM_SSE_KEEPALIVE_PING_BYTES
 
         if self._follow_up_iterator is not None:
             return await self._next_follow_up_chunk(self._follow_up_iterator)
@@ -286,7 +286,7 @@ class AgenticAnthropicStreamingIterator:
         if self._follow_up_chunk_task is None:
             self._follow_up_chunk_task = asyncio.create_task(_anext_or_none(follow_up_iterator))
         if not await self._completed_within_ping_interval(self._follow_up_chunk_task):
-            return PING_SSE_BYTES
+            return STREAM_SSE_KEEPALIVE_PING_BYTES
         chunk: Final = self._follow_up_chunk_task.result()
         self._follow_up_chunk_task = None
         if chunk is None:
diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py
index 159d7508f4e..2607ff411a9 100644
--- a/litellm/proxy/common_request_processing.py
+++ b/litellm/proxy/common_request_processing.py
@@ -27,6 +27,7 @@ from litellm.constants import (
     MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG,
     RETURN_RAW_MODEL_NAME_METADATA_KEY,
     STREAM_SSE_DATA_PREFIX,
+    STREAM_SSE_KEEPALIVE_PING_BYTES,
     UNSAFE_PROXY_RESPONSE_HEADERS,
 )
 from litellm.integrations.custom_guardrail import CustomGuardrail
@@ -2953,8 +2954,9 @@ class ProxyBaseLLMRequestProcessing:
                 # so a GeneratorExit on client disconnect is raised there and any
                 # statement after the yield never runs. The slow-path hook is
                 # awaited above, so a cancellation during it still leaves this
-                # False and refunds.
-                delivered_chunk = True
+                # False and refunds. A keepalive ping carries no provider output,
+                # so it must not suppress that refund.
+                delivered_chunk = delivered_chunk or chunk != STREAM_SSE_KEEPALIVE_PING_BYTES
                 yield serialize_chunk(chunk)
             stream_completed = True
         except (asyncio.CancelledError, GeneratorExit):
diff --git a/litellm/proxy/common_utils/sse_keepalive.py b/litellm/proxy/common_utils/sse_keepalive.py
index 6700700ff7c..6e0ea4db431 100644
--- a/litellm/proxy/common_utils/sse_keepalive.py
+++ b/litellm/proxy/common_utils/sse_keepalive.py
@@ -6,7 +6,9 @@ from typing import Final
 
 import anyio
 
-ANTHROPIC_PING_SSE_CHUNK: Final = 'event: ping\ndata: {"type": "ping"}\n\n'
+from litellm.constants import STREAM_SSE_KEEPALIVE_PING_CHUNK
+
+ANTHROPIC_PING_SSE_CHUNK: Final = STREAM_SSE_KEEPALIVE_PING_CHUNK
 
 
 def _coerce_interval(ping_interval_seconds: float | str | None) -> float | None:
diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_agentic_streaming_iterator.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_agentic_streaming_iterator.py
index d4aebf099d1..b0467430533 100644
--- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_agentic_streaming_iterator.py
+++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_agentic_streaming_iterator.py
@@ -13,8 +13,8 @@ import pytest
 
 sys.path.insert(0, os.path.abspath("../../../../.."))
 
+from litellm.constants import STREAM_SSE_KEEPALIVE_PING_BYTES
 from litellm.llms.anthropic.experimental_pass_through.messages.agentic_streaming_iterator import (
-    PING_SSE_BYTES,
     SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES,
     AgenticAnthropicStreamingIterator,
     _handle_content_block_delta,
@@ -858,10 +858,10 @@ class TestAgenticStreamingIteratorHoldBack:
         async for chunk in iterator:
             collected.append(chunk)
 
-        non_ping = [c for c in collected if c != PING_SSE_BYTES]
+        non_ping = [c for c in collected if c != STREAM_SSE_KEEPALIVE_PING_BYTES]
         assert non_ping == phase2_chunks
         assert b"litellm_content_retrieve" not in b"".join(collected)
-        assert collected[0] == PING_SSE_BYTES
+        assert collected[0] == STREAM_SSE_KEEPALIVE_PING_BYTES
 
     @pytest.mark.asyncio
     async def test_should_replay_buffer_verbatim_when_no_hook_fires(self):
@@ -877,7 +877,7 @@ class TestAgenticStreamingIteratorHoldBack:
         async for chunk in iterator:
             collected.append(chunk)
 
-        assert [c for c in collected if c != PING_SSE_BYTES] == chunks
+        assert [c for c in collected if c != STREAM_SSE_KEEPALIVE_PING_BYTES] == chunks
         mock_handler._call_agentic_completion_hooks.assert_awaited_once()
 
     @pytest.mark.asyncio
@@ -898,8 +898,8 @@ class TestAgenticStreamingIteratorHoldBack:
         async for chunk in iterator:
             collected.append(chunk)
 
-        assert collected.count(PING_SSE_BYTES) >= 2
-        assert [c for c in collected if c != PING_SSE_BYTES] == chunks
+        assert collected.count(STREAM_SSE_KEEPALIVE_PING_BYTES) >= 2
+        assert [c for c in collected if c != STREAM_SSE_KEEPALIVE_PING_BYTES] == chunks
 
     @pytest.mark.asyncio
     async def test_should_propagate_upstream_error_instead_of_partial_message(self):
@@ -919,7 +919,7 @@ class TestAgenticStreamingIteratorHoldBack:
             async for chunk in iterator:
                 collected.append(chunk)
 
-        assert all(c == PING_SSE_BYTES for c in collected)
+        assert all(c == STREAM_SSE_KEEPALIVE_PING_BYTES for c in collected)
         mock_handler._call_agentic_completion_hooks.assert_not_awaited()
 
     @pytest.mark.asyncio
@@ -945,8 +945,8 @@ class TestAgenticStreamingIteratorHoldBack:
         async for chunk in iterator:
             collected.append(chunk)
 
-        assert collected.count(PING_SSE_BYTES) >= 4
-        assert [c for c in collected if c != PING_SSE_BYTES] == phase2_chunks
+        assert collected.count(STREAM_SSE_KEEPALIVE_PING_BYTES) >= 4
+        assert [c for c in collected if c != STREAM_SSE_KEEPALIVE_PING_BYTES] == phase2_chunks
 
     @pytest.mark.asyncio
     async def test_should_error_instead_of_replaying_server_fulfilled_tool_use_when_hook_crashes(self):
@@ -962,7 +962,7 @@ class TestAgenticStreamingIteratorHoldBack:
         async for chunk in iterator:
             collected.append(chunk)
 
-        assert [c for c in collected if c != PING_SSE_BYTES] == [SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES]
+        assert [c for c in collected if c != STREAM_SSE_KEEPALIVE_PING_BYTES] == [SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES]
         assert b"litellm_content_retrieve" not in b"".join(collected)
 
     @pytest.mark.asyncio
@@ -979,7 +979,7 @@ class TestAgenticStreamingIteratorHoldBack:
         async for chunk in iterator:
             collected.append(chunk)
 
-        assert [c for c in collected if c != PING_SSE_BYTES] == [SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES]
+        assert [c for c in collected if c != STREAM_SSE_KEEPALIVE_PING_BYTES] == [SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES]
 
     @pytest.mark.asyncio
     async def test_should_replay_client_owned_tool_use_verbatim(self):
@@ -999,7 +999,7 @@ class TestAgenticStreamingIteratorHoldBack:
         async for chunk in iterator:
             collected.append(chunk)
 
-        assert [c for c in collected if c != PING_SSE_BYTES] == chunks
+        assert [c for c in collected if c != STREAM_SSE_KEEPALIVE_PING_BYTES] == chunks
 
     @pytest.mark.asyncio
     async def test_should_emit_pings_while_the_follow_up_stream_is_slow(self):
@@ -1023,8 +1023,8 @@ class TestAgenticStreamingIteratorHoldBack:
             collected.append(chunk)
 
         first_follow_up_index = collected.index(phase2_chunks[0])
-        assert collected[first_follow_up_index + 1] == PING_SSE_BYTES
-        assert [c for c in collected if c != PING_SSE_BYTES] == phase2_chunks
+        assert collected[first_follow_up_index + 1] == STREAM_SSE_KEEPALIVE_PING_BYTES
+        assert [c for c in collected if c != STREAM_SSE_KEEPALIVE_PING_BYTES] == phase2_chunks
 
     @pytest.mark.asyncio
     async def test_should_propagate_follow_up_stream_error(self):
@@ -1074,7 +1074,7 @@ class TestAgenticStreamingIteratorHoldBack:
         )
 
         first = await iterator.__anext__()
-        assert first == PING_SSE_BYTES
+        assert first == STREAM_SSE_KEEPALIVE_PING_BYTES
         assert iterator._drain_task is not None
 
         await iterator.aclose()
diff --git a/tests/test_litellm/proxy/test_budget_reservation.py b/tests/test_litellm/proxy/test_budget_reservation.py
index 34adb4d2091..e9a0e80752d 100644
--- a/tests/test_litellm/proxy/test_budget_reservation.py
+++ b/tests/test_litellm/proxy/test_budget_reservation.py
@@ -7,6 +7,7 @@ from fastapi import HTTPException
 
 import litellm
 from litellm.caching.dual_cache import DualCache
+from litellm.constants import STREAM_SSE_KEEPALIVE_PING_BYTES
 from litellm.proxy._types import (
     LiteLLM_BudgetTable,
     LiteLLM_EndUserTable,
@@ -2453,6 +2454,33 @@ async def test_streaming_cancel_after_chunk_keeps_reservation(
     streaming_logging_obj._arelease_max_parallel_requests_on_disconnect.assert_awaited_once()
 
 
+@pytest.mark.asyncio
+async def test_streaming_cancel_after_only_keepalive_pings_reconciles_to_input_cost(
+    spend_counter_state,
+):
+    counter_cache, key_cache = spend_counter_state
+    proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache)
+    valid_token, reservation = await _reserve_for_stream(
+        counter_cache, key_cache, proxy_logging_obj, "key-cancel-after-ping"
+    )
+
+    async def cancel_after_ping(user_api_key_dict, response, request_data):
+        yield STREAM_SSE_KEEPALIVE_PING_BYTES
+        raise asyncio.CancelledError()
+
+    generator, streaming_logging_obj = _drive_streaming_cancel(valid_token, cancel_after_ping)
+    received = []
+    with pytest.raises(asyncio.CancelledError):
+        async for chunk in generator:
+            received.append(chunk)
+
+    assert received == [STREAM_SSE_KEEPALIVE_PING_BYTES]
+    assert counter_cache.in_memory_cache.get_cache(
+        key="spend:key:key-cancel-after-ping"
+    ) == pytest.approx(0.5)
+    assert reservation["finalized"] is True
+
+
 @pytest.mark.asyncio
 async def test_release_budget_reservation_on_cancel_swallows_release_errors():
     # If the release itself fails (e.g. Redis unavailable) it must not escape

From 2d1ee3aab2fe6a37c80085009f789416fe191d4e Mon Sep 17 00:00:00 2001
From: mateo <mateo@berri.ai>
Date: Sat, 8 Aug 2026 20:39:55 +0000
Subject: [PATCH 006/105] fix(proxy): keep the reservation when a disconnect
 happens while provider output is held back

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../messages/agentic_streaming_iterator.py    |  5 ++
 litellm/proxy/common_request_processing.py    |  9 ++-
 .../proxy/test_budget_reservation.py          | 63 +++++++++++++++++++
 3 files changed, 76 insertions(+), 1 deletion(-)

diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py
index d6f4e51a09a..3d3d3a12b17 100644
--- a/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py
+++ b/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py
@@ -207,6 +207,11 @@ class AgenticAnthropicStreamingIterator:
         self._replay_index = 0
         self._error_emitted = False
 
+    @property
+    def has_buffered_provider_output(self) -> bool:
+        """Whether provider output was received but withheld from the client behind keepalive pings."""
+        return self._hold_back and bool(self._collected_bytes)
+
     def __aiter__(self):
         return self
 
diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py
index 2607ff411a9..3aeb7729c81 100644
--- a/litellm/proxy/common_request_processing.py
+++ b/litellm/proxy/common_request_processing.py
@@ -40,6 +40,9 @@ from litellm.litellm_core_utils.llm_response_utils.get_headers import (
     get_response_headers,
 )
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
+from litellm.llms.anthropic.experimental_pass_through.messages.agentic_streaming_iterator import (
+    AgenticAnthropicStreamingIterator,
+)
 from litellm.proxy._types import ProxyException, UserAPIKeyAuth
 from litellm.proxy.auth.auth_checks import can_key_call_resolved_model
 from litellm.proxy.auth.auth_utils import check_response_size_is_safe
@@ -95,6 +98,10 @@ _CLIENT_DISCONNECTED_ERROR_INFORMATION: Final[StandardLoggingPayloadErrorInforma
 }
 
 
+def _withheld_provider_output(response: object) -> bool:
+    return isinstance(response, AgenticAnthropicStreamingIterator) and response.has_buffered_provider_output
+
+
 def _should_return_raw_model_name(request_data: dict[str, object]) -> bool:
     return any(
         isinstance(metadata, dict) and metadata.get(RETURN_RAW_MODEL_NAME_METADATA_KEY) is True
@@ -2970,7 +2977,7 @@ class ProxyBaseLLMRequestProcessing:
             # only sees GeneratorExit on GC) cannot own the refund.
             if not stream_completed:
                 client_disconnected = True
-            if not delivered_chunk:
+            if not delivered_chunk and not _withheld_provider_output(response):
                 from litellm.proxy.spend_tracking.budget_reservation import (
                     release_budget_reservation_on_cancel,
                 )
diff --git a/tests/test_litellm/proxy/test_budget_reservation.py b/tests/test_litellm/proxy/test_budget_reservation.py
index e9a0e80752d..c3210dd7f4d 100644
--- a/tests/test_litellm/proxy/test_budget_reservation.py
+++ b/tests/test_litellm/proxy/test_budget_reservation.py
@@ -8,6 +8,9 @@ from fastapi import HTTPException
 import litellm
 from litellm.caching.dual_cache import DualCache
 from litellm.constants import STREAM_SSE_KEEPALIVE_PING_BYTES
+from litellm.llms.anthropic.experimental_pass_through.messages.agentic_streaming_iterator import (
+    AgenticAnthropicStreamingIterator,
+)
 from litellm.proxy._types import (
     LiteLLM_BudgetTable,
     LiteLLM_EndUserTable,
@@ -2376,6 +2379,11 @@ async def _reserve_for_stream(counter_cache, key_cache, proxy_logging_obj, token
     return valid_token, reservation
 
 
+async def _never_ending_stream():
+    yield b'event: message_start\ndata: {"type": "message_start"}\n\n'
+    await asyncio.sleep(30)
+
+
 def _drive_streaming_cancel(valid_token, iterator_hook):
     streaming_logging_obj = MagicMock()
     streaming_logging_obj.async_post_call_streaming_iterator_hook = iterator_hook
@@ -2481,6 +2489,61 @@ async def test_streaming_cancel_after_only_keepalive_pings_reconciles_to_input_c
     assert reservation["finalized"] is True
 
 
+@pytest.mark.asyncio
+async def test_streaming_cancel_while_holding_back_provider_output_keeps_reservation(
+    spend_counter_state,
+):
+    counter_cache, key_cache = spend_counter_state
+    proxy_logging_obj = ProxyLogging(user_api_key_cache=key_cache)
+    valid_token, reservation = await _reserve_for_stream(
+        counter_cache, key_cache, proxy_logging_obj, "key-cancel-held-back"
+    )
+
+    held_back = AgenticAnthropicStreamingIterator(
+        completion_stream=_never_ending_stream(),
+        http_handler=MagicMock(),
+        model="claude-haiku-4-5",
+        messages=[],
+        anthropic_messages_provider_config=MagicMock(),
+        anthropic_messages_optional_request_params={},
+        logging_obj=MagicMock(),
+        custom_llm_provider="anthropic",
+        kwargs={},
+        hold_back=True,
+        server_fulfilled_tool_names=frozenset({"headroom_retrieve"}),
+        ping_interval_seconds=0.01,
+    )
+
+    async def ping_then_cancel(user_api_key_dict, response, request_data):
+        yield await response.__anext__()
+        while not response.has_buffered_provider_output:
+            yield await response.__anext__()
+        raise asyncio.CancelledError()
+
+    streaming_logging_obj = MagicMock()
+    streaming_logging_obj.async_post_call_streaming_iterator_hook = ping_then_cancel
+    streaming_logging_obj._arelease_max_parallel_requests_on_disconnect = AsyncMock()
+    generator = ProxyBaseLLMRequestProcessing.async_streaming_data_generator(
+        response=held_back,
+        user_api_key_dict=valid_token,
+        request_data=_request_body(),
+        proxy_logging_obj=streaming_logging_obj,
+        serialize_chunk=lambda chunk: chunk,
+        serialize_error=lambda exc: str(exc),
+    )
+
+    received = []
+    with pytest.raises(asyncio.CancelledError):
+        async for chunk in generator:
+            received.append(chunk)
+
+    assert received and received == [STREAM_SSE_KEEPALIVE_PING_BYTES] * len(received)
+    assert counter_cache.in_memory_cache.get_cache(
+        key="spend:key:key-cancel-held-back"
+    ) == pytest.approx(2.0)
+    assert reservation.get("finalized") is not True
+
+
 @pytest.mark.asyncio
 async def test_release_budget_reservation_on_cancel_swallows_release_errors():
     # If the release itself fails (e.g. Redis unavailable) it must not escape

From b8680e6baed05863712a4c57a10b128ecd95475a Mon Sep 17 00:00:00 2001
From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Wed, 19 Aug 2026 18:22:31 +0000
Subject: [PATCH 007/105] fix(ui): render tag-based guardrail mode instead of
 crashing guardrails page

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../_components/guardrailTableColumns.tsx     | 13 +++++---
 .../_components/guardrail_info.test.tsx       | 30 +++++++++++++++++++
 .../guardrails/_components/guardrail_info.tsx |  5 ++--
 .../guardrail_info_helpers.test.tsx           | 29 ++++++++++++++++++
 .../_components/guardrail_info_helpers.tsx    | 13 ++++++++
 .../_components/guardrail_table.test.tsx      | 12 ++++++++
 .../src/components/guardrails/types.ts        |  7 ++++-
 7 files changed, 102 insertions(+), 7 deletions(-)

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrailTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrailTableColumns.tsx
index ec3d05a6907..53f1b1a03d3 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrailTableColumns.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrailTableColumns.tsx
@@ -15,7 +15,7 @@ import {
 } from "@/components/ui/dropdown-menu";
 import { cn } from "@/lib/cva.config";
 
-import { getGuardrailLogoAndName } from "./guardrail_info_helpers";
+import { formatGuardrailMode, getGuardrailLogoAndName } from "./guardrail_info_helpers";
 import { Logo } from "@/components/molecules/logo/Logo";
 
 const CONFIG_DELETE_HINT = "Config guardrails are defined in the config file and cannot be deleted from the dashboard.";
@@ -117,9 +117,14 @@ export const getGuardrailTableColumns = ({
     header: "Mode",
     size: 130,
     enableSorting: false,
-    cell: ({ row }) => (
-      <span className="font-mono text-xs text-muted-foreground">{row.original.litellm_params.mode}</span>
-    ),
+    cell: ({ row }) => {
+      const mode = formatGuardrailMode(row.original.litellm_params.mode);
+      return (
+        <span className="font-mono text-xs text-muted-foreground" title={mode || undefined}>
+          {mode || "-"}
+        </span>
+      );
+    },
   },
   {
     id: "default_on",
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx
index b6ee130d50a..3f6317ed366 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx
@@ -82,6 +82,36 @@ describe("Guardrail Info", () => {
     expect(getByText("Settings")).toBeInTheDocument();
   });
 
+  it("should render a tag-based mode object rather than crashing the detail view", async () => {
+    vi.mocked(networking.getGuardrailInfo).mockResolvedValue({
+      guardrail_id: "123",
+      guardrail_name: "Test Guardrail",
+      litellm_params: {
+        guardrail: "bedrock",
+        mode: { tags: { "Service-Type: internal-service": "post_call" }, default: ["pre_call", "post_call"] },
+        default_on: true,
+      },
+      created_at: "2024-01-01T00:00:00Z",
+      updated_at: "2024-01-01T00:00:00Z",
+      guardrail_definition_location: "database",
+    });
+
+    vi.mocked(networking.getGuardrailUISettings).mockResolvedValue({
+      supported_entities: [],
+      supported_actions: [],
+      pii_entity_categories: [],
+      supported_modes: ["pre_call", "post_call"],
+    });
+
+    vi.mocked(networking.getGuardrailProviderSpecificParams).mockResolvedValue({});
+
+    const { findAllByText } = render(
+      <GuardrailInfoView guardrailId="123" onClose={() => {}} accessToken="123" isAdmin={true} />,
+    );
+
+    expect(await findAllByText("pre_call, post_call (tag-based)")).not.toHaveLength(0);
+  });
+
   it("should render the provider logo from the bundled guardrail logo map", async () => {
     vi.mocked(networking.getGuardrailInfo).mockResolvedValue({
       guardrail_id: "123",
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx
index e80ddac932f..5e476a8accd 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx
@@ -35,6 +35,7 @@ import {
 import ContentFilterManager, { formatContentFilterDataForAPI } from "./content_filter/ContentFilterManager";
 import CustomCodeModal, { EditGuardrailData } from "./custom_code/CustomCodeModal";
 import {
+  formatGuardrailMode,
   getGuardrailLogoAndName,
   guardrail_provider_map,
   skipSystemMessageToChoice,
@@ -559,7 +560,7 @@ const GuardrailInfoView: React.FC<GuardrailInfoProps> = ({ guardrailId, onClose,
               <Card className="block p-6">
                 <p>Mode</p>
                 <div className="mt-2">
-                  <h3 className="text-lg font-medium">{guardrailData.litellm_params?.mode || "-"}</h3>
+                  <h3 className="text-lg font-medium">{formatGuardrailMode(guardrailData.litellm_params?.mode) || "-"}</h3>
                   <Badge variant={guardrailData.litellm_params?.default_on ? "secondary" : "outline"}>
                     {guardrailData.litellm_params?.default_on ? "Default On" : "Default Off"}
                   </Badge>
@@ -852,7 +853,7 @@ const GuardrailInfoView: React.FC<GuardrailInfoProps> = ({ guardrailId, onClose,
                     </div>
                     <div>
                       <p className="font-medium">Mode</p>
-                      <div>{guardrailData.litellm_params?.mode || "-"}</div>
+                      <div>{formatGuardrailMode(guardrailData.litellm_params?.mode) || "-"}</div>
                     </div>
                     <div>
                       <p className="font-medium">Default On</p>
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.test.tsx
index ec910673b8f..c5e07fe9624 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.test.tsx
@@ -14,6 +14,7 @@ import {
   choiceToSkipSystemForCreate,
   skipToolMessageToChoice,
   choiceToSkipToolForCreate,
+  formatGuardrailMode,
 } from "./guardrail_info_helpers";
 
 describe("guardrail_info_helpers", () => {
@@ -210,6 +211,34 @@ describe("guardrail_info_helpers", () => {
     });
   });
 
+  describe("formatGuardrailMode", () => {
+    it("renders a single mode and a list of modes", () => {
+      expect(formatGuardrailMode("pre_call")).toBe("pre_call");
+      expect(formatGuardrailMode(["pre_call", "post_call"])).toBe("pre_call, post_call");
+    });
+
+    it("flattens a tag-based mode object into deduped modes instead of returning it verbatim", () => {
+      const mode = {
+        tags: { "Service-Type: internal-service": "post_call", "Service-Type: batch": ["during_call", "post_call"] },
+        default: ["pre_call", "post_call"],
+      };
+
+      expect(formatGuardrailMode(mode)).toBe("pre_call, post_call, during_call (tag-based)");
+    });
+
+    it("handles a tag-based mode with no default and with no tags", () => {
+      expect(formatGuardrailMode({ tags: { "team: a": "post_call" } })).toBe("post_call (tag-based)");
+      expect(formatGuardrailMode({ default: "pre_call" })).toBe("pre_call (tag-based)");
+    });
+
+    it("returns an empty string for missing or unusable modes", () => {
+      expect(formatGuardrailMode(undefined)).toBe("");
+      expect(formatGuardrailMode(null)).toBe("");
+      expect(formatGuardrailMode({})).toBe("");
+      expect(formatGuardrailMode({ tags: {}, default: null })).toBe("");
+    });
+  });
+
   describe("skipSystemMessageToChoice / choiceToSkipSystemForCreate", () => {
     it("maps API values to form choices and back for create", () => {
       expect(skipSystemMessageToChoice(undefined)).toBe("inherit");
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx
index 12aaba0d696..c12529e6326 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx
@@ -110,6 +110,19 @@ export const toModeArray = (raw: unknown): string[] => {
   return [];
 };
 
+// Turns a guardrail mode into a renderable string. A mode is a single mode, a list of modes, or a
+// tag-based `{ tags, default }` object, which React refuses to render as a child
+export const formatGuardrailMode = (raw: unknown): string => {
+  const flat: string[] = toModeArray(raw);
+  if (flat.length > 0) return flat.join(", ");
+  if (raw === null || typeof raw !== "object") return "";
+
+  const { tags, default: fallback } = raw as { tags?: Record<string, unknown>; default?: unknown };
+  const tagged: string[] = tags && typeof tags === "object" ? Object.values(tags).flatMap(toModeArray) : [];
+  const modes: string[] = Array.from(new Set([...toModeArray(fallback), ...tagged]));
+  return modes.length > 0 ? `${modes.join(", ")} (tag-based)` : "";
+};
+
 // Resolves the supported modes for the selected provider, falling back to the global list
 export const getSupportedModesForProvider = (
   settings: { supported_modes?: string[]; supported_modes_by_provider?: Record<string, string[]> } | null,
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_table.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_table.test.tsx
index ee619dc7468..561a89a191a 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_table.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_table.test.tsx
@@ -46,6 +46,18 @@ describe("GuardrailTable", () => {
     expect(screen.getByText("m")).toBeInTheDocument();
   });
 
+  it("renders a tag-based mode object instead of crashing the table", () => {
+    const guardrail = makeGuardrail({
+      litellm_params: {
+        guardrail: "bedrock",
+        mode: { tags: { "Service-Type: internal-service": "post_call" }, default: ["pre_call", "post_call"] },
+        default_on: true,
+      },
+    });
+    render(<GuardrailTable guardrailsList={[guardrail]} {...baseProps} />);
+    expect(screen.getByText("pre_call, post_call (tag-based)")).toBeInTheDocument();
+  });
+
   it("deletes a DB guardrail through the actions menu", async () => {
     const user = userEvent.setup();
     const onDeleteClick = vi.fn();
diff --git a/ui/litellm-dashboard/src/components/guardrails/types.ts b/ui/litellm-dashboard/src/components/guardrails/types.ts
index e8ed27d9e45..0f5ce1c883d 100644
--- a/ui/litellm-dashboard/src/components/guardrails/types.ts
+++ b/ui/litellm-dashboard/src/components/guardrails/types.ts
@@ -18,12 +18,17 @@ export interface PiiConfigurationProps {
   entityCategories?: PiiEntityCategory[];
 }
 
+export type GuardrailMode =
+  | string
+  | string[]
+  | { tags?: Record<string, string | string[]>; default?: string | string[] | null };
+
 export interface Guardrail {
   guardrail_id: string;
   guardrail_name: string | null;
   litellm_params: {
     guardrail: string;
-    mode: string;
+    mode: GuardrailMode;
     default_on: boolean;
     pii_entities_config?: { [key: string]: string };
     [key: string]: any;

From 881aa2080871052a2173f7b3352df39fc0e61e03 Mon Sep 17 00:00:00 2001
From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Wed, 19 Aug 2026 18:31:42 +0000
Subject: [PATCH 008/105] fix(ui): format tag-based guardrail mode in delete
 modal, playground, and policy picker

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../guardrails/_components/GuardrailTestPlayground.tsx    | 8 ++++++--
 .../guardrails/_components/GuardrailsPanel.tsx            | 4 ++--
 .../(dashboard)/guardrails/_components/guardrail_info.tsx | 4 +++-
 .../policies/_components/guardrail_selection_modal.tsx    | 5 ++++-
 4 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailTestPlayground.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailTestPlayground.tsx
index fd8ed22867b..c64b5d7cb5c 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailTestPlayground.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailTestPlayground.tsx
@@ -6,13 +6,15 @@ import { toast } from "@/lib/toast";
 import { Card, CardContent } from "@/components/ui/card";
 import { InputGroup, InputGroupAddon, InputGroupInput } from "@/components/ui/input-group";
 import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner";
+import { GuardrailMode } from "@/components/guardrails/types";
+import { formatGuardrailMode } from "./guardrail_info_helpers";
 
 interface GuardrailItem {
   guardrail_id?: string;
   guardrail_name: string | null;
   litellm_params: {
     guardrail: string;
-    mode: string;
+    mode: GuardrailMode;
     default_on: boolean;
   };
   guardrail_info: Record<string, any> | null;
@@ -171,7 +173,9 @@ const GuardrailTestPlayground: React.FC<GuardrailTestPlaygroundProps> = ({
                           </div>
                           <div>
                             <span className="font-medium">Mode: </span>
-                            <span className="text-muted-foreground">{guardrail.litellm_params.mode}</span>
+                            <span className="text-muted-foreground">
+                              {formatGuardrailMode(guardrail.litellm_params.mode)}
+                            </span>
                           </div>
                         </div>
                       </li>
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailsPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailsPanel.tsx
index b4c29bd9c40..7e59abf8e3d 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailsPanel.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailsPanel.tsx
@@ -18,7 +18,7 @@ import GuardrailTestPlayground from "./GuardrailTestPlayground";
 import { toast } from "@/lib/toast";
 import { Guardrail } from "@/components/guardrails/types";
 import DeleteResourceModal from "@/components/common_components/DeleteResourceModal";
-import { getGuardrailLogoAndName } from "./guardrail_info_helpers";
+import { formatGuardrailMode, getGuardrailLogoAndName } from "./guardrail_info_helpers";
 import { CustomCodeModal } from "./custom_code";
 import GuardrailGarden from "./guardrail_garden";
 import { TeamGuardrailsTab } from "./TeamGuardrailsTab";
@@ -211,7 +211,7 @@ const GuardrailsPanel: React.FC<GuardrailsPanelProps> = ({ accessToken, userRole
                   { label: "Name", value: guardrailToDelete?.guardrail_name },
                   { label: "ID", value: guardrailToDelete?.guardrail_id, code: true },
                   { label: "Provider", value: providerDisplayName },
-                  { label: "Mode", value: guardrailToDelete?.litellm_params.mode },
+                  { label: "Mode", value: formatGuardrailMode(guardrailToDelete?.litellm_params.mode) },
                   {
                     label: "Default On",
                     value: guardrailToDelete?.litellm_params.default_on ? "Yes" : "No",
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx
index 5e476a8accd..d4a1885146d 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx
@@ -560,7 +560,9 @@ const GuardrailInfoView: React.FC<GuardrailInfoProps> = ({ guardrailId, onClose,
               <Card className="block p-6">
                 <p>Mode</p>
                 <div className="mt-2">
-                  <h3 className="text-lg font-medium">{formatGuardrailMode(guardrailData.litellm_params?.mode) || "-"}</h3>
+                  <h3 className="text-lg font-medium">
+                    {formatGuardrailMode(guardrailData.litellm_params?.mode) || "-"}
+                  </h3>
                   <Badge variant={guardrailData.litellm_params?.default_on ? "secondary" : "outline"}>
                     {guardrailData.litellm_params?.default_on ? "Default On" : "Default Off"}
                   </Badge>
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/guardrail_selection_modal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/guardrail_selection_modal.tsx
index 0b439462c1a..f87155db719 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/guardrail_selection_modal.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/guardrail_selection_modal.tsx
@@ -12,6 +12,7 @@ import {
 } from "@/components/ui/dialog";
 import { Separator } from "@/components/ui/separator";
 import { CheckCircle2, Info } from "lucide-react";
+import { formatGuardrailMode } from "@/app/(dashboard)/guardrails/_components/guardrail_info_helpers";
 
 interface GuardrailInfo {
   guardrail_name: string;
@@ -163,7 +164,9 @@ const GuardrailSelectionModal: React.FC<GuardrailSelectionModalProps> = ({
                     {/* Show guardrail type and mode */}
                     <div className="flex gap-2 mt-2">
                       <Badge variant="outline">{guardrail.definition?.litellm_params?.guardrail || "unknown"}</Badge>
-                      <Badge variant="secondary">{guardrail.definition?.litellm_params?.mode || "unknown"}</Badge>
+                      <Badge variant="secondary">
+                        {formatGuardrailMode(guardrail.definition?.litellm_params?.mode) || "unknown"}
+                      </Badge>
                       {guardrail.definition?.litellm_params?.patterns && (
                         <Badge variant="secondary">
                           {guardrail.definition.litellm_params.patterns.length} pattern(s)

From f80cb0d9f8e37539b39bf6412ef7f673c2074e58 Mon Sep 17 00:00:00 2001
From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Wed, 19 Aug 2026 18:33:06 +0000
Subject: [PATCH 009/105] refactor(ui): drop redundant comment above guardrail
 mode formatter

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../guardrails/_components/guardrail_info_helpers.tsx           | 2 --
 1 file changed, 2 deletions(-)

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx
index c12529e6326..83038b8e0e7 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx
@@ -110,8 +110,6 @@ export const toModeArray = (raw: unknown): string[] => {
   return [];
 };
 
-// Turns a guardrail mode into a renderable string. A mode is a single mode, a list of modes, or a
-// tag-based `{ tags, default }` object, which React refuses to render as a child
 export const formatGuardrailMode = (raw: unknown): string => {
   const flat: string[] = toModeArray(raw);
   if (flat.length > 0) return flat.join(", ");

From 9e86cfa7e994edd3ac77456a7b0edb974e8012ff Mon Sep 17 00:00:00 2001
From: milan <milan@berri.ai>
Date: Fri, 21 Aug 2026 01:56:02 +0000
Subject: [PATCH 010/105] fix(auth): support wildcard prefixes in jwt
 team_allowed_routes

team_allowed_routes and admin_allowed_routes only matched exact strings or named route groups, so a whole prefix of pass-through endpoints had to be listed route by route in config. Match trailing-wildcard patterns with the same helper the key-level allowed_routes check uses, so "/prefix/*" covers endpoints registered later.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 basedpyright-code-budget.json                 |  2 +-
 litellm/proxy/auth/auth_checks.py             |  5 +-
 litellm/proxy/auth/auth_utils.py              |  2 +-
 litellm/proxy/auth/route_checks.py            | 10 +--
 litellm/proxy/policy_engine/policy_matcher.py |  4 +-
 .../policy_engine/policy_resolve_endpoints.py |  8 +-
 .../proxy/auth/test_auth_checks.py            | 79 +++++++++++++++++++
 .../policies/_components/scope_validation.ts  |  2 +-
 8 files changed, 96 insertions(+), 16 deletions(-)

diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json
index 776aecbd883..46a06f77861 100644
--- a/basedpyright-code-budget.json
+++ b/basedpyright-code-budget.json
@@ -84,7 +84,7 @@
     "limit": 56
   },
   "reportPrivateUsage": {
-    "limit": 1823
+    "limit": 1817
   },
   "reportRedeclaration": {
     "limit": 8
diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py
index 12d6b44a648..9d8eedaa7dc 100644
--- a/litellm/proxy/auth/auth_checks.py
+++ b/litellm/proxy/auth/auth_checks.py
@@ -1128,7 +1128,8 @@ def _allowed_routes_check(user_route: str, allowed_routes: list) -> bool:
 
     Parameters:
     - user_route: str - the route the user is trying to call
-    - allowed_routes: List[str|LiteLLMRoutes] - the list of allowed routes for the user.
+    - allowed_routes: List[str|LiteLLMRoutes] - the list of allowed routes for the user. Entries are a route group name
+      (e.g. "openai_routes"), an exact route, or a trailing-wildcard prefix (e.g. "/tempus/*").
     """
     from starlette.routing import compile_path
 
@@ -1138,7 +1139,7 @@ def _allowed_routes_check(user_route: str, allowed_routes: list) -> bool:
                 regex, _, _ = compile_path(template)
                 if regex.match(user_route):
                     return True
-        elif allowed_route == user_route:
+        elif RouteChecks.route_matches_wildcard_pattern(route=user_route, pattern=allowed_route):
             return True
     return False
 
diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py
index ce662ee0374..1e6d8137d53 100644
--- a/litellm/proxy/auth/auth_utils.py
+++ b/litellm/proxy/auth/auth_utils.py
@@ -608,7 +608,7 @@ def route_in_additonal_public_routes(current_route: str):
 
         # Check wildcard patterns
         for route_pattern in routes_defined:
-            if RouteChecks._route_matches_wildcard_pattern(route=current_route, pattern=route_pattern):
+            if RouteChecks.route_matches_wildcard_pattern(route=current_route, pattern=route_pattern):
                 return True
 
         return False
diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py
index cea21ca088b..4dba2497bb9 100644
--- a/litellm/proxy/auth/route_checks.py
+++ b/litellm/proxy/auth/route_checks.py
@@ -181,7 +181,7 @@ class RouteChecks:
 
         # check if wildcard pattern is allowed
         for allowed_route in valid_token.allowed_routes:
-            if RouteChecks._route_matches_wildcard_pattern(route=route, pattern=allowed_route):
+            if RouteChecks.route_matches_wildcard_pattern(route=route, pattern=allowed_route):
                 return True
 
         if denied_auth_enforced_pass_through_route:
@@ -329,7 +329,7 @@ class RouteChecks:
                     route_allowed = True
                     break
 
-                if RouteChecks._route_matches_wildcard_pattern(route=route, pattern=allowed_route):
+                if RouteChecks.route_matches_wildcard_pattern(route=route, pattern=allowed_route):
                     route_allowed = True
                     break
 
@@ -397,7 +397,7 @@ class RouteChecks:
                     return True
             # Check for wildcard patterns like "/containers/*"
             if RouteChecks._is_wildcard_pattern(pattern=openai_route):
-                if RouteChecks._route_matches_wildcard_pattern(route=route, pattern=openai_route):
+                if RouteChecks.route_matches_wildcard_pattern(route=route, pattern=openai_route):
                     return True
 
         # Check for Google routes with placeholders like "/v1beta/models/{model_name}:generateContent"
@@ -517,7 +517,7 @@ class RouteChecks:
         return pattern.endswith("*")
 
     @staticmethod
-    def _route_matches_wildcard_pattern(route: str, pattern: str) -> bool:
+    def route_matches_wildcard_pattern(route: str, pattern: str) -> bool:
         """
         Check if route matches the wildcard pattern
 
@@ -594,7 +594,7 @@ class RouteChecks:
         # e.g calling /anthropic/v1/messages is allowed if allowed_routes has /anthropic/*
         #########################################################
         if any(
-            RouteChecks._route_matches_wildcard_pattern(route=route, pattern=allowed_route)
+            RouteChecks.route_matches_wildcard_pattern(route=route, pattern=allowed_route)
             for allowed_route in allowed_routes
             if RouteChecks._is_wildcard_pattern(pattern=allowed_route)
         ):
diff --git a/litellm/proxy/policy_engine/policy_matcher.py b/litellm/proxy/policy_engine/policy_matcher.py
index f66dc4e7bbe..001e4115374 100644
--- a/litellm/proxy/policy_engine/policy_matcher.py
+++ b/litellm/proxy/policy_engine/policy_matcher.py
@@ -30,7 +30,7 @@ class PolicyMatcher:
         """
         Check if a value matches any of the given patterns.
 
-        Uses the existing RouteChecks._route_matches_wildcard_pattern helper.
+        Uses the existing RouteChecks.route_matches_wildcard_pattern helper.
 
         Args:
             value: The value to check (e.g., team alias, key alias, model)
@@ -45,7 +45,7 @@ class PolicyMatcher:
 
         for pattern in patterns:
             # Use existing wildcard pattern matching helper
-            if RouteChecks._route_matches_wildcard_pattern(route=value, pattern=pattern):
+            if RouteChecks.route_matches_wildcard_pattern(route=value, pattern=pattern):
                 return True
 
         return False
diff --git a/litellm/proxy/policy_engine/policy_resolve_endpoints.py b/litellm/proxy/policy_engine/policy_resolve_endpoints.py
index 346586c1e5a..70b98933d0f 100644
--- a/litellm/proxy/policy_engine/policy_resolve_endpoints.py
+++ b/litellm/proxy/policy_engine/policy_resolve_endpoints.py
@@ -100,7 +100,7 @@ def _filter_keys_by_tags(keys: list, tag_patterns: list) -> tuple:
         key_alias = key.key_alias or ""
         key_tags = _get_tags_from_metadata(key.metadata, getattr(key, "metadata_json", None))
         if key_tags and any(
-            RouteChecks._route_matches_wildcard_pattern(route=tag, pattern=pat)
+            RouteChecks.route_matches_wildcard_pattern(route=tag, pattern=pat)
             for tag in key_tags
             for pat in tag_patterns
         ):
@@ -123,7 +123,7 @@ def _filter_teams_by_tags(teams: list, tag_patterns: list) -> tuple:
         team_alias = team.team_alias or ""
         team_tags = _get_tags_from_metadata(team.metadata)
         if team_tags and any(
-            RouteChecks._route_matches_wildcard_pattern(route=tag, pattern=pat)
+            RouteChecks.route_matches_wildcard_pattern(route=tag, pattern=pat)
             for tag in team_tags
             for pat in tag_patterns
         ):
@@ -152,7 +152,7 @@ async def _find_affected_by_team_patterns(
     for team in all_teams:
         team_alias = team.team_alias or ""
         if team_alias and any(
-            RouteChecks._route_matches_wildcard_pattern(route=team_alias, pattern=pat) for pat in team_patterns
+            RouteChecks.route_matches_wildcard_pattern(route=team_alias, pattern=pat) for pat in team_patterns
         ):
             if team_alias not in existing_teams:
                 new_teams.append(team_alias)
@@ -190,7 +190,7 @@ async def _find_affected_keys_by_alias(prisma_client: object, key_patterns: list
     for key in keys:
         key_alias = key.key_alias or ""
         if key_alias and any(
-            RouteChecks._route_matches_wildcard_pattern(route=key_alias, pattern=pat) for pat in key_patterns
+            RouteChecks.route_matches_wildcard_pattern(route=key_alias, pattern=pat) for pat in key_patterns
         ):
             if key_alias not in existing_keys:
                 affected.append(key_alias)
diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py
index 6b40fa1b324..0b174cda9d5 100644
--- a/tests/test_litellm/proxy/auth/test_auth_checks.py
+++ b/tests/test_litellm/proxy/auth/test_auth_checks.py
@@ -6897,3 +6897,82 @@ def test_model_has_no_cost_mapping_alias_to_a_group_priced_through_model_info_is
     router = _router_with_a_group_priced_through_model_info()
 
     assert model_has_no_cost_mapping(model="model-info-priced-alias", llm_router=router) is False
+
+@pytest.mark.parametrize(
+    "user_route, expected",
+    [
+        ("/tempus/v1/chat/completions", True),
+        ("/tempus/newly-registered-model/predict", True),
+        ("/tempus-other/v1/chat/completions", False),
+        ("/anthropic/v1/messages", False),
+    ],
+)
+def test_team_allowed_routes_wildcard_prefix_matches_unregistered_passthrough_routes(user_route, expected):
+    """A `/prefix/*` entry in `team_allowed_routes` must cover every route under that prefix, so
+    passthrough endpoints registered after the proxy config was written are reachable without an
+    exact-route config change."""
+    from litellm.proxy._types import LiteLLM_JWTAuth
+    from litellm.proxy.auth.auth_checks import allowed_routes_check
+
+    assert (
+        allowed_routes_check(
+            user_role=LitellmUserRoles.TEAM,
+            user_route=user_route,
+            litellm_proxy_roles=LiteLLM_JWTAuth(team_allowed_routes=["/tempus/*"]),
+        )
+        is expected
+    )
+
+
+def test_team_allowed_routes_exact_route_does_not_become_a_prefix_grant():
+    from litellm.proxy._types import LiteLLM_JWTAuth
+    from litellm.proxy.auth.auth_checks import allowed_routes_check
+
+    roles = LiteLLM_JWTAuth(team_allowed_routes=["/tempus/model-a"])
+
+    assert (
+        allowed_routes_check(user_role=LitellmUserRoles.TEAM, user_route="/tempus/model-a", litellm_proxy_roles=roles)
+        is True
+    )
+    assert (
+        allowed_routes_check(user_role=LitellmUserRoles.TEAM, user_route="/tempus/model-b", litellm_proxy_roles=roles)
+        is False
+    )
+
+
+def test_admin_allowed_routes_wildcard_prefix_is_honored():
+    from litellm.proxy._types import LiteLLM_JWTAuth
+    from litellm.proxy.auth.auth_checks import allowed_routes_check
+
+    roles = LiteLLM_JWTAuth(admin_allowed_routes=["/tempus/*"])
+
+    assert (
+        allowed_routes_check(
+            user_role=LitellmUserRoles.PROXY_ADMIN, user_route="/tempus/anything", litellm_proxy_roles=roles
+        )
+        is True
+    )
+    assert (
+        allowed_routes_check(
+            user_role=LitellmUserRoles.PROXY_ADMIN, user_route="/other/anything", litellm_proxy_roles=roles
+        )
+        is False
+    )
+
+
+def test_team_allowed_routes_named_route_group_still_resolves():
+    from litellm.proxy._types import LiteLLM_JWTAuth
+    from litellm.proxy.auth.auth_checks import allowed_routes_check
+
+    roles = LiteLLM_JWTAuth(team_allowed_routes=["openai_routes"])
+
+    assert (
+        allowed_routes_check(
+            user_role=LitellmUserRoles.TEAM, user_route="/v1/chat/completions", litellm_proxy_roles=roles
+        )
+        is True
+    )
+    assert (
+        allowed_routes_check(user_role=LitellmUserRoles.TEAM, user_route="/key/generate", litellm_proxy_roles=roles)
+        is False
+    )
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/scope_validation.ts b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/scope_validation.ts
index 7c49117088c..53a76dc5a1c 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/scope_validation.ts
+++ b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/scope_validation.ts
@@ -1,4 +1,4 @@
-// Mirrors request-time matching (RouteChecks._route_matches_wildcard_pattern): only a
+// Mirrors request-time matching (RouteChecks.route_matches_wildcard_pattern): only a
 // trailing "*" is a wildcard (prefix match). Anything else - including a "?" or a
 // non-trailing "*" - is compared by exact equality when a request is matched, so it is
 // treated as a concrete alias that must exist.

From 07416344cc8865c1867c51dd733582e04236aeef Mon Sep 17 00:00:00 2001
From: milan <milan@berri.ai>
Date: Fri, 21 Aug 2026 02:11:18 +0000
Subject: [PATCH 011/105] test(auth): use a generic route prefix in wildcard
 route tests

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/proxy/auth/auth_checks.py              |  2 +-
 .../proxy/auth/test_auth_checks.py             | 18 +++++++++---------
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py
index 9d8eedaa7dc..bf7a6a8f6c3 100644
--- a/litellm/proxy/auth/auth_checks.py
+++ b/litellm/proxy/auth/auth_checks.py
@@ -1129,7 +1129,7 @@ def _allowed_routes_check(user_route: str, allowed_routes: list) -> bool:
     Parameters:
     - user_route: str - the route the user is trying to call
     - allowed_routes: List[str|LiteLLMRoutes] - the list of allowed routes for the user. Entries are a route group name
-      (e.g. "openai_routes"), an exact route, or a trailing-wildcard prefix (e.g. "/tempus/*").
+      (e.g. "openai_routes"), an exact route, or a trailing-wildcard prefix (e.g. "/internal-models/*").
     """
     from starlette.routing import compile_path
 
diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py
index 0b174cda9d5..7fa16508054 100644
--- a/tests/test_litellm/proxy/auth/test_auth_checks.py
+++ b/tests/test_litellm/proxy/auth/test_auth_checks.py
@@ -6901,9 +6901,9 @@ def test_model_has_no_cost_mapping_alias_to_a_group_priced_through_model_info_is
 @pytest.mark.parametrize(
     "user_route, expected",
     [
-        ("/tempus/v1/chat/completions", True),
-        ("/tempus/newly-registered-model/predict", True),
-        ("/tempus-other/v1/chat/completions", False),
+        ("/internal-models/v1/chat/completions", True),
+        ("/internal-models/newly-registered-model/predict", True),
+        ("/internal-models-other/v1/chat/completions", False),
         ("/anthropic/v1/messages", False),
     ],
 )
@@ -6918,7 +6918,7 @@ def test_team_allowed_routes_wildcard_prefix_matches_unregistered_passthrough_ro
         allowed_routes_check(
             user_role=LitellmUserRoles.TEAM,
             user_route=user_route,
-            litellm_proxy_roles=LiteLLM_JWTAuth(team_allowed_routes=["/tempus/*"]),
+            litellm_proxy_roles=LiteLLM_JWTAuth(team_allowed_routes=["/internal-models/*"]),
         )
         is expected
     )
@@ -6928,14 +6928,14 @@ def test_team_allowed_routes_exact_route_does_not_become_a_prefix_grant():
     from litellm.proxy._types import LiteLLM_JWTAuth
     from litellm.proxy.auth.auth_checks import allowed_routes_check
 
-    roles = LiteLLM_JWTAuth(team_allowed_routes=["/tempus/model-a"])
+    roles = LiteLLM_JWTAuth(team_allowed_routes=["/internal-models/model-a"])
 
     assert (
-        allowed_routes_check(user_role=LitellmUserRoles.TEAM, user_route="/tempus/model-a", litellm_proxy_roles=roles)
+        allowed_routes_check(user_role=LitellmUserRoles.TEAM, user_route="/internal-models/model-a", litellm_proxy_roles=roles)
         is True
     )
     assert (
-        allowed_routes_check(user_role=LitellmUserRoles.TEAM, user_route="/tempus/model-b", litellm_proxy_roles=roles)
+        allowed_routes_check(user_role=LitellmUserRoles.TEAM, user_route="/internal-models/model-b", litellm_proxy_roles=roles)
         is False
     )
 
@@ -6944,11 +6944,11 @@ def test_admin_allowed_routes_wildcard_prefix_is_honored():
     from litellm.proxy._types import LiteLLM_JWTAuth
     from litellm.proxy.auth.auth_checks import allowed_routes_check
 
-    roles = LiteLLM_JWTAuth(admin_allowed_routes=["/tempus/*"])
+    roles = LiteLLM_JWTAuth(admin_allowed_routes=["/internal-models/*"])
 
     assert (
         allowed_routes_check(
-            user_role=LitellmUserRoles.PROXY_ADMIN, user_route="/tempus/anything", litellm_proxy_roles=roles
+            user_role=LitellmUserRoles.PROXY_ADMIN, user_route="/internal-models/anything", litellm_proxy_roles=roles
         )
         is True
     )

From cafc8c1455a7691b4cf2082bc809abeb3cc45af4 Mon Sep 17 00:00:00 2001
From: milan <milan@berri.ai>
Date: Fri, 21 Aug 2026 03:01:26 +0000
Subject: [PATCH 012/105] fix(proxy): store the actual selected model in spend
 logs for Azure Model Router

Co-authored-by: Filippo Mattia Menghi <filippo@menghi.dev>
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../spend_tracking/spend_tracking_utils.py    |  4 +-
 .../test_spend_tracking_utils.py              | 42 +++++++++++++++++++
 2 files changed, 45 insertions(+), 1 deletion(-)

diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py
index 0b56f0d8246..822f03873b3 100644
--- a/litellm/proxy/spend_tracking/spend_tracking_utils.py
+++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py
@@ -411,7 +411,9 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs
         or None
     )
     raw_model: Final = cast(str, kwargs.get("model") or "")
-    model_name: Final = reconstruct_model_name(raw_model, custom_llm_provider, metadata or {})
+    model_name: Final = (
+        standard_logging_payload.get("model") if standard_logging_payload is not None else None
+    ) or reconstruct_model_name(raw_model, custom_llm_provider, metadata or {})
 
     try:
         payload: Final[SpendLogsPayload] = SpendLogsPayload(
diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
index 9710dc44e99..b1a45fb84a3 100644
--- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
+++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
@@ -3241,3 +3241,45 @@ def test_get_logging_payload_failed_request_without_standard_logging_payload_lea
     assert payload["model_group"] == ""
     assert payload["api_base"] == ""
     assert payload["custom_llm_provider"] == ""
+
+
+def _model_router_spend_log_kwargs(slp_model: str | None) -> dict[str, Any]:
+    standard_logging_payload: Final = cast(
+        StandardLoggingPayload,
+        {
+            "model": slp_model,
+            "metadata": {},
+            "model_map_information": StandardLoggingModelInformation(
+                model_map_key="azure_ai/model_router", model_map_value=None
+            ),
+        },
+    )
+    return {
+        "model": "azure_ai/model_router/model-router",
+        "litellm_params": {"metadata": {"user_api_key": "sk-test-key"}},
+        "standard_logging_object": standard_logging_payload,
+    }
+
+
+@patch("litellm.proxy.proxy_server.master_key", None)
+@patch("litellm.proxy.proxy_server.general_settings", {})
+def test_get_logging_payload_uses_standard_logging_payload_model():
+    payload = get_logging_payload(
+        kwargs=_model_router_spend_log_kwargs(slp_model="azure_ai/gpt-5-mini"),
+        response_obj={},
+        start_time=datetime.datetime.now(timezone.utc),
+        end_time=datetime.datetime.now(timezone.utc),
+    )
+    assert payload["model"] == "azure_ai/gpt-5-mini"
+
+
+@patch("litellm.proxy.proxy_server.master_key", None)
+@patch("litellm.proxy.proxy_server.general_settings", {})
+def test_get_logging_payload_falls_back_to_kwargs_model_when_slp_model_missing():
+    payload = get_logging_payload(
+        kwargs=_model_router_spend_log_kwargs(slp_model=None),
+        response_obj={},
+        start_time=datetime.datetime.now(timezone.utc),
+        end_time=datetime.datetime.now(timezone.utc),
+    )
+    assert payload["model"] == "azure_ai/model_router/model-router"

From 57b367c78e6f691839a4c6dccf8ffe57bfb25478 Mon Sep 17 00:00:00 2001
From: milan <milan@berri.ai>
Date: Fri, 21 Aug 2026 03:25:06 +0000
Subject: [PATCH 013/105] refactor(tests): type the model router spend log
 kwargs helper

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../proxy/spend_tracking/test_spend_tracking_utils.py  | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
index b1a45fb84a3..9c97b2683b2 100644
--- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
+++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
@@ -6,6 +6,8 @@ import sys
 from datetime import timezone
 from typing import Any, Final, cast
 
+from typing_extensions import ReadOnly, TypedDict
+
 import pytest
 from fastapi.testclient import TestClient
 
@@ -3243,7 +3245,13 @@ def test_get_logging_payload_failed_request_without_standard_logging_payload_lea
     assert payload["custom_llm_provider"] == ""
 
 
-def _model_router_spend_log_kwargs(slp_model: str | None) -> dict[str, Any]:
+class _ModelRouterSpendLogKwargs(TypedDict):
+    model: ReadOnly[str]
+    litellm_params: ReadOnly[dict[str, dict[str, str]]]
+    standard_logging_object: ReadOnly[StandardLoggingPayload]
+
+
+def _model_router_spend_log_kwargs(slp_model: str | None) -> _ModelRouterSpendLogKwargs:
     standard_logging_payload: Final = cast(
         StandardLoggingPayload,
         {

From 0e96491554ea5b2bb51f1c8c79d4bb5c9718adaa Mon Sep 17 00:00:00 2001
From: Tin Chi Lo <tin@berri.ai>
Date: Fri, 21 Aug 2026 15:28:22 -0700
Subject: [PATCH 014/105] feat(router): per-group supported reasoning efforts
 with max and ultra levels

---
 .../transformation.py                         | 21 ++---
 .../llms/openai/chat/gpt_5_transformation.py  |  6 +-
 litellm/main.py                               |  4 +-
 ...odel_prices_and_context_window_backup.json | 76 ++++++++++++-----
 litellm/router.py                             |  9 ++
 .../reasoning_effort_capability.py            | 53 ++++++++++++
 litellm/types/llms/openai.py                  |  2 +-
 litellm/types/router.py                       |  1 +
 litellm/types/utils.py                        |  1 +
 litellm/utils.py                              |  1 +
 model_prices_and_context_window.json          | 76 ++++++++++++-----
 model_prices_and_context_window.schema.json   |  3 +
 ...responses_transformation_transformation.py | 13 ++-
 .../llms/openai/test_gpt5_transformation.py   | 33 ++++++++
 .../response_api_endpoints/test_endpoints.py  |  4 +-
 .../test_reasoning_effort_capability.py       | 79 +++++++++++++++++
 tests/test_litellm/test_router.py             | 84 +++++++++++++++++++
 tests/test_litellm/test_utils.py              |  1 +
 .../add_model/ComplexityRouterConfig.test.tsx | 41 ++++++++-
 .../add_model/ComplexityRouterConfig.tsx      | 12 ++-
 .../add_model/TierModelEffortRows.tsx         | 37 ++++----
 .../add_model/complexity_router_tiers.ts      | 12 ++-
 .../llm_calls/fetch_models.test.tsx           | 37 +++++++-
 .../src/components/llm_calls/fetch_models.tsx | 45 ++++++----
 ui/litellm-dashboard/src/lib/http/schema.d.ts |  2 +
 25 files changed, 551 insertions(+), 102 deletions(-)
 create mode 100644 litellm/router_utils/reasoning_effort_capability.py
 create mode 100644 tests/test_litellm/router_utils/test_reasoning_effort_capability.py

diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py
index b94e91b3034..f94bf34e8d3 100644
--- a/litellm/completion_extras/litellm_responses_transformation/transformation.py
+++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py
@@ -1113,22 +1113,13 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             litellm.reasoning_auto_summary or os.getenv("LITELLM_REASONING_AUTO_SUMMARY", "false").lower() == "true"
         )
 
-        # If string is passed, map with optional summary based on flag/env var
-        if reasoning_effort == "none":
-            return Reasoning(effort="none", summary="detailed") if auto_summary_enabled else Reasoning(effort="none")
-        elif reasoning_effort == "high":
-            return Reasoning(effort="high", summary="detailed") if auto_summary_enabled else Reasoning(effort="high")
-        elif reasoning_effort == "xhigh":
-            return Reasoning(effort="xhigh", summary="detailed") if auto_summary_enabled else Reasoning(effort="xhigh")
-        elif reasoning_effort == "medium":
+        # Level-agnostic: providers own effort validation, so an unknown level (max, ultra, future
+        # ones) passes through instead of being silently dropped here.
+        if reasoning_effort:
             return (
-                Reasoning(effort="medium", summary="detailed") if auto_summary_enabled else Reasoning(effort="medium")
-            )
-        elif reasoning_effort == "low":
-            return Reasoning(effort="low", summary="detailed") if auto_summary_enabled else Reasoning(effort="low")
-        elif reasoning_effort == "minimal":
-            return (
-                Reasoning(effort="minimal", summary="detailed") if auto_summary_enabled else Reasoning(effort="minimal")
+                Reasoning(effort=reasoning_effort, summary="detailed")
+                if auto_summary_enabled
+                else Reasoning(effort=reasoning_effort)
             )
         return None
 
diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py
index ffa3de0d5c6..c55640f6a1f 100644
--- a/litellm/llms/openai/chat/gpt_5_transformation.py
+++ b/litellm/llms/openai/chat/gpt_5_transformation.py
@@ -16,7 +16,7 @@ def _normalize_reasoning_effort_for_chat_completion(
 ) -> str | None:
     """Convert reasoning_effort to the string format expected by OpenAI chat completion API.
 
-    The chat completion API expects a simple string: 'none', 'low', 'medium', 'high', or 'xhigh'.
+    The chat completion API expects a simple effort string ('none' through 'ultra').
     Config/deployments may pass the Responses API format: {'effort': 'high', 'summary': 'detailed'}.
     """
     if value is None:
@@ -222,8 +222,8 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
                 if "reasoning_effort" in optional_params:
                     optional_params["reasoning_effort"] = normalized
 
-        if effective_effort == "xhigh":
-            # xhigh is an opt-in capability: only allow if model explicitly supports it.
+        if effective_effort in ("xhigh", "max", "ultra"):
+            # xhigh/max/ultra are opt-in capabilities: only allow if the model explicitly supports them.
             if not self._supports_reasoning_effort_level(model, effective_effort):
                 if litellm.drop_params or drop_params:
                     non_default_params.pop("reasoning_effort", None)
diff --git a/litellm/main.py b/litellm/main.py
index 84931c63544..1cac723c179 100644
--- a/litellm/main.py
+++ b/litellm/main.py
@@ -416,7 +416,7 @@ async def acompletion(
     logprobs: bool | None = None,
     top_logprobs: int | None = None,
     deployment_id=None,
-    reasoning_effort: Literal["none", "minimal", "low", "medium", "high", "xhigh", "default"] | None = None,
+    reasoning_effort: Literal["none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra", "default"] | None = None,
     verbosity: Literal["low", "medium", "high"] | None = None,
     safety_identifier: str | None = None,
     service_tier: str | None = None,
@@ -4920,7 +4920,7 @@ def completion(
     logit_bias: dict | None = None,
     user: str | None = None,
     # openai v1.0+ new params
-    reasoning_effort: Literal["none", "minimal", "low", "medium", "high", "xhigh", "default"] | None = None,
+    reasoning_effort: Literal["none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra", "default"] | None = None,
     verbosity: Literal["low", "medium", "high"] | None = None,
     response_format: dict | type[BaseModel] | None = None,
     seed: int | None = None,
diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index c36018feb9d..4a275273fb2 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -6639,7 +6639,9 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false
+        "supports_minimal_reasoning_effort": false,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "azure/gpt-5.6-sol": {
         "cache_read_input_token_cost": 5e-07,
@@ -6690,7 +6692,9 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false
+        "supports_minimal_reasoning_effort": false,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "azure/gpt-5.6-terra": {
         "cache_read_input_token_cost": 2e-07,
@@ -6741,7 +6745,9 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false
+        "supports_minimal_reasoning_effort": false,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "azure/gpt-5.6-luna": {
         "cache_read_input_token_cost": 2e-08,
@@ -6792,7 +6798,9 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false
+        "supports_minimal_reasoning_effort": false,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "azure/us/gpt-5.6": {
         "cache_read_input_token_cost": 5.5e-07,
@@ -6839,7 +6847,9 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false
+        "supports_minimal_reasoning_effort": false,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "azure/us/gpt-5.6-sol": {
         "cache_read_input_token_cost": 5.5e-07,
@@ -6887,7 +6897,9 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false
+        "supports_minimal_reasoning_effort": false,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "azure/us/gpt-5.6-terra": {
         "cache_read_input_token_cost": 2.2e-07,
@@ -6935,7 +6947,9 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false
+        "supports_minimal_reasoning_effort": false,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "azure/us/gpt-5.6-luna": {
         "cache_read_input_token_cost": 2.2e-08,
@@ -6983,7 +6997,9 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false
+        "supports_minimal_reasoning_effort": false,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "azure/eu/gpt-5.6": {
         "cache_read_input_token_cost": 5.5e-07,
@@ -7030,7 +7046,9 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false
+        "supports_minimal_reasoning_effort": false,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "azure/eu/gpt-5.6-sol": {
         "cache_read_input_token_cost": 5.5e-07,
@@ -7078,7 +7096,9 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false
+        "supports_minimal_reasoning_effort": false,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "azure/eu/gpt-5.6-terra": {
         "cache_read_input_token_cost": 2.2e-07,
@@ -7126,7 +7146,9 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false
+        "supports_minimal_reasoning_effort": false,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "azure/eu/gpt-5.6-luna": {
         "cache_read_input_token_cost": 2.2e-08,
@@ -7174,7 +7196,9 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false
+        "supports_minimal_reasoning_effort": false,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "azure/gpt-5.5": {
         "deprecation_date": "2027-10-26",
@@ -26405,7 +26429,9 @@
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_web_search": true,
-        "supports_xhigh_reasoning_effort": true
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "gpt-5.6-sol": {
         "cache_creation_input_token_cost": 5e-06,
@@ -26469,7 +26495,9 @@
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_web_search": true,
-        "supports_xhigh_reasoning_effort": true
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "gpt-5.6-terra": {
         "cache_creation_input_token_cost": 2.5e-06,
@@ -26532,7 +26560,9 @@
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_web_search": true,
-        "supports_xhigh_reasoning_effort": true
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "gpt-5.6-luna": {
         "cache_creation_input_token_cost": 2.5e-07,
@@ -26595,7 +26625,9 @@
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_web_search": true,
-        "supports_xhigh_reasoning_effort": true
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "gpt-5.6-cyber": {
         "cache_creation_input_token_cost": 1.5625e-05,
@@ -49028,7 +49060,9 @@
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "bedrock_mantle/openai.gpt-5.6-terra": {
         "input_cost_per_token": 2.2e-06,
@@ -49060,7 +49094,9 @@
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "bedrock_mantle/openai.gpt-5.6-luna": {
         "input_cost_per_token": 2.2e-07,
@@ -49092,7 +49128,9 @@
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "us.openai.gpt-5.6-sol": {
         "input_cost_per_token": 5.5e-06,
diff --git a/litellm/router.py b/litellm/router.py
index 045fd32847c..8cd1b804928 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -167,6 +167,10 @@ from litellm.router_utils.pre_call_checks.model_rate_limit_check import (
 from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import (
     PromptCachingDeploymentCheck,
 )
+from litellm.router_utils.reasoning_effort_capability import (
+    intersect_supported_reasoning_efforts,
+    resolve_supported_reasoning_efforts,
+)
 from litellm.router_utils.router_callbacks.track_deployment_metrics import (
     increment_deployment_failures_for_current_minute,
     increment_deployment_successes_for_current_minute,
@@ -9558,6 +9562,11 @@ class Router:
                 if model_info.get("rpm", None) is not None and _deployment_rpm is None:
                     _deployment_rpm = model_info.get("rpm")
 
+            model_group_info.supported_reasoning_efforts = intersect_supported_reasoning_efforts(
+                model_group_info.supported_reasoning_efforts,
+                resolve_supported_reasoning_efforts(model_info),
+            )
+
             if _deployment_tpm is not None:
                 if total_tpm is None:
                     total_tpm = 0
diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py
new file mode 100644
index 00000000000..f59c53e9287
--- /dev/null
+++ b/litellm/router_utils/reasoning_effort_capability.py
@@ -0,0 +1,53 @@
+"""Resolve which reasoning_effort values a deployment, and by intersection a model group, accepts.
+
+The model-map flags carry different polarity per level, mirroring the provider gates
+(gpt_5_transformation.py restricts xhigh to explicit opt-in and treats minimal/low as opt-out;
+anthropic/chat/transformation.py rejects only xhigh/max without an explicit flag): medium and high
+are unconditional for any reasoning model, none/minimal/low are supported unless the map explicitly
+says false, and xhigh/max require an explicit true. Shipping the resolved list keeps that polarity
+in one place instead of re-encoding it in every consumer.
+"""
+
+from collections.abc import Mapping, Sequence
+from typing import Final
+
+REASONING_EFFORT_CAPABILITY_ORDER: Final = ("none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra")
+
+_OPT_OUT_FLAGS: Final = (
+    ("none", "supports_none_reasoning_effort"),
+    ("minimal", "supports_minimal_reasoning_effort"),
+    ("low", "supports_low_reasoning_effort"),
+)
+_OPT_IN_FLAGS: Final = (
+    ("xhigh", "supports_xhigh_reasoning_effort"),
+    ("max", "supports_max_reasoning_effort"),
+    ("ultra", "supports_ultra_reasoning_effort"),
+)
+_UNCONDITIONAL_EFFORTS: Final = frozenset(("medium", "high"))
+
+
+def resolve_supported_reasoning_efforts(model_info: Mapping[str, object]) -> tuple[str, ...] | None:
+    """None = no capability metadata for this deployment (e.g. a model absent from the model map,
+    whose stub info carries no supports_reasoning key at all); () = reasoning unsupported."""
+    if "supports_reasoning" not in model_info:
+        return None
+    if model_info.get("supports_reasoning") is not True:
+        return ()
+    opt_out: Final = frozenset(effort for effort, flag in _OPT_OUT_FLAGS if model_info.get(flag) is not False)
+    opt_in: Final = frozenset(effort for effort, flag in _OPT_IN_FLAGS if model_info.get(flag) is True)
+    allowed: Final = opt_out | _UNCONDITIONAL_EFFORTS | opt_in
+    return tuple(effort for effort in REASONING_EFFORT_CAPABILITY_ORDER if effort in allowed)
+
+
+def intersect_supported_reasoning_efforts(
+    current: Sequence[str] | None,
+    resolved: Sequence[str] | None,
+) -> tuple[str, ...] | None:
+    """Deployments without metadata (None) never narrow the group; an effort survives only when
+    every deployment with metadata accepts it, so the group offers nothing routing could reject."""
+    if resolved is None:
+        return tuple(current) if current is not None else None
+    if current is None:
+        return tuple(resolved)
+    keep: Final = frozenset(current) & frozenset(resolved)
+    return tuple(effort for effort in REASONING_EFFORT_CAPABILITY_ORDER if effort in keep)
diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py
index e7a3f825455..e3d28a0097f 100644
--- a/litellm/types/llms/openai.py
+++ b/litellm/types/llms/openai.py
@@ -1840,7 +1840,7 @@ ResponsesAPIStreamingResponse = Annotated[
 ]
 
 
-REASONING_EFFORT = Literal["none", "minimal", "low", "medium", "high", "xhigh"]
+REASONING_EFFORT = Literal["none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra"]
 
 
 class OpenAIRealtimeStreamSession(TypedDict, total=False):
diff --git a/litellm/types/router.py b/litellm/types/router.py
index 9fd5cfa96ef..d4c735387a5 100644
--- a/litellm/types/router.py
+++ b/litellm/types/router.py
@@ -637,6 +637,7 @@ class ModelGroupInfo(BaseModel):
     supports_url_context: bool = Field(default=False)
     supports_reasoning: bool = Field(default=False)
     supports_function_calling: bool = Field(default=False)
+    supported_reasoning_efforts: tuple[str, ...] | None = Field(default=None)
     supported_openai_params: list[str] | None = Field(default=[])
     configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None
 
diff --git a/litellm/types/utils.py b/litellm/types/utils.py
index 4d59650f410..6a5e4c24dce 100644
--- a/litellm/types/utils.py
+++ b/litellm/types/utils.py
@@ -164,6 +164,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False):
     supports_low_reasoning_effort: bool | None
     supports_xhigh_reasoning_effort: bool | None
     supports_max_reasoning_effort: bool | None
+    supports_ultra_reasoning_effort: bool | None  # writable-ok: Pydantic warns on ReadOnly TypedDict fields
     supports_output_config: bool | None
     supports_image_size: bool | None
     bedrock_output_config_effort_ceiling: Literal["low", "medium", "high", "max", "xhigh"] | None
diff --git a/litellm/utils.py b/litellm/utils.py
index 5b2ef93edb3..a74cf23a4ae 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -5764,6 +5764,7 @@ def _get_model_info_helper(
                 supports_low_reasoning_effort=_model_info.get("supports_low_reasoning_effort", None),
                 supports_xhigh_reasoning_effort=_model_info.get("supports_xhigh_reasoning_effort", None),
                 supports_max_reasoning_effort=_model_info.get("supports_max_reasoning_effort", None),
+                supports_ultra_reasoning_effort=_model_info.get("supports_ultra_reasoning_effort", None),
                 bedrock_output_config_effort_ceiling=_model_info.get("bedrock_output_config_effort_ceiling", None),
                 bedrock_converse_supports_strict_tools=_model_info.get("bedrock_converse_supports_strict_tools", None),
                 supports_computer_use=_model_info.get("supports_computer_use", None),
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index c36018feb9d..4a275273fb2 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -6639,7 +6639,9 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false
+        "supports_minimal_reasoning_effort": false,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "azure/gpt-5.6-sol": {
         "cache_read_input_token_cost": 5e-07,
@@ -6690,7 +6692,9 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false
+        "supports_minimal_reasoning_effort": false,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "azure/gpt-5.6-terra": {
         "cache_read_input_token_cost": 2e-07,
@@ -6741,7 +6745,9 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false
+        "supports_minimal_reasoning_effort": false,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "azure/gpt-5.6-luna": {
         "cache_read_input_token_cost": 2e-08,
@@ -6792,7 +6798,9 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false
+        "supports_minimal_reasoning_effort": false,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "azure/us/gpt-5.6": {
         "cache_read_input_token_cost": 5.5e-07,
@@ -6839,7 +6847,9 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false
+        "supports_minimal_reasoning_effort": false,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "azure/us/gpt-5.6-sol": {
         "cache_read_input_token_cost": 5.5e-07,
@@ -6887,7 +6897,9 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false
+        "supports_minimal_reasoning_effort": false,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "azure/us/gpt-5.6-terra": {
         "cache_read_input_token_cost": 2.2e-07,
@@ -6935,7 +6947,9 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false
+        "supports_minimal_reasoning_effort": false,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "azure/us/gpt-5.6-luna": {
         "cache_read_input_token_cost": 2.2e-08,
@@ -6983,7 +6997,9 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false
+        "supports_minimal_reasoning_effort": false,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "azure/eu/gpt-5.6": {
         "cache_read_input_token_cost": 5.5e-07,
@@ -7030,7 +7046,9 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false
+        "supports_minimal_reasoning_effort": false,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "azure/eu/gpt-5.6-sol": {
         "cache_read_input_token_cost": 5.5e-07,
@@ -7078,7 +7096,9 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false
+        "supports_minimal_reasoning_effort": false,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "azure/eu/gpt-5.6-terra": {
         "cache_read_input_token_cost": 2.2e-07,
@@ -7126,7 +7146,9 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false
+        "supports_minimal_reasoning_effort": false,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "azure/eu/gpt-5.6-luna": {
         "cache_read_input_token_cost": 2.2e-08,
@@ -7174,7 +7196,9 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false
+        "supports_minimal_reasoning_effort": false,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "azure/gpt-5.5": {
         "deprecation_date": "2027-10-26",
@@ -26405,7 +26429,9 @@
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_web_search": true,
-        "supports_xhigh_reasoning_effort": true
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "gpt-5.6-sol": {
         "cache_creation_input_token_cost": 5e-06,
@@ -26469,7 +26495,9 @@
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_web_search": true,
-        "supports_xhigh_reasoning_effort": true
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "gpt-5.6-terra": {
         "cache_creation_input_token_cost": 2.5e-06,
@@ -26532,7 +26560,9 @@
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_web_search": true,
-        "supports_xhigh_reasoning_effort": true
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "gpt-5.6-luna": {
         "cache_creation_input_token_cost": 2.5e-07,
@@ -26595,7 +26625,9 @@
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_web_search": true,
-        "supports_xhigh_reasoning_effort": true
+        "supports_xhigh_reasoning_effort": true,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "gpt-5.6-cyber": {
         "cache_creation_input_token_cost": 1.5625e-05,
@@ -49028,7 +49060,9 @@
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "bedrock_mantle/openai.gpt-5.6-terra": {
         "input_cost_per_token": 2.2e-06,
@@ -49060,7 +49094,9 @@
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "bedrock_mantle/openai.gpt-5.6-luna": {
         "input_cost_per_token": 2.2e-07,
@@ -49092,7 +49128,9 @@
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "supports_max_reasoning_effort": true,
+        "supports_ultra_reasoning_effort": true
     },
     "us.openai.gpt-5.6-sol": {
         "input_cost_per_token": 5.5e-06,
diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json
index f7f60c7666d..75bf3d47f35 100644
--- a/model_prices_and_context_window.schema.json
+++ b/model_prices_and_context_window.schema.json
@@ -702,6 +702,9 @@
         "supports_tool_search": {
           "type": "boolean"
         },
+        "supports_ultra_reasoning_effort": {
+          "type": "boolean"
+        },
         "supports_url_context": {
           "type": "boolean"
         },
diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py
index 1fb74b2b7bf..a0aecfafdbc 100644
--- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py
+++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py
@@ -1585,10 +1585,15 @@ def test_map_reasoning_effort_adds_summary_detailed(monkeypatch):
         assert result_dict["summary"] == "custom_summary"
         print("✓ Dict input is passed through without modification")
 
-        # Test 5: None/unknown values return None
-        result_unknown = handler._map_reasoning_effort("unknown_value")
-        assert result_unknown is None
-        print("✓ Unknown reasoning_effort values return None")
+        # Test 5: levels this bridge does not enumerate (max, ultra, future ones) pass through so the
+        # provider can judge them, instead of being silently dropped before the request is built
+        from litellm.types.llms.openai import Reasoning
+
+        for effort in ("max", "ultra", "unknown_value"):
+            result_passthrough = handler._map_reasoning_effort(effort)
+            assert result_passthrough == Reasoning(effort=effort)
+        assert handler._map_reasoning_effort("") is None
+        print("✓ Unenumerated reasoning_effort levels pass through to the provider")
 
         print(
             "✓ All reasoning_effort behaviors work correctly with flag/env var control"
diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py
index d279b119efe..7595c64da07 100644
--- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py
+++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py
@@ -1309,3 +1309,36 @@ def test_responses_gpt54_allow_temperature_effort_none(
         drop_params=False,
     )
     assert params["temperature"] == 0.7
+
+
+@pytest.mark.parametrize("effort", ["max", "ultra"])
+def test_gpt5_6_allows_opt_in_reasoning_efforts(config: OpenAIConfig, effort: str):
+    params = config.map_openai_params(
+        non_default_params={"reasoning_effort": effort},
+        optional_params={},
+        model="gpt-5.6",
+        drop_params=False,
+    )
+    assert params["reasoning_effort"] == effort
+
+
+@pytest.mark.parametrize("effort", ["max", "ultra"])
+def test_gpt5_rejects_opt_in_reasoning_efforts_for_other_models(config: OpenAIConfig, effort: str):
+    with pytest.raises(litellm.utils.UnsupportedParamsError):
+        config.map_openai_params(
+            non_default_params={"reasoning_effort": effort},
+            optional_params={},
+            model="gpt-5.1",
+            drop_params=False,
+        )
+
+
+@pytest.mark.parametrize("effort", ["max", "ultra"])
+def test_gpt5_drops_opt_in_reasoning_efforts_when_requested(config: OpenAIConfig, effort: str):
+    params = config.map_openai_params(
+        non_default_params={"reasoning_effort": effort},
+        optional_params={},
+        model="gpt-5.1",
+        drop_params=True,
+    )
+    assert "reasoning_effort" not in params
diff --git a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py
index 9177944df2d..a906b0e638f 100644
--- a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py
+++ b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py
@@ -1352,7 +1352,9 @@ class TestParseCursorModelVariant:
             ("gemini-3.0-pro-thinking-low", "gemini-3.0-pro", "low"),
             ("claude-opus-5-fast", "claude-opus-5", None),
             ("gpt-5.6-sol", "gpt-5.6-sol", None),
-            ("foo-thinking-ultra-fast", "foo-thinking-ultra", None),
+            ("gpt-5.6-thinking-ultra-fast", "gpt-5.6", "ultra"),
+            ("gpt-5.6-thinking-max", "gpt-5.6", "max"),
+            ("foo-thinking-mega-fast", "foo-thinking-mega", None),
             ("-thinking-high", "-thinking-high", None),
         ],
     )
diff --git a/tests/test_litellm/router_utils/test_reasoning_effort_capability.py b/tests/test_litellm/router_utils/test_reasoning_effort_capability.py
new file mode 100644
index 00000000000..ebc7952081c
--- /dev/null
+++ b/tests/test_litellm/router_utils/test_reasoning_effort_capability.py
@@ -0,0 +1,79 @@
+from litellm.router_utils.reasoning_effort_capability import (
+    intersect_supported_reasoning_efforts,
+    resolve_supported_reasoning_efforts,
+)
+
+
+class TestResolveSupportedReasoningEfforts:
+    def test_no_metadata_resolves_to_unknown(self):
+        assert resolve_supported_reasoning_efforts({}) is None
+
+    def test_non_reasoning_model_supports_no_efforts(self):
+        assert resolve_supported_reasoning_efforts({"supports_reasoning": None}) == ()
+        assert resolve_supported_reasoning_efforts({"supports_reasoning": False}) == ()
+
+    def test_reasoning_model_with_no_flags_gets_the_opt_out_levels_only(self):
+        # The kimi shape: supports_reasoning true, zero effort flags. medium/high are unconditional,
+        # none/minimal/low are opt-out so absence means supported, xhigh/max are opt-in so absence
+        # means unsupported.
+        assert resolve_supported_reasoning_efforts({"supports_reasoning": True}) == (
+            "none",
+            "minimal",
+            "low",
+            "medium",
+            "high",
+        )
+
+    def test_explicit_false_removes_an_opt_out_level(self):
+        # The gpt-5.5-pro shape from the model map: only medium/high/xhigh are accepted upstream.
+        resolved = resolve_supported_reasoning_efforts(
+            {
+                "supports_reasoning": True,
+                "supports_none_reasoning_effort": False,
+                "supports_minimal_reasoning_effort": False,
+                "supports_low_reasoning_effort": False,
+                "supports_xhigh_reasoning_effort": True,
+            }
+        )
+        assert resolved == ("medium", "high", "xhigh")
+
+    def test_explicit_true_adds_the_opt_in_levels(self):
+        # The claude-opus shape: xhigh and max explicitly true, everything else absent.
+        resolved = resolve_supported_reasoning_efforts(
+            {
+                "supports_reasoning": True,
+                "supports_xhigh_reasoning_effort": True,
+                "supports_max_reasoning_effort": True,
+            }
+        )
+        assert resolved == ("none", "minimal", "low", "medium", "high", "xhigh", "max")
+
+    def test_ultra_is_opt_in(self):
+        without_flag = resolve_supported_reasoning_efforts({"supports_reasoning": True})
+        with_flag = resolve_supported_reasoning_efforts(
+            {"supports_reasoning": True, "supports_ultra_reasoning_effort": True}
+        )
+        assert without_flag is not None and "ultra" not in without_flag
+        assert with_flag is not None and with_flag[-1] == "ultra"
+
+    def test_opt_in_flag_set_false_stays_excluded(self):
+        resolved = resolve_supported_reasoning_efforts(
+            {"supports_reasoning": True, "supports_xhigh_reasoning_effort": False}
+        )
+        assert resolved is not None
+        assert "xhigh" not in resolved
+
+
+class TestIntersectSupportedReasoningEfforts:
+    def test_unknown_never_narrows(self):
+        assert intersect_supported_reasoning_efforts(["medium", "high"], None) == ("medium", "high")
+        assert intersect_supported_reasoning_efforts(None, ["medium", "high"]) == ("medium", "high")
+        assert intersect_supported_reasoning_efforts(None, None) is None
+
+    def test_intersection_keeps_canonical_order(self):
+        assert intersect_supported_reasoning_efforts(
+            ["max", "high", "medium", "xhigh"], ["xhigh", "medium", "minimal"]
+        ) == ("medium", "xhigh")
+
+    def test_disjoint_sets_intersect_to_empty(self):
+        assert intersect_supported_reasoning_efforts(["max"], ["minimal"]) == ()
diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py
index d00fbf589e3..095c962328a 100644
--- a/tests/test_litellm/test_router.py
+++ b/tests/test_litellm/test_router.py
@@ -8878,3 +8878,87 @@ class TestAzureBaseModelFallbackLogging:
             deployment=None, received_model_name="my-group", id="azure-base-model-test-id"
         )
         assert model_info["max_input_tokens"] == litellm.model_cost["azure/gpt-4o-mini"]["max_input_tokens"]
+
+def test_model_group_info_intersects_supported_reasoning_efforts():
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "smart-group",
+                "litellm_params": {"model": "anthropic/opus-like"},
+                "model_info": {"id": "opus-like-deployment"},
+            },
+            {
+                "model_name": "smart-group",
+                "litellm_params": {"model": "openai/mini-like"},
+                "model_info": {"id": "mini-like-deployment"},
+            },
+        ]
+    )
+
+    def _model_info(model_id: str, model_name: str):
+        if model_id == "opus-like-deployment":
+            return {
+                "key": model_name,
+                "litellm_provider": "anthropic",
+                "mode": "chat",
+                "supports_reasoning": True,
+                "supports_xhigh_reasoning_effort": True,
+                "supports_max_reasoning_effort": True,
+            }
+        return {
+            "key": model_name,
+            "litellm_provider": "openai",
+            "mode": "chat",
+            "supports_reasoning": True,
+            "supports_none_reasoning_effort": False,
+            "supports_minimal_reasoning_effort": True,
+            "supports_xhigh_reasoning_effort": False,
+        }
+
+    with patch.object(router, "get_deployment_model_info", side_effect=_model_info):
+        result = router._set_model_group_info(
+            model_group="smart-group",
+            user_facing_model_group_name="smart-group",
+        )
+
+    assert result is not None
+    # opus-like offers all seven levels, mini-like lacks none/xhigh/max; only the common set survives,
+    # so the group never advertises an effort routing could hand to a deployment that rejects it.
+    assert result.supported_reasoning_efforts == ("minimal", "low", "medium", "high")
+
+
+def test_model_group_info_reasoning_efforts_ignore_deployments_without_metadata():
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "smart-group",
+                "litellm_params": {"model": "anthropic/opus-like"},
+                "model_info": {"id": "opus-like-deployment"},
+            },
+            {
+                "model_name": "smart-group",
+                "litellm_params": {"model": "openai/unmapped-model"},
+                "model_info": {"id": "unmapped-deployment"},
+            },
+        ]
+    )
+
+    def _model_info(model_id: str, model_name: str):
+        if model_id == "opus-like-deployment":
+            return {
+                "key": model_name,
+                "litellm_provider": "anthropic",
+                "mode": "chat",
+                "supports_reasoning": True,
+                "supports_max_reasoning_effort": True,
+            }
+        return {"key": model_name, "litellm_provider": "openai", "mode": "chat"}
+
+    with patch.object(router, "get_deployment_model_info", side_effect=_model_info):
+        result = router._set_model_group_info(
+            model_group="smart-group",
+            user_facing_model_group_name="smart-group",
+        )
+
+    assert result is not None
+    assert result.supported_reasoning_efforts == ("none", "minimal", "low", "medium", "high", "max")
diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py
index 27cf9067914..af69d117e04 100644
--- a/tests/test_litellm/test_utils.py
+++ b/tests/test_litellm/test_utils.py
@@ -997,6 +997,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
                 "supports_none_reasoning_effort": {"type": "boolean"},
                 "supports_xhigh_reasoning_effort": {"type": "boolean"},
                 "supports_max_reasoning_effort": {"type": "boolean"},
+                "supports_ultra_reasoning_effort": {"type": "boolean"},
                 "supports_adaptive_thinking": {"type": "boolean"},
                 "supports_legacy_thinking": {"type": "boolean"},
                 "thinking_always_on": {"type": "boolean"},
diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx
index 0a848ed7deb..a26197aa243 100644
--- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx
@@ -8,7 +8,12 @@ vi.mock(
 );
 
 const mockModelInfo = [
-  { model_group: "gpt-4", mode: "chat", supports_reasoning: true },
+  {
+    model_group: "gpt-4",
+    mode: "chat",
+    supports_reasoning: true,
+    supported_reasoning_efforts: ["medium", "high", "xhigh"],
+  },
   { model_group: "gpt-3.5-turbo", mode: "chat" },
   { model_group: "claude-3-opus", mode: "chat", supports_reasoning: true },
   { model_group: "text-embedding-3-small", mode: "embedding" },
@@ -943,3 +948,37 @@ describe("ComplexityRouterConfig reasoning effort gating", () => {
     ).toHaveTextContent("low");
   });
 });
+
+describe("ComplexityRouterConfig per-model effort filtering", () => {
+  it("offers only the efforts the model group supports", async () => {
+    renderWithProviders(<ComplexityRouterConfig {...baseProps} />);
+    const user = userEvent.setup();
+    await user.click(screen.getByRole("combobox", { name: "Reasoning effort for gpt-4 in the Complex tier" }));
+    const options = (await screen.findAllByRole("option")).map((option) => option.textContent);
+    expect(options).toEqual(["Default", "medium", "high", "xhigh"]);
+  });
+
+  it("falls back to every effort when the group only reports supports_reasoning", async () => {
+    renderWithProviders(<ComplexityRouterConfig {...baseProps} />);
+    const user = userEvent.setup();
+    await user.click(
+      screen.getByRole("combobox", { name: "Reasoning effort for claude-3-opus in the Reasoning tier" }),
+    );
+    const options = (await screen.findAllByRole("option")).map((option) => option.textContent);
+    expect(options).toEqual(["Default", "none", "minimal", "low", "medium", "high", "xhigh"]);
+  });
+
+  // Hand-authored configs can carry a level outside the supported set (e.g. max); it must render
+  // and stay clearable rather than being masked as Default.
+  it("keeps showing a stored effort outside the supported set", () => {
+    renderWithProviders(
+      <ComplexityRouterConfig
+        {...baseProps}
+        value={{ ...defaultValue, tier_model_params: { COMPLEX: { "gpt-4": { reasoning_effort: "max" } } } }}
+      />,
+    );
+    expect(screen.getByRole("combobox", { name: "Reasoning effort for gpt-4 in the Complex tier" })).toHaveTextContent(
+      "max",
+    );
+  });
+});
diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
index fc731e2c77f..e72905c2f73 100644
--- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
@@ -13,6 +13,7 @@ import { ModelGroup } from "@/components/llm_calls/fetch_models";
 import AdaptiveRoutingConfig from "./AdaptiveRoutingConfig";
 import ClassificationMethodConfig from "./ClassificationMethodConfig";
 import {
+  REASONING_EFFORT_OPTIONS,
   ReasoningEffort,
   TierModelParamsByTier,
   pruneTierModelParams,
@@ -251,8 +252,13 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
   const defaultModel = resolveComplexityDefaultModel(value.tiers, value.default_model);
 
   // Embedding models can't serve a chat-completion role, so they're excluded here.
-  const reasoningModels = new Set(
-    modelInfo.filter((model) => model.supports_reasoning).map((model) => model.model_group),
+  // The backend list is the per-group intersection of accepted effort levels; when a proxy does not
+  // send it yet, fall back to the coarse supports_reasoning gate with every level offered.
+  const effortOptionsByModel: Record<string, string[]> = Object.fromEntries(
+    modelInfo.map((model) => [
+      model.model_group,
+      model.supported_reasoning_efforts ?? (model.supports_reasoning ? [...REASONING_EFFORT_OPTIONS] : []),
+    ]),
   );
 
   const modelOptions = modelInfo
@@ -365,7 +371,7 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
                   <TierModelEffortRows
                     tierLabel={label}
                     models={value.tiers[tier]}
-                    reasoningModels={reasoningModels}
+                    effortOptionsByModel={effortOptionsByModel}
                     paramsByModel={value.tier_model_params?.[tier]}
                     onEffortChange={(model, effort) => handleTierModelEffortChange(tier, model, effort)}
                   />
diff --git a/ui/litellm-dashboard/src/components/add_model/TierModelEffortRows.tsx b/ui/litellm-dashboard/src/components/add_model/TierModelEffortRows.tsx
index 67f583bd894..0c93b3228dc 100644
--- a/ui/litellm-dashboard/src/components/add_model/TierModelEffortRows.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/TierModelEffortRows.tsx
@@ -2,20 +2,19 @@ import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@
 import { SimpleTooltip } from "@/components/ui/tooltip";
 import { Info } from "lucide-react";
 import React from "react";
-import { REASONING_EFFORT_OPTIONS, ReasoningEffort, TierModelParams } from "./complexity_router_tiers";
+import { ReasoningEffort, TierModelParams } from "./complexity_router_tiers";
 
 const PROVIDER_DEFAULT = "__provider_default__";
 
-const asEffort = (params: TierModelParams | undefined): ReasoningEffort | undefined => {
+const storedEffort = (params: TierModelParams | undefined): ReasoningEffort | undefined => {
   const stored = params?.reasoning_effort;
-  if (typeof stored !== "string") return undefined;
-  return REASONING_EFFORT_OPTIONS.find((option) => option === stored);
+  return typeof stored === "string" && stored ? stored : undefined;
 };
 
 interface TierModelEffortRowsProps {
   tierLabel: string;
   models: string[];
-  reasoningModels: ReadonlySet<string>;
+  effortOptionsByModel: Record<string, string[]>;
   paramsByModel: Record<string, TierModelParams> | undefined;
   onEffortChange: (model: string, effort: ReasoningEffort | undefined) => void;
 }
@@ -23,14 +22,21 @@ interface TierModelEffortRowsProps {
 const TierModelEffortRows: React.FC<TierModelEffortRowsProps> = ({
   tierLabel,
   models,
-  reasoningModels,
+  effortOptionsByModel,
   paramsByModel,
   onEffortChange,
 }) => {
-  const shown = models.filter(
-    (model) => reasoningModels.has(model) || Object.keys(paramsByModel?.[model] ?? {}).length > 0,
-  );
-  if (shown.length === 0) return null;
+  const rows = models
+    .map((model) => {
+      const effort = storedEffort(paramsByModel?.[model]);
+      const supported = effortOptionsByModel[model] ?? [];
+      // A stored effort outside the supported set (hand-authored, or capabilities changed since it
+      // was saved) stays listed so it renders and can be cleared.
+      const options = effort !== undefined && !supported.includes(effort) ? [...supported, effort] : supported;
+      return { model, effort, options };
+    })
+    .filter(({ model, options }) => options.length > 0 || Object.keys(paramsByModel?.[model] ?? {}).length > 0);
+  if (rows.length === 0) return null;
   return (
     <div className="mt-2 space-y-1">
       <div className="flex items-center gap-1">
@@ -41,18 +47,17 @@ const TierModelEffortRows: React.FC<TierModelEffortRowsProps> = ({
           <Info className="size-3 text-muted-foreground/70" />
         </SimpleTooltip>
       </div>
-      {shown.map((model) => (
+      {rows.map(({ model, effort, options }) => (
         <div key={model} className="flex items-center justify-between gap-2">
           <span className="truncate text-xs">{model}</span>
           <Select
             items={[
               { value: PROVIDER_DEFAULT, label: "Default" },
-              ...REASONING_EFFORT_OPTIONS.map((option) => ({ value: option, label: option })),
+              ...options.map((option) => ({ value: option, label: option })),
             ]}
-            value={asEffort(paramsByModel?.[model]) ?? PROVIDER_DEFAULT}
+            value={effort ?? PROVIDER_DEFAULT}
             onValueChange={(selected: string | null) =>
-              selected !== null &&
-              onEffortChange(model, selected === PROVIDER_DEFAULT ? undefined : (selected as ReasoningEffort))
+              selected !== null && onEffortChange(model, selected === PROVIDER_DEFAULT ? undefined : selected)
             }
           >
             <SelectTrigger
@@ -64,7 +69,7 @@ const TierModelEffortRows: React.FC<TierModelEffortRowsProps> = ({
             </SelectTrigger>
             <SelectContent>
               <SelectItem value={PROVIDER_DEFAULT}>Default</SelectItem>
-              {REASONING_EFFORT_OPTIONS.map((option) => (
+              {options.map((option) => (
                 <SelectItem key={option} value={option}>
                   {option}
                 </SelectItem>
diff --git a/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts b/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts
index cc12746a755..2ea1915ca03 100644
--- a/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts
+++ b/ui/litellm-dashboard/src/components/add_model/complexity_router_tiers.ts
@@ -5,8 +5,18 @@ export type TierModelParams = Record<string, unknown>;
 
 export type TierModelParamsByTier = Record<string, Record<string, TierModelParams>>;
 
+/**
+ * Fallback offered only when a proxy does not report supported_reasoning_efforts per model group.
+ * max is deliberately absent: it is opt-in per model in the model map, so a capability-blind list
+ * must not offer it; a proxy that reports capabilities supplies max itself where supported.
+ */
 export const REASONING_EFFORT_OPTIONS = ["none", "minimal", "low", "medium", "high", "xhigh"] as const;
-export type ReasoningEffort = (typeof REASONING_EFFORT_OPTIONS)[number];
+
+/**
+ * Open on purpose: the valid set is per model group at runtime (supported_reasoning_efforts), and
+ * hand-authored configs can carry any level, so the known literals only add autocompletion.
+ */
+export type ReasoningEffort = (typeof REASONING_EFFORT_OPTIONS)[number] | (string & {});
 
 const asRecord = (raw: unknown): Record<string, unknown> | undefined =>
   typeof raw === "object" && raw !== null && !Array.isArray(raw) ? (raw as Record<string, unknown>) : undefined;
diff --git a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.test.tsx b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.test.tsx
index bd691c7f629..2f35cf5ef6c 100644
--- a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.test.tsx
+++ b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.test.tsx
@@ -1,5 +1,5 @@
 import { beforeEach, describe, expect, it, vi } from "vitest";
-import { modelAvailableCall } from "@/components/networking";
+import { modelAvailableCall, modelHubCall } from "@/components/networking";
 import { fetchAvailableModelsForTeam } from "./fetch_models";
 
 vi.mock("@/components/networking", () => ({
@@ -31,3 +31,38 @@ describe("fetchAvailableModelsForTeam", () => {
     expect(await fetchAvailableModelsForTeam("token", "team-123")).toEqual([]);
   });
 });
+
+describe("fetchAvailableModelsForTeam capability join", () => {
+  it("carries model-group capabilities onto team-allowed names that have a group entry", async () => {
+    modelAvailableCallMock.mockResolvedValue({ data: [{ id: "gpt-5-mini" }, { id: "team-only-byok" }] });
+    vi.mocked(modelHubCall).mockResolvedValue({
+      data: [
+        {
+          model_group: "gpt-5-mini",
+          mode: "chat",
+          supports_reasoning: true,
+          supported_reasoning_efforts: ["minimal", "low", "medium", "high"],
+        },
+      ],
+    });
+
+    const models = await fetchAvailableModelsForTeam("token", "team-123");
+
+    expect(models).toEqual([
+      {
+        model_group: "gpt-5-mini",
+        mode: "chat",
+        supports_reasoning: true,
+        supported_reasoning_efforts: ["minimal", "low", "medium", "high"],
+      },
+      { model_group: "team-only-byok" },
+    ]);
+  });
+
+  it("keeps the team list usable when the group-info fetch fails", async () => {
+    modelAvailableCallMock.mockResolvedValue({ data: [{ id: "gpt-5-mini" }] });
+    vi.mocked(modelHubCall).mockRejectedValue(new Error("boom"));
+
+    expect(await fetchAvailableModelsForTeam("token", "team-123")).toEqual([{ model_group: "gpt-5-mini" }]);
+  });
+});
diff --git a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx
index 4f4c901a1c2..93f89a44813 100644
--- a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx
+++ b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx
@@ -7,6 +7,7 @@ export interface ModelGroup {
   model_group: string;
   mode?: string;
   supports_reasoning?: boolean;
+  supported_reasoning_efforts?: string[];
 }
 
 interface AvailableModel {
@@ -15,15 +16,36 @@ interface AvailableModel {
   id?: string | null;
   mode?: string | null;
   supports_reasoning?: boolean | null;
+  supported_reasoning_efforts?: string[] | null;
 }
 
+const toModelGroup = (item: AvailableModel): ModelGroup => {
+  const groupName = (item.model_group || item.id || item.model_name) ?? "";
+  return {
+    model_group: groupName,
+    ...(item.mode && { mode: item.mode }),
+    ...(item.supports_reasoning === true && { supports_reasoning: true }),
+    ...(item.supported_reasoning_efforts && { supported_reasoning_efforts: item.supported_reasoning_efforts }),
+  };
+};
+
+/**
+ * /models carries no capability metadata, so the team-allowed names are joined against
+ * /model_group/info (one extra parallel request; the endpoint has no team filter of its own, and it
+ * serves team-scoped tokens row-filtered, verified live). A name without a group entry keeps every
+ * capability field absent; a failed group fetch is console.error'd by fetchAvailableModels.
+ */
 export const fetchAvailableModelsForTeam = async (accessToken: string, teamId: string): Promise<ModelGroup[]> => {
-  const response = await modelAvailableCall(accessToken, "", "", false, teamId);
+  const [response, groups] = await Promise.all([
+    modelAvailableCall(accessToken, "", "", false, teamId),
+    fetchAvailableModels(accessToken).catch(() => [] as ModelGroup[]),
+  ]);
+  const byGroup = new Map(groups.map((group) => [group.model_group, group]));
   const modelNames: string[] = (response?.data ?? []).map((model: { id: string }) => model.id);
 
   return excludeProxyWideSentinel(Array.from(new Set(modelNames)))
     .sort((a, b) => a.localeCompare(b))
-    .map((model) => ({ model_group: model }));
+    .map((model) => byGroup.get(model) ?? { model_group: model });
 };
 
 /**
@@ -32,20 +54,11 @@ export const fetchAvailableModelsForTeam = async (accessToken: string, teamId: s
 export const fetchAvailableModels = async (accessToken: string): Promise<ModelGroup[]> => {
   try {
     const fetchedModels = await modelHubCall(accessToken);
-
-    if (fetchedModels?.data.length > 0) {
-      const models: ModelGroup[] = fetchedModels.data
-        .map((item: AvailableModel) => ({
-          model_group: item.model_group || item.id || item.model_name || "",
-          mode: item.mode || undefined,
-          supports_reasoning: item.supports_reasoning === true || undefined,
-        }))
-        .filter((model: ModelGroup) => model.model_group !== "");
-
-      models.sort((a, b) => a.model_group.localeCompare(b.model_group));
-      return Array.from(new Map(models.map((model) => [model.model_group, model])).values());
-    }
-    return [];
+    const models: ModelGroup[] = (fetchedModels?.data ?? [])
+      .map(toModelGroup)
+      .filter((model: ModelGroup) => model.model_group !== "")
+      .sort((a: ModelGroup, b: ModelGroup) => a.model_group.localeCompare(b.model_group));
+    return Array.from(new Map(models.map((model) => [model.model_group, model])).values());
   } catch (error) {
     console.error("Error fetching model info:", error);
     throw error;
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index 8deec1ed897..31207a9ce91 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -29526,6 +29526,8 @@ export interface components {
              * @default []
              */
             supported_openai_params: string[] | null;
+            /** Supported Reasoning Efforts */
+            supported_reasoning_efforts?: string[] | null;
             /**
              * Supports Function Calling
              * @default false

From 764a112d687d0f0c5ab1cfdcdc077b984e111be6 Mon Sep 17 00:00:00 2001
From: Tin Chi Lo <tin@berri.ai>
Date: Fri, 21 Aug 2026 15:34:56 -0700
Subject: [PATCH 015/105] fix(map): keep flag-thin gpt-5.6 entries unflagged

---
 litellm/model_prices_and_context_window_backup.json | 12 +++---------
 model_prices_and_context_window.json                | 12 +++---------
 2 files changed, 6 insertions(+), 18 deletions(-)

diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 4a275273fb2..a66b7963e1f 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -49060,9 +49060,7 @@
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
-        "supports_vision": true,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_vision": true
     },
     "bedrock_mantle/openai.gpt-5.6-terra": {
         "input_cost_per_token": 2.2e-06,
@@ -49094,9 +49092,7 @@
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
-        "supports_vision": true,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_vision": true
     },
     "bedrock_mantle/openai.gpt-5.6-luna": {
         "input_cost_per_token": 2.2e-07,
@@ -49128,9 +49124,7 @@
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
-        "supports_vision": true,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_vision": true
     },
     "us.openai.gpt-5.6-sol": {
         "input_cost_per_token": 5.5e-06,
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index 4a275273fb2..a66b7963e1f 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -49060,9 +49060,7 @@
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
-        "supports_vision": true,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_vision": true
     },
     "bedrock_mantle/openai.gpt-5.6-terra": {
         "input_cost_per_token": 2.2e-06,
@@ -49094,9 +49092,7 @@
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
-        "supports_vision": true,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_vision": true
     },
     "bedrock_mantle/openai.gpt-5.6-luna": {
         "input_cost_per_token": 2.2e-07,
@@ -49128,9 +49124,7 @@
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
-        "supports_vision": true,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_vision": true
     },
     "us.openai.gpt-5.6-sol": {
         "input_cost_per_token": 5.5e-06,

From a5dd0200233270742877838a2a446766f3f3c12f Mon Sep 17 00:00:00 2001
From: Tin Chi Lo <tin@berri.ai>
Date: Fri, 21 Aug 2026 15:53:55 -0700
Subject: [PATCH 016/105] fix: keep ultra plumbed but unasserted, and stop the
 bridge forwarding default

---
 .../transformation.py                         |  5 +-
 ...odel_prices_and_context_window_backup.json | 48 +++++++------------
 model_prices_and_context_window.json          | 48 +++++++------------
 model_prices_and_context_window.schema.json   |  3 --
 ...responses_transformation_transformation.py |  1 +
 .../llms/openai/test_gpt5_transformation.py   | 20 ++++++--
 6 files changed, 52 insertions(+), 73 deletions(-)

diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py
index f94bf34e8d3..d1228ac2610 100644
--- a/litellm/completion_extras/litellm_responses_transformation/transformation.py
+++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py
@@ -1114,8 +1114,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         )
 
         # Level-agnostic: providers own effort validation, so an unknown level (max, ultra, future
-        # ones) passes through instead of being silently dropped here.
-        if reasoning_effort:
+        # ones) passes through instead of being silently dropped here. "default" means no override,
+        # so it maps to nothing rather than reaching a provider as a literal effort value.
+        if reasoning_effort and reasoning_effort != "default":
             return (
                 Reasoning(effort=reasoning_effort, summary="detailed")
                 if auto_summary_enabled
diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index a66b7963e1f..e6c76983e91 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -6640,8 +6640,7 @@
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "azure/gpt-5.6-sol": {
         "cache_read_input_token_cost": 5e-07,
@@ -6693,8 +6692,7 @@
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "azure/gpt-5.6-terra": {
         "cache_read_input_token_cost": 2e-07,
@@ -6746,8 +6744,7 @@
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "azure/gpt-5.6-luna": {
         "cache_read_input_token_cost": 2e-08,
@@ -6799,8 +6796,7 @@
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "azure/us/gpt-5.6": {
         "cache_read_input_token_cost": 5.5e-07,
@@ -6848,8 +6844,7 @@
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "azure/us/gpt-5.6-sol": {
         "cache_read_input_token_cost": 5.5e-07,
@@ -6898,8 +6893,7 @@
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "azure/us/gpt-5.6-terra": {
         "cache_read_input_token_cost": 2.2e-07,
@@ -6948,8 +6942,7 @@
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "azure/us/gpt-5.6-luna": {
         "cache_read_input_token_cost": 2.2e-08,
@@ -6998,8 +6991,7 @@
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "azure/eu/gpt-5.6": {
         "cache_read_input_token_cost": 5.5e-07,
@@ -7047,8 +7039,7 @@
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "azure/eu/gpt-5.6-sol": {
         "cache_read_input_token_cost": 5.5e-07,
@@ -7097,8 +7088,7 @@
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "azure/eu/gpt-5.6-terra": {
         "cache_read_input_token_cost": 2.2e-07,
@@ -7147,8 +7137,7 @@
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "azure/eu/gpt-5.6-luna": {
         "cache_read_input_token_cost": 2.2e-08,
@@ -7197,8 +7186,7 @@
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "azure/gpt-5.5": {
         "deprecation_date": "2027-10-26",
@@ -26430,8 +26418,7 @@
         "supports_vision": true,
         "supports_web_search": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "gpt-5.6-sol": {
         "cache_creation_input_token_cost": 5e-06,
@@ -26496,8 +26483,7 @@
         "supports_vision": true,
         "supports_web_search": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "gpt-5.6-terra": {
         "cache_creation_input_token_cost": 2.5e-06,
@@ -26561,8 +26547,7 @@
         "supports_vision": true,
         "supports_web_search": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "gpt-5.6-luna": {
         "cache_creation_input_token_cost": 2.5e-07,
@@ -26626,8 +26611,7 @@
         "supports_vision": true,
         "supports_web_search": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "gpt-5.6-cyber": {
         "cache_creation_input_token_cost": 1.5625e-05,
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index a66b7963e1f..e6c76983e91 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -6640,8 +6640,7 @@
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "azure/gpt-5.6-sol": {
         "cache_read_input_token_cost": 5e-07,
@@ -6693,8 +6692,7 @@
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "azure/gpt-5.6-terra": {
         "cache_read_input_token_cost": 2e-07,
@@ -6746,8 +6744,7 @@
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "azure/gpt-5.6-luna": {
         "cache_read_input_token_cost": 2e-08,
@@ -6799,8 +6796,7 @@
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "azure/us/gpt-5.6": {
         "cache_read_input_token_cost": 5.5e-07,
@@ -6848,8 +6844,7 @@
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "azure/us/gpt-5.6-sol": {
         "cache_read_input_token_cost": 5.5e-07,
@@ -6898,8 +6893,7 @@
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "azure/us/gpt-5.6-terra": {
         "cache_read_input_token_cost": 2.2e-07,
@@ -6948,8 +6942,7 @@
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "azure/us/gpt-5.6-luna": {
         "cache_read_input_token_cost": 2.2e-08,
@@ -6998,8 +6991,7 @@
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "azure/eu/gpt-5.6": {
         "cache_read_input_token_cost": 5.5e-07,
@@ -7047,8 +7039,7 @@
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "azure/eu/gpt-5.6-sol": {
         "cache_read_input_token_cost": 5.5e-07,
@@ -7097,8 +7088,7 @@
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "azure/eu/gpt-5.6-terra": {
         "cache_read_input_token_cost": 2.2e-07,
@@ -7147,8 +7137,7 @@
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "azure/eu/gpt-5.6-luna": {
         "cache_read_input_token_cost": 2.2e-08,
@@ -7197,8 +7186,7 @@
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "azure/gpt-5.5": {
         "deprecation_date": "2027-10-26",
@@ -26430,8 +26418,7 @@
         "supports_vision": true,
         "supports_web_search": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "gpt-5.6-sol": {
         "cache_creation_input_token_cost": 5e-06,
@@ -26496,8 +26483,7 @@
         "supports_vision": true,
         "supports_web_search": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "gpt-5.6-terra": {
         "cache_creation_input_token_cost": 2.5e-06,
@@ -26561,8 +26547,7 @@
         "supports_vision": true,
         "supports_web_search": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "gpt-5.6-luna": {
         "cache_creation_input_token_cost": 2.5e-07,
@@ -26626,8 +26611,7 @@
         "supports_vision": true,
         "supports_web_search": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_max_reasoning_effort": true,
-        "supports_ultra_reasoning_effort": true
+        "supports_max_reasoning_effort": true
     },
     "gpt-5.6-cyber": {
         "cache_creation_input_token_cost": 1.5625e-05,
diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json
index 75bf3d47f35..f7f60c7666d 100644
--- a/model_prices_and_context_window.schema.json
+++ b/model_prices_and_context_window.schema.json
@@ -702,9 +702,6 @@
         "supports_tool_search": {
           "type": "boolean"
         },
-        "supports_ultra_reasoning_effort": {
-          "type": "boolean"
-        },
         "supports_url_context": {
           "type": "boolean"
         },
diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py
index a0aecfafdbc..5c76fe5e7da 100644
--- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py
+++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py
@@ -1593,6 +1593,7 @@ def test_map_reasoning_effort_adds_summary_detailed(monkeypatch):
             result_passthrough = handler._map_reasoning_effort(effort)
             assert result_passthrough == Reasoning(effort=effort)
         assert handler._map_reasoning_effort("") is None
+        assert handler._map_reasoning_effort("default") is None
         print("✓ Unenumerated reasoning_effort levels pass through to the provider")
 
         print(
diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py
index 7595c64da07..5af767f0a86 100644
--- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py
+++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py
@@ -1311,15 +1311,27 @@ def test_responses_gpt54_allow_temperature_effort_none(
     assert params["temperature"] == 0.7
 
 
-@pytest.mark.parametrize("effort", ["max", "ultra"])
-def test_gpt5_6_allows_opt_in_reasoning_efforts(config: OpenAIConfig, effort: str):
+def test_gpt5_6_allows_reasoning_effort_max(config: OpenAIConfig):
     params = config.map_openai_params(
-        non_default_params={"reasoning_effort": effort},
+        non_default_params={"reasoning_effort": "max"},
         optional_params={},
         model="gpt-5.6",
         drop_params=False,
     )
-    assert params["reasoning_effort"] == effort
+    assert params["reasoning_effort"] == "max"
+
+
+def test_gpt5_6_rejects_reasoning_effort_ultra_until_the_map_opts_in(config: OpenAIConfig):
+    # ultra is plumbed as an opt-in level but no map entry asserts it: OpenAI's model guidance
+    # documents effort values only up to max, and the builder guide frames ultra as multi-agent
+    # orchestration. A verified supports_ultra_reasoning_effort flag lights this up with no code.
+    with pytest.raises(litellm.utils.UnsupportedParamsError):
+        config.map_openai_params(
+            non_default_params={"reasoning_effort": "ultra"},
+            optional_params={},
+            model="gpt-5.6",
+            drop_params=False,
+        )
 
 
 @pytest.mark.parametrize("effort", ["max", "ultra"])

From b0fee551fdc21e1df844e313d15a51d93cf0f0b7 Mon Sep 17 00:00:00 2001
From: Mateo <ihideallmypotatoes@users.noreply.github.com>
Date: Fri, 21 Aug 2026 18:12:51 -0700
Subject: [PATCH 017/105] fix(router): resolve the none reasoning effort as
 opt-in on azure

AzureOpenAIGPT5Config raises UnsupportedParamsError on reasoning_effort='none'
unless the model map flags it, while OpenAI never refuses the level, so a single
opt-out polarity advertised none for 61 azure deployments that reject it. Defer
to _supports_factory when the azure flag is absent so the advertised list and the
request gate agree on every azure gpt-5 model in the map.

Also wrap the widened reasoning_effort Literal in main.py, which ruff format
flagged over the line limit.
---
 litellm/main.py                               |  6 ++-
 .../reasoning_effort_capability.py            | 34 ++++++++++++--
 .../test_reasoning_effort_capability.py       | 44 +++++++++++++++++++
 3 files changed, 79 insertions(+), 5 deletions(-)

diff --git a/litellm/main.py b/litellm/main.py
index 1cac723c179..a1b41315b85 100644
--- a/litellm/main.py
+++ b/litellm/main.py
@@ -416,7 +416,8 @@ async def acompletion(
     logprobs: bool | None = None,
     top_logprobs: int | None = None,
     deployment_id=None,
-    reasoning_effort: Literal["none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra", "default"] | None = None,
+    reasoning_effort: Literal["none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra", "default"]
+    | None = None,
     verbosity: Literal["low", "medium", "high"] | None = None,
     safety_identifier: str | None = None,
     service_tier: str | None = None,
@@ -4920,7 +4921,8 @@ def completion(
     logit_bias: dict | None = None,
     user: str | None = None,
     # openai v1.0+ new params
-    reasoning_effort: Literal["none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra", "default"] | None = None,
+    reasoning_effort: Literal["none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra", "default"]
+    | None = None,
     verbosity: Literal["low", "medium", "high"] | None = None,
     response_format: dict | type[BaseModel] | None = None,
     seed: int | None = None,
diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py
index f59c53e9287..593b5599703 100644
--- a/litellm/router_utils/reasoning_effort_capability.py
+++ b/litellm/router_utils/reasoning_effort_capability.py
@@ -3,9 +3,15 @@
 The model-map flags carry different polarity per level, mirroring the provider gates
 (gpt_5_transformation.py restricts xhigh to explicit opt-in and treats minimal/low as opt-out;
 anthropic/chat/transformation.py rejects only xhigh/max without an explicit flag): medium and high
-are unconditional for any reasoning model, none/minimal/low are supported unless the map explicitly
+are unconditional for any reasoning model, minimal/low are supported unless the map explicitly
 says false, and xhigh/max require an explicit true. Shipping the resolved list keeps that polarity
 in one place instead of re-encoding it in every consumer.
+
+The none level is the one flag whose polarity is provider-dependent. OpenAI never refuses it on the
+request path (azure/chat/gpt_5_transformation.py is the only caller that does, and it raises
+UnsupportedParamsError unless supports_none_reasoning_effort is explicitly true), so none is opt-out
+everywhere except azure, where it is opt-in. Resolving it the other way for azure would advertise a
+level litellm itself rejects, which is the failure this module exists to prevent.
 """
 
 from collections.abc import Mapping, Sequence
@@ -14,7 +20,6 @@ from typing import Final
 REASONING_EFFORT_CAPABILITY_ORDER: Final = ("none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra")
 
 _OPT_OUT_FLAGS: Final = (
-    ("none", "supports_none_reasoning_effort"),
     ("minimal", "supports_minimal_reasoning_effort"),
     ("low", "supports_low_reasoning_effort"),
 )
@@ -24,6 +29,28 @@ _OPT_IN_FLAGS: Final = (
     ("ultra", "supports_ultra_reasoning_effort"),
 )
 _UNCONDITIONAL_EFFORTS: Final = frozenset(("medium", "high"))
+_NONE_FLAG: Final = "supports_none_reasoning_effort"
+_NONE_OPT_IN_PROVIDERS: Final = frozenset(("azure",))
+
+
+def _supports_none_reasoning_effort(model_info: Mapping[str, object]) -> bool:
+    """Opt-in on azure, whose gpt-5 config raises on reasoning_effort='none' without an explicit
+    true; opt-out elsewhere, where no request path refuses the level. A missing azure flag defers to
+    _supports_factory, the same resolver the azure gate calls, so its bare-model-name fallback
+    (azure/gpt-5.2 inheriting the flag from gpt-5.2) reaches both sides alike."""
+    flag: Final = model_info.get(_NONE_FLAG)
+    if model_info.get("litellm_provider") not in _NONE_OPT_IN_PROVIDERS:
+        return flag is not False
+    if flag is not None:
+        return flag is True
+    model_key: Final = model_info.get("key")
+    if not isinstance(model_key, str):
+        return False
+    from litellm.utils import (
+        _supports_factory,  # pyright: ignore[reportPrivateUsage]  # the resolver the azure gate itself calls; a public wrapper would fork the fallback
+    )
+
+    return _supports_factory(model=model_key, custom_llm_provider=None, key=_NONE_FLAG)
 
 
 def resolve_supported_reasoning_efforts(model_info: Mapping[str, object]) -> tuple[str, ...] | None:
@@ -35,7 +62,8 @@ def resolve_supported_reasoning_efforts(model_info: Mapping[str, object]) -> tup
         return ()
     opt_out: Final = frozenset(effort for effort, flag in _OPT_OUT_FLAGS if model_info.get(flag) is not False)
     opt_in: Final = frozenset(effort for effort, flag in _OPT_IN_FLAGS if model_info.get(flag) is True)
-    allowed: Final = opt_out | _UNCONDITIONAL_EFFORTS | opt_in
+    none_level: Final = frozenset(("none",)) if _supports_none_reasoning_effort(model_info) else frozenset()
+    allowed: Final = opt_out | _UNCONDITIONAL_EFFORTS | opt_in | none_level
     return tuple(effort for effort in REASONING_EFFORT_CAPABILITY_ORDER if effort in allowed)
 
 
diff --git a/tests/test_litellm/router_utils/test_reasoning_effort_capability.py b/tests/test_litellm/router_utils/test_reasoning_effort_capability.py
index ebc7952081c..1c7d5355549 100644
--- a/tests/test_litellm/router_utils/test_reasoning_effort_capability.py
+++ b/tests/test_litellm/router_utils/test_reasoning_effort_capability.py
@@ -1,3 +1,5 @@
+import pytest
+
 from litellm.router_utils.reasoning_effort_capability import (
     intersect_supported_reasoning_efforts,
     resolve_supported_reasoning_efforts,
@@ -64,6 +66,48 @@ class TestResolveSupportedReasoningEfforts:
         assert "xhigh" not in resolved
 
 
+class TestNoneLevelPolarity:
+    def test_none_stays_opt_out_off_azure(self):
+        resolved = resolve_supported_reasoning_efforts(
+            {"supports_reasoning": True, "litellm_provider": "openai", "key": "gpt-5-mini"}
+        )
+        assert resolved is not None and "none" in resolved
+
+    def test_azure_without_the_flag_does_not_advertise_none(self):
+        resolved = resolve_supported_reasoning_efforts(
+            {"supports_reasoning": True, "litellm_provider": "azure", "key": "azure/unmapped-deployment"}
+        )
+        assert resolved == ("minimal", "low", "medium", "high")
+
+    def test_azure_with_the_flag_advertises_none(self):
+        resolved = resolve_supported_reasoning_efforts(
+            {
+                "supports_reasoning": True,
+                "litellm_provider": "azure",
+                "supports_none_reasoning_effort": True,
+                "key": "azure/unmapped-deployment",
+            }
+        )
+        assert resolved is not None and "none" in resolved
+
+    @pytest.mark.parametrize(
+        "model_key",
+        ["azure/gpt-5", "azure/gpt-5-mini", "azure/gpt-5-nano", "azure/gpt-5.2", "azure/gpt-5.6"],
+    )
+    def test_azure_advertisement_matches_the_azure_request_gate(self, model_key):
+        """AzureOpenAIGPT5Config raises UnsupportedParamsError on reasoning_effort='none' for models
+        it does not flag, so advertising the level there would offer routing a 400."""
+        from litellm.llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config
+        from litellm.utils import _get_model_info_helper
+
+        model_info = dict(_get_model_info_helper(model=model_key.split("/", 1)[1], custom_llm_provider="azure"))
+        resolved = resolve_supported_reasoning_efforts(model_info)
+
+        assert resolved is not None
+        gate_accepts_none = AzureOpenAIGPT5Config._supports_reasoning_effort_level(model_key, "none")
+        assert ("none" in resolved) is gate_accepts_none
+
+
 class TestIntersectSupportedReasoningEfforts:
     def test_unknown_never_narrows(self):
         assert intersect_supported_reasoning_efforts(["medium", "high"], None) == ("medium", "high")

From 6ec49ea7ddfa3c36139c9d28ca6521b1656c84b7 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Fri, 21 Aug 2026 18:25:39 -0700
Subject: [PATCH 018/105] fix(map): stop claiming max reasoning effort for
 gpt-5.6 on chat completions

/v1/chat/completions rejects reasoning_effort=max on every gpt-5.6 snapshot with
"Unsupported value: 'reasoning_effort' does not support 'max' with this model.
Supported values are: 'none', 'low', 'medium', 'high', and 'xhigh'", so the 16
gpt-5.6 entries that asserted supports_max_reasoning_effort had model groups
advertising a level routing would always get a 400 for.

The flag reads as chat-surface capability everywhere else in the map: before
this branch no openai or azure entry carried it at all, only anthropic-family
ones whose chat endpoint really does take max. Dropping it lines the gpt-5.6
advertisement up with what the endpoint accepts and lets the chat gate refuse
the level with our own error instead of forwarding it into a provider 400.

/v1/responses does accept max for gpt-5.6 and is unaffected: the responses
config gates none alone, so the cursor thinking-max variant keeps resolving.
---
 ...odel_prices_and_context_window_backup.json | 48 +++++++------------
 model_prices_and_context_window.json          | 48 +++++++------------
 .../llms/openai/test_gpt5_transformation.py   | 30 ++++++++++--
 3 files changed, 57 insertions(+), 69 deletions(-)

diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index e6c76983e91..c36018feb9d 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -6639,8 +6639,7 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true
+        "supports_minimal_reasoning_effort": false
     },
     "azure/gpt-5.6-sol": {
         "cache_read_input_token_cost": 5e-07,
@@ -6691,8 +6690,7 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true
+        "supports_minimal_reasoning_effort": false
     },
     "azure/gpt-5.6-terra": {
         "cache_read_input_token_cost": 2e-07,
@@ -6743,8 +6741,7 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true
+        "supports_minimal_reasoning_effort": false
     },
     "azure/gpt-5.6-luna": {
         "cache_read_input_token_cost": 2e-08,
@@ -6795,8 +6792,7 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true
+        "supports_minimal_reasoning_effort": false
     },
     "azure/us/gpt-5.6": {
         "cache_read_input_token_cost": 5.5e-07,
@@ -6843,8 +6839,7 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true
+        "supports_minimal_reasoning_effort": false
     },
     "azure/us/gpt-5.6-sol": {
         "cache_read_input_token_cost": 5.5e-07,
@@ -6892,8 +6887,7 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true
+        "supports_minimal_reasoning_effort": false
     },
     "azure/us/gpt-5.6-terra": {
         "cache_read_input_token_cost": 2.2e-07,
@@ -6941,8 +6935,7 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true
+        "supports_minimal_reasoning_effort": false
     },
     "azure/us/gpt-5.6-luna": {
         "cache_read_input_token_cost": 2.2e-08,
@@ -6990,8 +6983,7 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true
+        "supports_minimal_reasoning_effort": false
     },
     "azure/eu/gpt-5.6": {
         "cache_read_input_token_cost": 5.5e-07,
@@ -7038,8 +7030,7 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true
+        "supports_minimal_reasoning_effort": false
     },
     "azure/eu/gpt-5.6-sol": {
         "cache_read_input_token_cost": 5.5e-07,
@@ -7087,8 +7078,7 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true
+        "supports_minimal_reasoning_effort": false
     },
     "azure/eu/gpt-5.6-terra": {
         "cache_read_input_token_cost": 2.2e-07,
@@ -7136,8 +7126,7 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true
+        "supports_minimal_reasoning_effort": false
     },
     "azure/eu/gpt-5.6-luna": {
         "cache_read_input_token_cost": 2.2e-08,
@@ -7185,8 +7174,7 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true
+        "supports_minimal_reasoning_effort": false
     },
     "azure/gpt-5.5": {
         "deprecation_date": "2027-10-26",
@@ -26417,8 +26405,7 @@
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_web_search": true,
-        "supports_xhigh_reasoning_effort": true,
-        "supports_max_reasoning_effort": true
+        "supports_xhigh_reasoning_effort": true
     },
     "gpt-5.6-sol": {
         "cache_creation_input_token_cost": 5e-06,
@@ -26482,8 +26469,7 @@
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_web_search": true,
-        "supports_xhigh_reasoning_effort": true,
-        "supports_max_reasoning_effort": true
+        "supports_xhigh_reasoning_effort": true
     },
     "gpt-5.6-terra": {
         "cache_creation_input_token_cost": 2.5e-06,
@@ -26546,8 +26532,7 @@
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_web_search": true,
-        "supports_xhigh_reasoning_effort": true,
-        "supports_max_reasoning_effort": true
+        "supports_xhigh_reasoning_effort": true
     },
     "gpt-5.6-luna": {
         "cache_creation_input_token_cost": 2.5e-07,
@@ -26610,8 +26595,7 @@
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_web_search": true,
-        "supports_xhigh_reasoning_effort": true,
-        "supports_max_reasoning_effort": true
+        "supports_xhigh_reasoning_effort": true
     },
     "gpt-5.6-cyber": {
         "cache_creation_input_token_cost": 1.5625e-05,
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index e6c76983e91..c36018feb9d 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -6639,8 +6639,7 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true
+        "supports_minimal_reasoning_effort": false
     },
     "azure/gpt-5.6-sol": {
         "cache_read_input_token_cost": 5e-07,
@@ -6691,8 +6690,7 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true
+        "supports_minimal_reasoning_effort": false
     },
     "azure/gpt-5.6-terra": {
         "cache_read_input_token_cost": 2e-07,
@@ -6743,8 +6741,7 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true
+        "supports_minimal_reasoning_effort": false
     },
     "azure/gpt-5.6-luna": {
         "cache_read_input_token_cost": 2e-08,
@@ -6795,8 +6792,7 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true
+        "supports_minimal_reasoning_effort": false
     },
     "azure/us/gpt-5.6": {
         "cache_read_input_token_cost": 5.5e-07,
@@ -6843,8 +6839,7 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true
+        "supports_minimal_reasoning_effort": false
     },
     "azure/us/gpt-5.6-sol": {
         "cache_read_input_token_cost": 5.5e-07,
@@ -6892,8 +6887,7 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true
+        "supports_minimal_reasoning_effort": false
     },
     "azure/us/gpt-5.6-terra": {
         "cache_read_input_token_cost": 2.2e-07,
@@ -6941,8 +6935,7 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true
+        "supports_minimal_reasoning_effort": false
     },
     "azure/us/gpt-5.6-luna": {
         "cache_read_input_token_cost": 2.2e-08,
@@ -6990,8 +6983,7 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true
+        "supports_minimal_reasoning_effort": false
     },
     "azure/eu/gpt-5.6": {
         "cache_read_input_token_cost": 5.5e-07,
@@ -7038,8 +7030,7 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true
+        "supports_minimal_reasoning_effort": false
     },
     "azure/eu/gpt-5.6-sol": {
         "cache_read_input_token_cost": 5.5e-07,
@@ -7087,8 +7078,7 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true
+        "supports_minimal_reasoning_effort": false
     },
     "azure/eu/gpt-5.6-terra": {
         "cache_read_input_token_cost": 2.2e-07,
@@ -7136,8 +7126,7 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true
+        "supports_minimal_reasoning_effort": false
     },
     "azure/eu/gpt-5.6-luna": {
         "cache_read_input_token_cost": 2.2e-08,
@@ -7185,8 +7174,7 @@
         "supports_web_search": true,
         "supports_none_reasoning_effort": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_minimal_reasoning_effort": false,
-        "supports_max_reasoning_effort": true
+        "supports_minimal_reasoning_effort": false
     },
     "azure/gpt-5.5": {
         "deprecation_date": "2027-10-26",
@@ -26417,8 +26405,7 @@
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_web_search": true,
-        "supports_xhigh_reasoning_effort": true,
-        "supports_max_reasoning_effort": true
+        "supports_xhigh_reasoning_effort": true
     },
     "gpt-5.6-sol": {
         "cache_creation_input_token_cost": 5e-06,
@@ -26482,8 +26469,7 @@
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_web_search": true,
-        "supports_xhigh_reasoning_effort": true,
-        "supports_max_reasoning_effort": true
+        "supports_xhigh_reasoning_effort": true
     },
     "gpt-5.6-terra": {
         "cache_creation_input_token_cost": 2.5e-06,
@@ -26546,8 +26532,7 @@
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_web_search": true,
-        "supports_xhigh_reasoning_effort": true,
-        "supports_max_reasoning_effort": true
+        "supports_xhigh_reasoning_effort": true
     },
     "gpt-5.6-luna": {
         "cache_creation_input_token_cost": 2.5e-07,
@@ -26610,8 +26595,7 @@
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_web_search": true,
-        "supports_xhigh_reasoning_effort": true,
-        "supports_max_reasoning_effort": true
+        "supports_xhigh_reasoning_effort": true
     },
     "gpt-5.6-cyber": {
         "cache_creation_input_token_cost": 1.5625e-05,
diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py
index 5af767f0a86..f2fc7934164 100644
--- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py
+++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py
@@ -1311,14 +1311,34 @@ def test_responses_gpt54_allow_temperature_effort_none(
     assert params["temperature"] == 0.7
 
 
-def test_gpt5_6_allows_reasoning_effort_max(config: OpenAIConfig):
-    params = config.map_openai_params(
-        non_default_params={"reasoning_effort": "max"},
-        optional_params={},
+@pytest.mark.parametrize("model", ["gpt-5.6", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"])
+def test_gpt5_6_rejects_reasoning_effort_max_on_chat_completions(config: OpenAIConfig, model: str):
+    """/v1/chat/completions answers max with "Unsupported value: 'reasoning_effort' does not support
+    'max' with this model. Supported values are: 'none', 'low', 'medium', 'high', and 'xhigh'" on
+    every gpt-5.6 snapshot, so no gpt-5.6 map entry asserts supports_max_reasoning_effort and the
+    level is refused here instead of being forwarded into a provider 400."""
+    with pytest.raises(litellm.utils.UnsupportedParamsError):
+        config.map_openai_params(
+            non_default_params={"reasoning_effort": "max"},
+            optional_params={},
+            model=model,
+            drop_params=False,
+        )
+
+
+def test_gpt5_6_keeps_reasoning_effort_max_on_the_responses_api(
+    responses_config: OpenAIResponsesAPIConfig,
+):
+    """/v1/responses accepts max for gpt-5.6, and that is the surface the cursor thinking-max
+    variant resolves onto, so the responses path keeps carrying the level chat refuses."""
+    params = responses_config.map_openai_params(
+        response_api_optional_params=ResponsesAPIOptionalRequestParams(
+            reasoning={"effort": "max"},
+        ),
         model="gpt-5.6",
         drop_params=False,
     )
-    assert params["reasoning_effort"] == "max"
+    assert params["reasoning"] == {"effort": "max"}
 
 
 def test_gpt5_6_rejects_reasoning_effort_ultra_until_the_map_opts_in(config: OpenAIConfig):

From 18528b1a6306321f927bcad5fa609fb54500ef86 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Fri, 21 Aug 2026 19:16:28 -0700
Subject: [PATCH 019/105] fix(reasoning): keep the chat gate on xhigh and stop
 empty groups zeroing the picker

The chat-completions gate only ever owned xhigh. Widening it to max and ultra
made gpt-5.6 answer 400 on requests litellm itself converts to /v1/responses,
where max is valid, because the gate runs before the bridge decision. No map
entry asserts either flag, so the widened gate could only ever reject.

An empty per-group intersection now falls back to the capability-blind level
list in the dashboard, matching what the picker showed before the field
existed, and ModelGroupInfo tolerates whatever shape an operator writes under
supported_reasoning_efforts instead of failing the whole /model_group/info
response.
---
 .../llms/openai/chat/gpt_5_transformation.py  |  4 +-
 .../reasoning_effort_capability.py            |  8 +-
 litellm/types/router.py                       | 13 ++++
 .../llms/openai/test_gpt5_transformation.py   | 77 ++++++++++++-------
 tests/test_litellm/test_router.py             | 35 ++++++++-
 .../add_model/ComplexityRouterConfig.test.tsx | 18 +++++
 .../add_model/ComplexityRouterConfig.tsx      | 23 ++++--
 7 files changed, 137 insertions(+), 41 deletions(-)

diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py
index c55640f6a1f..3a79e6570e9 100644
--- a/litellm/llms/openai/chat/gpt_5_transformation.py
+++ b/litellm/llms/openai/chat/gpt_5_transformation.py
@@ -222,8 +222,8 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
                 if "reasoning_effort" in optional_params:
                     optional_params["reasoning_effort"] = normalized
 
-        if effective_effort in ("xhigh", "max", "ultra"):
-            # xhigh/max/ultra are opt-in capabilities: only allow if the model explicitly supports them.
+        if effective_effort == "xhigh":
+            # xhigh is an opt-in capability: only allow if model explicitly supports it.
             if not self._supports_reasoning_effort_level(model, effective_effort):
                 if litellm.drop_params or drop_params:
                     non_default_params.pop("reasoning_effort", None)
diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py
index 593b5599703..7c8b5523d7c 100644
--- a/litellm/router_utils/reasoning_effort_capability.py
+++ b/litellm/router_utils/reasoning_effort_capability.py
@@ -54,8 +54,12 @@ def _supports_none_reasoning_effort(model_info: Mapping[str, object]) -> bool:
 
 
 def resolve_supported_reasoning_efforts(model_info: Mapping[str, object]) -> tuple[str, ...] | None:
-    """None = no capability metadata for this deployment (e.g. a model absent from the model map,
-    whose stub info carries no supports_reasoning key at all); () = reasoning unsupported."""
+    """None = the caller passed no supports_reasoning key at all; () = this deployment adds no
+    effort levels to its group. The router always supplies the key, so a deployment absent from the
+    model map arrives with supports_reasoning None and lands on (), the same answer a mapped
+    non-reasoning model gets: the group cannot promise a level on behalf of a model nothing is known
+    about. () therefore reads as "no usable answer" downstream, which is why the dashboard falls
+    back to the capability-blind level list on an empty group rather than hiding the control."""
     if "supports_reasoning" not in model_info:
         return None
     if model_info.get("supports_reasoning") is not True:
diff --git a/litellm/types/router.py b/litellm/types/router.py
index d4c735387a5..fd32b70405b 100644
--- a/litellm/types/router.py
+++ b/litellm/types/router.py
@@ -641,6 +641,19 @@ class ModelGroupInfo(BaseModel):
     supported_openai_params: list[str] | None = Field(default=[])
     configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None
 
+    @field_validator("supported_reasoning_efforts", mode="before")
+    @classmethod
+    def _accept_only_well_formed_reasoning_efforts(cls, value: object) -> tuple[str, ...] | None:
+        """Deployment model_info reaches this model through a **kwargs splat, so an operator can put
+        any shape under this key. Anything that is not a level or a list of levels resolves to None
+        and lets the computed intersection stand, because raising here fails the whole
+        /model_group/info response rather than the one group that carries the bad value."""
+        if isinstance(value, str):
+            return (value,)
+        if isinstance(value, (list, tuple)) and all(isinstance(level, str) for level in value):
+            return tuple(value)
+        return None
+
     def __init__(self, **data) -> None:
         for field_name, field_type in get_type_hints(self.__class__).items():
             if field_type is bool and data.get(field_name) is None:
diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py
index f2fc7934164..eaae3d8c2c5 100644
--- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py
+++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py
@@ -1312,18 +1312,32 @@ def test_responses_gpt54_allow_temperature_effort_none(
 
 
 @pytest.mark.parametrize("model", ["gpt-5.6", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"])
-def test_gpt5_6_rejects_reasoning_effort_max_on_chat_completions(config: OpenAIConfig, model: str):
+def test_gpt5_6_forwards_reasoning_effort_max_for_the_responses_bridge(config: OpenAIConfig, model: str):
+    """A chat request carrying tools or a reasoning summary is converted to /v1/responses further
+    down main.py, and that surface accepts max. This runs before litellm has decided to bridge, so
+    refusing max here would break the cursor thinking-max shape that works today. Plain chat
+    completions still answer max with a provider 400, and the capability list below is what keeps
+    the level out of the picker."""
+    params = config.map_openai_params(
+        non_default_params={"reasoning_effort": "max"},
+        optional_params={},
+        model=model,
+        drop_params=False,
+    )
+    assert params["reasoning_effort"] == "max"
+
+
+@pytest.mark.parametrize("model", ["gpt-5.6", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"])
+def test_gpt5_6_never_advertises_reasoning_effort_max(model: str):
     """/v1/chat/completions answers max with "Unsupported value: 'reasoning_effort' does not support
-    'max' with this model. Supported values are: 'none', 'low', 'medium', 'high', and 'xhigh'" on
-    every gpt-5.6 snapshot, so no gpt-5.6 map entry asserts supports_max_reasoning_effort and the
-    level is refused here instead of being forwarded into a provider 400."""
-    with pytest.raises(litellm.utils.UnsupportedParamsError):
-        config.map_openai_params(
-            non_default_params={"reasoning_effort": "max"},
-            optional_params={},
-            model=model,
-            drop_params=False,
-        )
+    'max' with this model. Supported values are: 'none', 'low', 'medium', 'high', and 'xhigh'", so no
+    gpt-5.6 entry asserts supports_max_reasoning_effort and the advertised set stops at xhigh."""
+    from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts
+
+    resolved = resolve_supported_reasoning_efforts(litellm.get_model_info(model))
+    assert resolved is not None
+    assert "max" not in resolved
+    assert "xhigh" in resolved
 
 
 def test_gpt5_6_keeps_reasoning_effort_max_on_the_responses_api(
@@ -1341,34 +1355,43 @@ def test_gpt5_6_keeps_reasoning_effort_max_on_the_responses_api(
     assert params["reasoning"] == {"effort": "max"}
 
 
-def test_gpt5_6_rejects_reasoning_effort_ultra_until_the_map_opts_in(config: OpenAIConfig):
-    # ultra is plumbed as an opt-in level but no map entry asserts it: OpenAI's model guidance
-    # documents effort values only up to max, and the builder guide frames ultra as multi-agent
-    # orchestration. A verified supports_ultra_reasoning_effort flag lights this up with no code.
-    with pytest.raises(litellm.utils.UnsupportedParamsError):
-        config.map_openai_params(
-            non_default_params={"reasoning_effort": "ultra"},
-            optional_params={},
-            model="gpt-5.6",
-            drop_params=False,
-        )
+def test_gpt5_6_never_advertises_reasoning_effort_ultra():
+    """ultra is plumbed as an opt-in level but no map entry asserts it: OpenAI's model guidance
+    documents effort values only up to max, and /v1/responses answers ultra with a 400. A verified
+    supports_ultra_reasoning_effort flag lights it up with no code change."""
+    from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts
+
+    resolved = resolve_supported_reasoning_efforts(litellm.get_model_info("gpt-5.6"))
+    assert resolved is not None
+    assert "ultra" not in resolved
 
 
 @pytest.mark.parametrize("effort", ["max", "ultra"])
-def test_gpt5_rejects_opt_in_reasoning_efforts_for_other_models(config: OpenAIConfig, effort: str):
+def test_gpt5_forwards_levels_the_chat_gate_does_not_own(config: OpenAIConfig, effort: str):
+    """Only xhigh is gated on this surface. max and ultra reach the provider (or the responses
+    bridge) and are answered there, which is what happened before per-group capabilities existed."""
+    params = config.map_openai_params(
+        non_default_params={"reasoning_effort": effort},
+        optional_params={},
+        model="gpt-5.1",
+        drop_params=False,
+    )
+    assert params["reasoning_effort"] == effort
+
+
+def test_gpt5_rejects_xhigh_for_models_without_the_flag(config: OpenAIConfig):
     with pytest.raises(litellm.utils.UnsupportedParamsError):
         config.map_openai_params(
-            non_default_params={"reasoning_effort": effort},
+            non_default_params={"reasoning_effort": "xhigh"},
             optional_params={},
             model="gpt-5.1",
             drop_params=False,
         )
 
 
-@pytest.mark.parametrize("effort", ["max", "ultra"])
-def test_gpt5_drops_opt_in_reasoning_efforts_when_requested(config: OpenAIConfig, effort: str):
+def test_gpt5_drops_xhigh_when_requested(config: OpenAIConfig):
     params = config.map_openai_params(
-        non_default_params={"reasoning_effort": effort},
+        non_default_params={"reasoning_effort": "xhigh"},
         optional_params={},
         model="gpt-5.1",
         drop_params=True,
diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py
index 095c962328a..62c6411c3cc 100644
--- a/tests/test_litellm/test_router.py
+++ b/tests/test_litellm/test_router.py
@@ -8927,7 +8927,11 @@ def test_model_group_info_intersects_supported_reasoning_efforts():
     assert result.supported_reasoning_efforts == ("minimal", "low", "medium", "high")
 
 
-def test_model_group_info_reasoning_efforts_ignore_deployments_without_metadata():
+def test_model_group_info_reasoning_efforts_empty_when_a_deployment_is_off_the_map():
+    """The router fills every ModelInfo key, so a deployment absent from the model map arrives with
+    supports_reasoning None rather than with the key missing, and the group can no longer promise a
+    level on its behalf. The dashboard reads the empty result as "nothing known" and falls back to
+    the capability-blind picker, which is what it showed before this field existed."""
     router = litellm.Router(
         model_list=[
             {
@@ -8952,7 +8956,7 @@ def test_model_group_info_reasoning_efforts_ignore_deployments_without_metadata(
                 "supports_reasoning": True,
                 "supports_max_reasoning_effort": True,
             }
-        return {"key": model_name, "litellm_provider": "openai", "mode": "chat"}
+        return {"key": model_name, "litellm_provider": "openai", "mode": None, "supports_reasoning": None}
 
     with patch.object(router, "get_deployment_model_info", side_effect=_model_info):
         result = router._set_model_group_info(
@@ -8961,4 +8965,29 @@ def test_model_group_info_reasoning_efforts_ignore_deployments_without_metadata(
         )
 
     assert result is not None
-    assert result.supported_reasoning_efforts == ("none", "minimal", "low", "medium", "high", "max")
+    assert result.supported_reasoning_efforts == ()
+
+
+@pytest.mark.parametrize(
+    "configured, expected",
+    [
+        ("high", ("high",)),
+        (["low", "high"], ("low", "high")),
+        (17, None),
+        ({"effort": "high"}, None),
+        ([1, 2], None),
+    ],
+)
+def test_model_group_info_tolerates_any_configured_reasoning_efforts_shape(configured, expected):
+    """Deployment model_info is splatted into ModelGroupInfo, so this key arrives with whatever an
+    operator wrote in the config. A shape pydantic cannot validate used to fail the whole
+    /model_group/info response, not just the group carrying it."""
+    from litellm.types.router import ModelGroupInfo
+
+    info = ModelGroupInfo(
+        model_group="g",
+        providers=["openai"],
+        supported_reasoning_efforts=configured,
+    )
+
+    assert info.supported_reasoning_efforts == expected
diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx
index a26197aa243..09a710c8729 100644
--- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx
@@ -968,6 +968,24 @@ describe("ComplexityRouterConfig per-model effort filtering", () => {
     expect(options).toEqual(["Default", "none", "minimal", "low", "medium", "high", "xhigh"]);
   });
 
+  it("falls back to every effort when the group intersects to nothing", async () => {
+    renderWithProviders(
+      <ComplexityRouterConfig
+        {...baseProps}
+        modelInfo={[
+          ...mockModelInfo.filter((model) => model.model_group !== "claude-3-opus"),
+          { model_group: "claude-3-opus", mode: "chat", supports_reasoning: true, supported_reasoning_efforts: [] },
+        ]}
+      />,
+    );
+    const user = userEvent.setup();
+    await user.click(
+      screen.getByRole("combobox", { name: "Reasoning effort for claude-3-opus in the Reasoning tier" }),
+    );
+    const options = (await screen.findAllByRole("option")).map((option) => option.textContent);
+    expect(options).toEqual(["Default", "none", "minimal", "low", "medium", "high", "xhigh"]);
+  });
+
   // Hand-authored configs can carry a level outside the supported set (e.g. max); it must render
   // and stay clearable rather than being masked as Default.
   it("keeps showing a stored effort outside the supported set", () => {
diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
index e72905c2f73..77e093b63d9 100644
--- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
@@ -226,6 +226,20 @@ export const effectiveTierLabel = (tier: keyof ComplexityTiers, tierLabels: Comp
 export const planModeEligibleTiers = (tiers: ComplexityTiers): Array<keyof ComplexityTiers> =>
   TIER_KEYS.filter((tier) => (tiers[tier] ?? []).length > 0);
 
+/**
+ * The backend list is the per-group intersection of accepted effort levels. An empty intersection
+ * carries no more information than a proxy that does not send the field yet (one deployment absent
+ * from the model map empties it), so both fall back to the coarse supports_reasoning gate with every
+ * level offered, which is what this dropdown showed before the field existed.
+ */
+const effortOptionsForModel = (model: ModelGroup): string[] => {
+  const advertised = model.supported_reasoning_efforts ?? [];
+  if (advertised.length > 0) {
+    return [...advertised];
+  }
+  return model.supports_reasoning ? [...REASONING_EFFORT_OPTIONS] : [];
+};
+
 const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
   modelInfo,
   value,
@@ -251,16 +265,11 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
   const derivedDefaultModel = resolveComplexityDefaultModel(value.tiers);
   const defaultModel = resolveComplexityDefaultModel(value.tiers, value.default_model);
 
-  // Embedding models can't serve a chat-completion role, so they're excluded here.
-  // The backend list is the per-group intersection of accepted effort levels; when a proxy does not
-  // send it yet, fall back to the coarse supports_reasoning gate with every level offered.
   const effortOptionsByModel: Record<string, string[]> = Object.fromEntries(
-    modelInfo.map((model) => [
-      model.model_group,
-      model.supported_reasoning_efforts ?? (model.supports_reasoning ? [...REASONING_EFFORT_OPTIONS] : []),
-    ]),
+    modelInfo.map((model) => [model.model_group, effortOptionsForModel(model)]),
   );
 
+  // Embedding models can't serve a chat-completion role, so they're excluded here.
   const modelOptions = modelInfo
     .filter((model) => model.mode !== "embedding")
     .map((model) => ({

From 96acac1f9152454518116578fdcc6b36c7cbb868 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Fri, 21 Aug 2026 19:34:41 -0700
Subject: [PATCH 020/105] fix(router): tell an unknown deployment apart from a
 known non-reasoning one

get_model_info answers supports_reasoning None both for a model absent from
the map, which the router registers under a synthesized entry, and for a
mapped model that simply is not a reasoning model. Reading both as "adds no
levels" let one custom deployment wipe every level its mapped siblings agreed
on.

The synthesized entry carries no mode, which every real map entry for a
routable model does, so an unset flag with no mode now resolves to unknown and
never narrows its group. A group that genuinely shares no level still
advertises none, and the dashboard drops the effort control for it instead of
offering levels routing would refuse.
---
 .../reasoning_effort_capability.py            | 18 ++++---
 .../test_reasoning_effort_capability.py       | 10 +++-
 tests/test_litellm/test_router.py             | 47 +++++++++++++++++--
 .../add_model/ComplexityRouterConfig.test.tsx | 13 +++--
 .../add_model/ComplexityRouterConfig.tsx      | 21 +++------
 5 files changed, 75 insertions(+), 34 deletions(-)

diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py
index 7c8b5523d7c..53a159af7d1 100644
--- a/litellm/router_utils/reasoning_effort_capability.py
+++ b/litellm/router_utils/reasoning_effort_capability.py
@@ -54,15 +54,19 @@ def _supports_none_reasoning_effort(model_info: Mapping[str, object]) -> bool:
 
 
 def resolve_supported_reasoning_efforts(model_info: Mapping[str, object]) -> tuple[str, ...] | None:
-    """None = the caller passed no supports_reasoning key at all; () = this deployment adds no
-    effort levels to its group. The router always supplies the key, so a deployment absent from the
-    model map arrives with supports_reasoning None and lands on (), the same answer a mapped
-    non-reasoning model gets: the group cannot promise a level on behalf of a model nothing is known
-    about. () therefore reads as "no usable answer" downstream, which is why the dashboard falls
-    back to the capability-blind level list on an empty group rather than hiding the control."""
+    """None = nothing is known about this deployment, so it must not narrow its group; () = this is a
+    known model that accepts no effort level, which correctly empties the group. Keeping the two
+    apart matters because the router registers a deployment absent from the model map under a
+    synthesized entry, and get_model_info then answers with supports_reasoning None exactly as it
+    does for a mapped non-reasoning model. That entry carries no mode, which every real map entry for
+    a routable model does, so an unset flag with no mode is the unknown case and one custom model in
+    a group no longer wipes the levels its mapped siblings agree on."""
     if "supports_reasoning" not in model_info:
         return None
-    if model_info.get("supports_reasoning") is not True:
+    supports_reasoning: Final = model_info.get("supports_reasoning")
+    if supports_reasoning is None and model_info.get("mode") is None:
+        return None
+    if supports_reasoning is not True:
         return ()
     opt_out: Final = frozenset(effort for effort, flag in _OPT_OUT_FLAGS if model_info.get(flag) is not False)
     opt_in: Final = frozenset(effort for effort, flag in _OPT_IN_FLAGS if model_info.get(flag) is True)
diff --git a/tests/test_litellm/router_utils/test_reasoning_effort_capability.py b/tests/test_litellm/router_utils/test_reasoning_effort_capability.py
index 1c7d5355549..f3d71d19c47 100644
--- a/tests/test_litellm/router_utils/test_reasoning_effort_capability.py
+++ b/tests/test_litellm/router_utils/test_reasoning_effort_capability.py
@@ -11,8 +11,16 @@ class TestResolveSupportedReasoningEfforts:
         assert resolve_supported_reasoning_efforts({}) is None
 
     def test_non_reasoning_model_supports_no_efforts(self):
-        assert resolve_supported_reasoning_efforts({"supports_reasoning": None}) == ()
         assert resolve_supported_reasoning_efforts({"supports_reasoning": False}) == ()
+        assert resolve_supported_reasoning_efforts({"mode": "chat", "supports_reasoning": None}) == ()
+
+    def test_unset_flag_on_an_entry_with_no_mode_resolves_to_unknown(self):
+        # The router registers a deployment absent from the model map under a synthesized entry, and
+        # get_model_info then answers with supports_reasoning None just as it does for a mapped
+        # non-reasoning model. Only the missing mode tells them apart, and reading the synthesized
+        # one as () would let one custom model empty every level its mapped siblings agree on.
+        assert resolve_supported_reasoning_efforts({"supports_reasoning": None}) is None
+        assert resolve_supported_reasoning_efforts({"mode": None, "supports_reasoning": None}) is None
 
     def test_reasoning_model_with_no_flags_gets_the_opt_out_levels_only(self):
         # The kimi shape: supports_reasoning true, zero effort flags. medium/high are unconditional,
diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py
index 62c6411c3cc..ba23075e7cc 100644
--- a/tests/test_litellm/test_router.py
+++ b/tests/test_litellm/test_router.py
@@ -8927,11 +8927,11 @@ def test_model_group_info_intersects_supported_reasoning_efforts():
     assert result.supported_reasoning_efforts == ("minimal", "low", "medium", "high")
 
 
-def test_model_group_info_reasoning_efforts_empty_when_a_deployment_is_off_the_map():
+def test_model_group_info_reasoning_efforts_ignore_a_deployment_off_the_map():
     """The router fills every ModelInfo key, so a deployment absent from the model map arrives with
-    supports_reasoning None rather than with the key missing, and the group can no longer promise a
-    level on its behalf. The dashboard reads the empty result as "nothing known" and falls back to
-    the capability-blind picker, which is what it showed before this field existed."""
+    supports_reasoning None rather than with the key missing. Its synthesized entry carries no mode,
+    which is what separates it from a mapped non-reasoning model, and nothing being known about it is
+    no reason to drop the levels the rest of the group agrees on."""
     router = litellm.Router(
         model_list=[
             {
@@ -8964,6 +8964,45 @@ def test_model_group_info_reasoning_efforts_empty_when_a_deployment_is_off_the_m
             user_facing_model_group_name="smart-group",
         )
 
+    assert result is not None
+    assert result.supported_reasoning_efforts == ("none", "minimal", "low", "medium", "high", "max")
+
+
+def test_model_group_info_reasoning_efforts_empty_on_a_mapped_non_reasoning_deployment():
+    """A group mixing a reasoning model with one the map knows is not a reasoning model shares no
+    level, so it advertises none and the picker offers nothing rather than a level routing would
+    hand to a deployment that rejects it."""
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "mixed-group",
+                "litellm_params": {"model": "anthropic/opus-like"},
+                "model_info": {"id": "opus-like-deployment"},
+            },
+            {
+                "model_name": "mixed-group",
+                "litellm_params": {"model": "openai/plain-chat"},
+                "model_info": {"id": "plain-chat-deployment"},
+            },
+        ]
+    )
+
+    def _model_info(model_id: str, model_name: str):
+        if model_id == "opus-like-deployment":
+            return {
+                "key": model_name,
+                "litellm_provider": "anthropic",
+                "mode": "chat",
+                "supports_reasoning": True,
+            }
+        return {"key": model_name, "litellm_provider": "openai", "mode": "chat", "supports_reasoning": None}
+
+    with patch.object(router, "get_deployment_model_info", side_effect=_model_info):
+        result = router._set_model_group_info(
+            model_group="mixed-group",
+            user_facing_model_group_name="mixed-group",
+        )
+
     assert result is not None
     assert result.supported_reasoning_efforts == ()
 
diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx
index 09a710c8729..dbc5bd6ee86 100644
--- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx
@@ -968,7 +968,9 @@ describe("ComplexityRouterConfig per-model effort filtering", () => {
     expect(options).toEqual(["Default", "none", "minimal", "low", "medium", "high", "xhigh"]);
   });
 
-  it("falls back to every effort when the group intersects to nothing", async () => {
+  // An empty list is the group's own answer that its deployments share no level, which is different
+  // from the field being absent, so the control is dropped rather than falling back to every level.
+  it("offers no effort at all when the group intersects to nothing", () => {
     renderWithProviders(
       <ComplexityRouterConfig
         {...baseProps}
@@ -978,12 +980,9 @@ describe("ComplexityRouterConfig per-model effort filtering", () => {
         ]}
       />,
     );
-    const user = userEvent.setup();
-    await user.click(
-      screen.getByRole("combobox", { name: "Reasoning effort for claude-3-opus in the Reasoning tier" }),
-    );
-    const options = (await screen.findAllByRole("option")).map((option) => option.textContent);
-    expect(options).toEqual(["Default", "none", "minimal", "low", "medium", "high", "xhigh"]);
+    expect(
+      screen.queryByRole("combobox", { name: "Reasoning effort for claude-3-opus in the Reasoning tier" }),
+    ).not.toBeInTheDocument();
   });
 
   // Hand-authored configs can carry a level outside the supported set (e.g. max); it must render
diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
index 77e093b63d9..a113e3643cc 100644
--- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
@@ -226,20 +226,6 @@ export const effectiveTierLabel = (tier: keyof ComplexityTiers, tierLabels: Comp
 export const planModeEligibleTiers = (tiers: ComplexityTiers): Array<keyof ComplexityTiers> =>
   TIER_KEYS.filter((tier) => (tiers[tier] ?? []).length > 0);
 
-/**
- * The backend list is the per-group intersection of accepted effort levels. An empty intersection
- * carries no more information than a proxy that does not send the field yet (one deployment absent
- * from the model map empties it), so both fall back to the coarse supports_reasoning gate with every
- * level offered, which is what this dropdown showed before the field existed.
- */
-const effortOptionsForModel = (model: ModelGroup): string[] => {
-  const advertised = model.supported_reasoning_efforts ?? [];
-  if (advertised.length > 0) {
-    return [...advertised];
-  }
-  return model.supports_reasoning ? [...REASONING_EFFORT_OPTIONS] : [];
-};
-
 const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
   modelInfo,
   value,
@@ -265,8 +251,13 @@ const ComplexityRouterConfig: React.FC<ComplexityRouterConfigProps> = ({
   const derivedDefaultModel = resolveComplexityDefaultModel(value.tiers);
   const defaultModel = resolveComplexityDefaultModel(value.tiers, value.default_model);
 
+  // An absent list means the proxy does not send the field yet, so every level is offered as before.
+  // An empty list is the group's own answer that its deployments share no level, and is left empty.
   const effortOptionsByModel: Record<string, string[]> = Object.fromEntries(
-    modelInfo.map((model) => [model.model_group, effortOptionsForModel(model)]),
+    modelInfo.map((model) => [
+      model.model_group,
+      model.supported_reasoning_efforts ?? (model.supports_reasoning ? [...REASONING_EFFORT_OPTIONS] : []),
+    ]),
   );
 
   // Embedding models can't serve a chat-completion role, so they're excluded here.

From 6648ebe3cc9eaa8a6aa82270e4c6ba0e78eb0fde Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Fri, 21 Aug 2026 19:59:40 -0700
Subject: [PATCH 021/105] fix(router): always compute a group's reasoning
 efforts rather than seeding them from model_info

The ModelGroupInfo splat let a supported_reasoning_efforts value left in a deployment's
model_info seed the group, so it narrowed the group from whichever deployment was read
first and was silently ignored on every other one. The field is derived from the group's
deployments, so start it unset and let the intersection fill it in.

Also correct two docstring claims that did not match the code: the anthropic chat path
gates xhigh and max on the output_config path only, and the mode signal separates an
unknown deployment from a known non-reasoning one only while that deployment carries no
model_info of its own.
---
 litellm/router.py                             |  1 +
 .../reasoning_effort_capability.py            | 29 ++++++++++----
 tests/test_litellm/test_router.py             | 40 +++++++++++++++++++
 3 files changed, 62 insertions(+), 8 deletions(-)

diff --git a/litellm/router.py b/litellm/router.py
index 8cd1b804928..bd5f97b27c4 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -9485,6 +9485,7 @@ class Router:
                         "model_group": user_facing_model_group_name,
                         "providers": [llm_provider],
                         **model_info,
+                        "supported_reasoning_efforts": None,
                     }
                 )
             else:
diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py
index 53a159af7d1..d445d611cd6 100644
--- a/litellm/router_utils/reasoning_effort_capability.py
+++ b/litellm/router_utils/reasoning_effort_capability.py
@@ -1,11 +1,18 @@
 """Resolve which reasoning_effort values a deployment, and by intersection a model group, accepts.
 
 The model-map flags carry different polarity per level, mirroring the provider gates
-(gpt_5_transformation.py restricts xhigh to explicit opt-in and treats minimal/low as opt-out;
-anthropic/chat/transformation.py rejects only xhigh/max without an explicit flag): medium and high
-are unconditional for any reasoning model, minimal/low are supported unless the map explicitly
-says false, and xhigh/max require an explicit true. Shipping the resolved list keeps that polarity
-in one place instead of re-encoding it in every consumer.
+(gpt_5_transformation.py restricts xhigh to explicit opt-in and treats minimal/low as opt-out):
+medium and high are unconditional for any reasoning model, minimal/low are supported unless the map
+explicitly says false, and xhigh/max require an explicit true. Shipping the resolved list keeps that
+polarity in one place instead of re-encoding it in every consumer.
+
+Only openai and azure gate xhigh and max on the request path. anthropic/chat/transformation.py
+gates them on the output_config path alone, so its reasoning_effort path maps every level to a
+thinking budget whatever the map says, and a claude group whose entry omits
+supports_xhigh_reasoning_effort stops offering a level litellm would have forwarded. That is the
+deliberate trade: an explicit flag is the only signal that the tier is a real one rather than
+litellm quietly rounding the level to a budget, and what gets dropped is advisory metadata rather
+than a restriction on the request path.
 
 The none level is the one flag whose polarity is provider-dependent. OpenAI never refuses it on the
 request path (azure/chat/gpt_5_transformation.py is the only caller that does, and it raises
@@ -58,9 +65,15 @@ def resolve_supported_reasoning_efforts(model_info: Mapping[str, object]) -> tup
     known model that accepts no effort level, which correctly empties the group. Keeping the two
     apart matters because the router registers a deployment absent from the model map under a
     synthesized entry, and get_model_info then answers with supports_reasoning None exactly as it
-    does for a mapped non-reasoning model. That entry carries no mode, which every real map entry for
-    a routable model does, so an unset flag with no mode is the unknown case and one custom model in
-    a group no longer wipes the levels its mapped siblings agree on."""
+    does for a mapped non-reasoning model. Mode is what separates them: every map entry for a
+    routable model declares one, so an unset flag with no mode is read as unknown and one custom
+    model in a group no longer wipes the levels its mapped siblings agree on.
+
+    The separation is only as good as that signal. An off-map deployment carrying any model_info of
+    its own is registered under its deployment id with mode defaulting to chat, so it reads as a
+    known non-reasoning model and does still empty its group, with supports_reasoning on that
+    deployment as the way out. Telling the two apart for real needs provenance that the flattened
+    ModelInfo does not carry."""
     if "supports_reasoning" not in model_info:
         return None
     supports_reasoning: Final = model_info.get("supports_reasoning")
diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py
index ba23075e7cc..b8a5a454613 100644
--- a/tests/test_litellm/test_router.py
+++ b/tests/test_litellm/test_router.py
@@ -9007,6 +9007,46 @@ def test_model_group_info_reasoning_efforts_empty_on_a_mapped_non_reasoning_depl
     assert result.supported_reasoning_efforts == ()
 
 
+def test_model_group_info_reasoning_efforts_ignore_a_value_declared_in_model_info():
+    """The group's levels are computed from its deployments, so a value an operator left in one
+    deployment's model_info must not seed them. Seeding let the first deployment read narrow the
+    whole group while the same value on any other deployment was silently ignored."""
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "declared-group",
+                "litellm_params": {"model": "openai/first-reasoner"},
+                "model_info": {"id": "first-deployment"},
+            },
+            {
+                "model_name": "declared-group",
+                "litellm_params": {"model": "openai/second-reasoner"},
+                "model_info": {"id": "second-deployment"},
+            },
+        ]
+    )
+
+    def _model_info(model_id: str, model_name: str):
+        info = {
+            "key": model_name,
+            "litellm_provider": "openai",
+            "mode": "chat",
+            "supports_reasoning": True,
+        }
+        if model_id == "first-deployment":
+            info["supported_reasoning_efforts"] = ("high",)
+        return info
+
+    with patch.object(router, "get_deployment_model_info", side_effect=_model_info):
+        result = router._set_model_group_info(
+            model_group="declared-group",
+            user_facing_model_group_name="declared-group",
+        )
+
+    assert result is not None
+    assert result.supported_reasoning_efforts == ("none", "minimal", "low", "medium", "high")
+
+
 @pytest.mark.parametrize(
     "configured, expected",
     [

From 5b5f0004bad3edfb3c6f18a7ed23cdb029ff1738 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Fri, 21 Aug 2026 20:05:45 -0700
Subject: [PATCH 022/105] revert(ui): drop the team-path join against
 /model_group/info

The author does not want fetchAvailableModelsForTeam fanning out a second request,
so it goes back to the single /models call it was before. The team path carries no
capability metadata again, which is what it did prior to this branch, and the effort
options still come from /model_group/info on the non-team path.
---
 .../llm_calls/fetch_models.test.tsx           | 37 +------------------
 .../src/components/llm_calls/fetch_models.tsx | 14 +------
 2 files changed, 3 insertions(+), 48 deletions(-)

diff --git a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.test.tsx b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.test.tsx
index 2f35cf5ef6c..bd691c7f629 100644
--- a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.test.tsx
+++ b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.test.tsx
@@ -1,5 +1,5 @@
 import { beforeEach, describe, expect, it, vi } from "vitest";
-import { modelAvailableCall, modelHubCall } from "@/components/networking";
+import { modelAvailableCall } from "@/components/networking";
 import { fetchAvailableModelsForTeam } from "./fetch_models";
 
 vi.mock("@/components/networking", () => ({
@@ -31,38 +31,3 @@ describe("fetchAvailableModelsForTeam", () => {
     expect(await fetchAvailableModelsForTeam("token", "team-123")).toEqual([]);
   });
 });
-
-describe("fetchAvailableModelsForTeam capability join", () => {
-  it("carries model-group capabilities onto team-allowed names that have a group entry", async () => {
-    modelAvailableCallMock.mockResolvedValue({ data: [{ id: "gpt-5-mini" }, { id: "team-only-byok" }] });
-    vi.mocked(modelHubCall).mockResolvedValue({
-      data: [
-        {
-          model_group: "gpt-5-mini",
-          mode: "chat",
-          supports_reasoning: true,
-          supported_reasoning_efforts: ["minimal", "low", "medium", "high"],
-        },
-      ],
-    });
-
-    const models = await fetchAvailableModelsForTeam("token", "team-123");
-
-    expect(models).toEqual([
-      {
-        model_group: "gpt-5-mini",
-        mode: "chat",
-        supports_reasoning: true,
-        supported_reasoning_efforts: ["minimal", "low", "medium", "high"],
-      },
-      { model_group: "team-only-byok" },
-    ]);
-  });
-
-  it("keeps the team list usable when the group-info fetch fails", async () => {
-    modelAvailableCallMock.mockResolvedValue({ data: [{ id: "gpt-5-mini" }] });
-    vi.mocked(modelHubCall).mockRejectedValue(new Error("boom"));
-
-    expect(await fetchAvailableModelsForTeam("token", "team-123")).toEqual([{ model_group: "gpt-5-mini" }]);
-  });
-});
diff --git a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx
index 93f89a44813..96b87887d43 100644
--- a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx
+++ b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx
@@ -29,23 +29,13 @@ const toModelGroup = (item: AvailableModel): ModelGroup => {
   };
 };
 
-/**
- * /models carries no capability metadata, so the team-allowed names are joined against
- * /model_group/info (one extra parallel request; the endpoint has no team filter of its own, and it
- * serves team-scoped tokens row-filtered, verified live). A name without a group entry keeps every
- * capability field absent; a failed group fetch is console.error'd by fetchAvailableModels.
- */
 export const fetchAvailableModelsForTeam = async (accessToken: string, teamId: string): Promise<ModelGroup[]> => {
-  const [response, groups] = await Promise.all([
-    modelAvailableCall(accessToken, "", "", false, teamId),
-    fetchAvailableModels(accessToken).catch(() => [] as ModelGroup[]),
-  ]);
-  const byGroup = new Map(groups.map((group) => [group.model_group, group]));
+  const response = await modelAvailableCall(accessToken, "", "", false, teamId);
   const modelNames: string[] = (response?.data ?? []).map((model: { id: string }) => model.id);
 
   return excludeProxyWideSentinel(Array.from(new Set(modelNames)))
     .sort((a, b) => a.localeCompare(b))
-    .map((model) => byGroup.get(model) ?? { model_group: model });
+    .map((model) => ({ model_group: model }));
 };
 
 /**

From 67f3cf0f0f8f5a565985dc4acd173ebf6d62c6d2 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 22 Aug 2026 17:41:54 -0700
Subject: [PATCH 023/105] fix(router): abstain on unknown reasoning efforts
 instead of guessing

A reasoning model whose map entry names no effort flag now resolves to None, so
the API omits the field and the dashboard keeps its six-level fallback, and a
deployment counts as catalog-known only when the map supplied its mode, so an
operator writing model_info on an off-map deployment no longer empties the
levels its mapped siblings agree on.

Also drops the ultra level nothing asked for, forwards every level the public
literal names across the chat to Responses bridge, and removes the unreachable
supported_reasoning_efforts validator.
---
 .../transformation.py                         |   8 +-
 .../llms/openai/chat/gpt_5_transformation.py  |   2 +-
 litellm/main.py                               |   6 +-
 litellm/router.py                             |   5 +-
 .../reasoning_effort_capability.py            | 168 +++++++++++-------
 litellm/types/llms/openai.py                  |   2 +-
 litellm/types/router.py                       |  13 --
 litellm/types/utils.py                        |   1 -
 litellm/utils.py                              |   1 -
 ...responses_transformation_transformation.py |  12 +-
 .../llms/openai/test_gpt5_transformation.py   |  24 +--
 .../response_api_endpoints/test_endpoints.py  |   2 +-
 .../test_reasoning_effort_capability.py       | 167 +++++++++++------
 tests/test_litellm/test_router.py             |  55 +++---
 tests/test_litellm/test_utils.py              |   1 -
 .../add_model/TierModelEffortRows.test.tsx    |  78 ++++++++
 .../add_model/TierModelEffortRows.tsx         |  39 ++--
 .../llm_calls/fetch_models.test.tsx           |  35 +++-
 .../src/components/llm_calls/fetch_models.tsx |   3 +-
 19 files changed, 421 insertions(+), 201 deletions(-)
 create mode 100644 ui/litellm-dashboard/src/components/add_model/TierModelEffortRows.test.tsx

diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py
index d1228ac2610..17815976b4a 100644
--- a/litellm/completion_extras/litellm_responses_transformation/transformation.py
+++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py
@@ -5,7 +5,7 @@ Handler for transforming /chat/completions api requests to litellm.responses req
 import json
 import os
 from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
-from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, Union, cast
+from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, Union, cast, get_args
 
 from openai.types.responses.custom_tool_param import CustomToolParam
 from openai.types.responses.response_input_param import (
@@ -35,6 +35,7 @@ from litellm.responses.sse_output_recovery import (
 )
 from litellm.responses.utils import normalize_responses_api_stream_options
 from litellm.types.llms.openai import (
+    REASONING_EFFORT,
     ChatCompletionAnnotation,
     ChatCompletionReasoningItem,
     ChatCompletionToolCallChunk,
@@ -1113,10 +1114,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             litellm.reasoning_auto_summary or os.getenv("LITELLM_REASONING_AUTO_SUMMARY", "false").lower() == "true"
         )
 
-        # Level-agnostic: providers own effort validation, so an unknown level (max, ultra, future
-        # ones) passes through instead of being silently dropped here. "default" means no override,
-        # so it maps to nothing rather than reaching a provider as a literal effort value.
-        if reasoning_effort and reasoning_effort != "default":
+        if reasoning_effort in get_args(REASONING_EFFORT):
             return (
                 Reasoning(effort=reasoning_effort, summary="detailed")
                 if auto_summary_enabled
diff --git a/litellm/llms/openai/chat/gpt_5_transformation.py b/litellm/llms/openai/chat/gpt_5_transformation.py
index 3a79e6570e9..3a65e4a9426 100644
--- a/litellm/llms/openai/chat/gpt_5_transformation.py
+++ b/litellm/llms/openai/chat/gpt_5_transformation.py
@@ -16,7 +16,7 @@ def _normalize_reasoning_effort_for_chat_completion(
 ) -> str | None:
     """Convert reasoning_effort to the string format expected by OpenAI chat completion API.
 
-    The chat completion API expects a simple effort string ('none' through 'ultra').
+    The chat completion API expects an effort string such as 'low' or 'high'.
     Config/deployments may pass the Responses API format: {'effort': 'high', 'summary': 'detailed'}.
     """
     if value is None:
diff --git a/litellm/main.py b/litellm/main.py
index a1b41315b85..c2dbf595bf6 100644
--- a/litellm/main.py
+++ b/litellm/main.py
@@ -416,8 +416,7 @@ async def acompletion(
     logprobs: bool | None = None,
     top_logprobs: int | None = None,
     deployment_id=None,
-    reasoning_effort: Literal["none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra", "default"]
-    | None = None,
+    reasoning_effort: Literal["none", "minimal", "low", "medium", "high", "xhigh", "max", "default"] | None = None,
     verbosity: Literal["low", "medium", "high"] | None = None,
     safety_identifier: str | None = None,
     service_tier: str | None = None,
@@ -4921,8 +4920,7 @@ def completion(
     logit_bias: dict | None = None,
     user: str | None = None,
     # openai v1.0+ new params
-    reasoning_effort: Literal["none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra", "default"]
-    | None = None,
+    reasoning_effort: Literal["none", "minimal", "low", "medium", "high", "xhigh", "max", "default"] | None = None,
     verbosity: Literal["low", "medium", "high"] | None = None,
     response_format: dict | type[BaseModel] | None = None,
     seed: int | None = None,
diff --git a/litellm/router.py b/litellm/router.py
index bd5f97b27c4..766dcd67986 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -168,6 +168,7 @@ from litellm.router_utils.pre_call_checks.prompt_caching_deployment_check import
     PromptCachingDeploymentCheck,
 )
 from litellm.router_utils.reasoning_effort_capability import (
+    deployment_is_catalog_mapped,
     intersect_supported_reasoning_efforts,
     resolve_supported_reasoning_efforts,
 )
@@ -9443,6 +9444,8 @@ class Router:
             except Exception:
                 model_info = None
 
+            deployment_is_mapped = deployment_is_catalog_mapped(model_info, model_info_dict)
+
             # get llm provider
             litellm_model, llm_provider = "", ""
             try:
@@ -9565,7 +9568,7 @@ class Router:
 
             model_group_info.supported_reasoning_efforts = intersect_supported_reasoning_efforts(
                 model_group_info.supported_reasoning_efforts,
-                resolve_supported_reasoning_efforts(model_info),
+                resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=deployment_is_mapped),
             )
 
             if _deployment_tpm is not None:
diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py
index d445d611cd6..d20b1151803 100644
--- a/litellm/router_utils/reasoning_effort_capability.py
+++ b/litellm/router_utils/reasoning_effort_capability.py
@@ -1,91 +1,131 @@
 """Resolve which reasoning_effort values a deployment, and by intersection a model group, accepts.
 
-The model-map flags carry different polarity per level, mirroring the provider gates
-(gpt_5_transformation.py restricts xhigh to explicit opt-in and treats minimal/low as opt-out):
-medium and high are unconditional for any reasoning model, minimal/low are supported unless the map
-explicitly says false, and xhigh/max require an explicit true. Shipping the resolved list keeps that
-polarity in one place instead of re-encoding it in every consumer.
+The model map's supports_*_reasoning_effort flags are the only signal, and each level's polarity
+mirrors how a request path reads that same flag. medium and high are unconditional for a reasoning
+model. minimal and low are opt-out: openai/chat/gpt_5_transformation.py refuses them only when the
+map says false. xhigh and max are opt-in. none is opt-out everywhere except the azure gpt-5 family,
+whose config raises UnsupportedParamsError without an explicit true.
 
-Only openai and azure gate xhigh and max on the request path. anthropic/chat/transformation.py
-gates them on the output_config path alone, so its reasoning_effort path maps every level to a
-thinking budget whatever the map says, and a claude group whose entry omits
-supports_xhigh_reasoning_effort stops offering a level litellm would have forwarded. That is the
-deliberate trade: an explicit flag is the only signal that the tier is a real one rather than
-litellm quietly rounding the level to a budget, and what gets dropped is advisory metadata rather
-than a restriction on the request path.
+xhigh is gated on the request path by the openai and azure gpt-5 configs. max is not gated there at
+all: every entry carrying supports_max_reasoning_effort is Claude-family, and
+anthropic/chat/transformation.py gates max on the output_config path while its reasoning_effort
+path maps any level to a thinking budget. Making max opt-in is a deliberate trade, then, since an
+explicit flag is the only signal that the tier is a real one rather than litellm rounding the level
+to a budget, and a missing flag costs advisory metadata rather than a rejected request.
 
-The none level is the one flag whose polarity is provider-dependent. OpenAI never refuses it on the
-request path (azure/chat/gpt_5_transformation.py is the only caller that does, and it raises
-UnsupportedParamsError unless supports_none_reasoning_effort is explicitly true), so none is opt-out
-everywhere except azure, where it is opt-in. Resolving it the other way for azure would advertise a
-level litellm itself rejects, which is the failure this module exists to prevent.
+A deployment the map describes with no effort flags at all resolves to None rather than to the
+opt-out defaults. 689 of the map's 854 reasoning entries carry no flag, and the o-series, xai and
+bedrock nova entries among them take neither none nor minimal, so composing a set out of the
+defaults alone would advertise levels those providers reject.
+
+The advertisement order is the REASONING_EFFORT declaration order, which is presentation only. It
+is not a strength scale and does not reconcile with bedrock's output_config ceiling order in
+llms/bedrock/common_utils.py, which ranks max below xhigh while the thinking-budget constants rank
+it above.
 """
 
 from collections.abc import Mapping, Sequence
-from typing import Final
+from typing import Final, get_args
 
-REASONING_EFFORT_CAPABILITY_ORDER: Final = ("none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra")
+import litellm
+from litellm.types.llms.openai import REASONING_EFFORT
 
-_OPT_OUT_FLAGS: Final = (
+REASONING_EFFORT_ADVERTISEMENT_ORDER: Final = get_args(REASONING_EFFORT)
+
+_EFFORT_FLAGS: Final = (
+    ("none", "supports_none_reasoning_effort"),
     ("minimal", "supports_minimal_reasoning_effort"),
     ("low", "supports_low_reasoning_effort"),
-)
-_OPT_IN_FLAGS: Final = (
     ("xhigh", "supports_xhigh_reasoning_effort"),
     ("max", "supports_max_reasoning_effort"),
-    ("ultra", "supports_ultra_reasoning_effort"),
 )
+_OPT_OUT_EFFORTS: Final = ("minimal", "low")
+_OPT_IN_EFFORTS: Final = ("xhigh", "max")
 _UNCONDITIONAL_EFFORTS: Final = frozenset(("medium", "high"))
-_NONE_FLAG: Final = "supports_none_reasoning_effort"
-_NONE_OPT_IN_PROVIDERS: Final = frozenset(("azure",))
 
 
-def _supports_none_reasoning_effort(model_info: Mapping[str, object]) -> bool:
-    """Opt-in on azure, whose gpt-5 config raises on reasoning_effort='none' without an explicit
-    true; opt-out elsewhere, where no request path refuses the level. A missing azure flag defers to
-    _supports_factory, the same resolver the azure gate calls, so its bare-model-name fallback
-    (azure/gpt-5.2 inheriting the flag from gpt-5.2) reaches both sides alike."""
-    flag: Final = model_info.get(_NONE_FLAG)
-    if model_info.get("litellm_provider") not in _NONE_OPT_IN_PROVIDERS:
+def _bare_model_entry(model_info: Mapping[str, object]) -> Mapping[str, object]:
+    """The unprefixed twin of a provider-prefixed map entry, which is where the flags often live:
+    azure/gpt-5-mini carries none of them while gpt-5-mini carries all three. The request-path
+    gates resolve through the same twin (_supports_factory, #20885), so reading it here is what
+    keeps the advertisement and the gate on the same answer."""
+    key: Final = model_info.get("key")
+    provider: Final = model_info.get("litellm_provider")
+    if not isinstance(key, str) or not isinstance(provider, str) or not key.startswith(f"{provider}/"):
+        return {}
+    entry: Final[Mapping[str, object] | None] = litellm.model_cost.get(key.removeprefix(f"{provider}/"))
+    return entry if entry is not None else {}
+
+
+def _declared_effort_flags(model_info: Mapping[str, object]) -> Mapping[str, object]:
+    bare: Final = _bare_model_entry(model_info)
+    return {
+        effort: model_info.get(flag) if model_info.get(flag) is not None else bare.get(flag)
+        for effort, flag in _EFFORT_FLAGS
+    }
+
+
+def _supports_none_reasoning_effort(model_info: Mapping[str, object], flag: object) -> bool:
+    """Opt-in only where a request path refuses the level. AzureOpenAIGPT5Config raises
+    UnsupportedParamsError on reasoning_effort='none' without an explicit true, and it is selected
+    only for the gpt-5 family, so every other azure deployment keeps the opt-out default."""
+    if model_info.get("litellm_provider") != "azure":
         return flag is not False
-    if flag is not None:
-        return flag is True
-    model_key: Final = model_info.get("key")
-    if not isinstance(model_key, str):
+
+    from litellm.llms.azure.chat.gpt_5_transformation import AzureOpenAIGPT5Config
+
+    key: Final = model_info.get("key")
+    if not isinstance(key, str) or not AzureOpenAIGPT5Config.is_model_gpt_5_model(key):
+        return flag is not False
+    return flag is True
+
+
+def deployment_is_catalog_mapped(
+    resolved_model_info: Mapping[str, object] | None,
+    operator_model_info: Mapping[str, object],
+) -> bool:
+    """Whether the model map described this deployment, as opposed to the operator describing it.
+
+    Every deployment is registered in the cost map under its own id, so a mode the operator wrote
+    on an off-map deployment reads back here exactly like one the catalog supplied. Excluding it is
+    what stops such a deployment from claiming to be a known non-reasoning model and emptying the
+    levels its mapped siblings agree on.
+    """
+    if resolved_model_info is None or resolved_model_info.get("mode") is None:
         return False
-    from litellm.utils import (
-        _supports_factory,  # pyright: ignore[reportPrivateUsage]  # the resolver the azure gate itself calls; a public wrapper would fork the fallback
-    )
-
-    return _supports_factory(model=model_key, custom_llm_provider=None, key=_NONE_FLAG)
+    return operator_model_info.get("mode") is None
 
 
-def resolve_supported_reasoning_efforts(model_info: Mapping[str, object]) -> tuple[str, ...] | None:
-    """None = nothing is known about this deployment, so it must not narrow its group; () = this is a
-    known model that accepts no effort level, which correctly empties the group. Keeping the two
-    apart matters because the router registers a deployment absent from the model map under a
-    synthesized entry, and get_model_info then answers with supports_reasoning None exactly as it
-    does for a mapped non-reasoning model. Mode is what separates them: every map entry for a
-    routable model declares one, so an unset flag with no mode is read as unknown and one custom
-    model in a group no longer wipes the levels its mapped siblings agree on.
+def resolve_supported_reasoning_efforts(
+    model_info: Mapping[str, object],
+    *,
+    deployment_is_mapped: bool,
+) -> tuple[str, ...] | None:
+    """None = nothing is known about this deployment, so it must not narrow its group; () = a known
+    model that accepts no effort level, which correctly empties the group.
 
-    The separation is only as good as that signal. An off-map deployment carrying any model_info of
-    its own is registered under its deployment id with mode defaulting to chat, so it reads as a
-    known non-reasoning model and does still empty its group, with supports_reasoning on that
-    deployment as the way out. Telling the two apart for real needs provenance that the flattened
-    ModelInfo does not carry."""
-    if "supports_reasoning" not in model_info:
-        return None
+    Telling those apart needs provenance the flattened ModelInfo does not carry. A deployment the
+    map does not describe arrives with supports_reasoning None, exactly like a mapped non-reasoning
+    model: 2273 of the map's 3165 entries omit the key rather than setting it false, so reading an
+    unset flag as () would let one custom deployment empty every level its mapped siblings agree
+    on. deployment_is_mapped is that provenance, and an operator who wants either answer for an
+    off-map deployment gets it by setting supports_reasoning explicitly.
+    """
     supports_reasoning: Final = model_info.get("supports_reasoning")
-    if supports_reasoning is None and model_info.get("mode") is None:
-        return None
     if supports_reasoning is not True:
-        return ()
-    opt_out: Final = frozenset(effort for effort, flag in _OPT_OUT_FLAGS if model_info.get(flag) is not False)
-    opt_in: Final = frozenset(effort for effort, flag in _OPT_IN_FLAGS if model_info.get(flag) is True)
-    none_level: Final = frozenset(("none",)) if _supports_none_reasoning_effort(model_info) else frozenset()
+        return () if supports_reasoning is False or deployment_is_mapped else None
+
+    flags: Final = _declared_effort_flags(model_info)
+    if all(value is None for value in flags.values()):
+        return None
+
+    opt_out: Final = frozenset(effort for effort in _OPT_OUT_EFFORTS if flags[effort] is not False)
+    opt_in: Final = frozenset(effort for effort in _OPT_IN_EFFORTS if flags[effort] is True)
+    none_level: Final = (
+        frozenset(("none",)) if _supports_none_reasoning_effort(model_info, flags["none"]) else frozenset()
+    )
     allowed: Final = opt_out | _UNCONDITIONAL_EFFORTS | opt_in | none_level
-    return tuple(effort for effort in REASONING_EFFORT_CAPABILITY_ORDER if effort in allowed)
+    return tuple(effort for effort in REASONING_EFFORT_ADVERTISEMENT_ORDER if effort in allowed)
 
 
 def intersect_supported_reasoning_efforts(
@@ -99,4 +139,4 @@ def intersect_supported_reasoning_efforts(
     if current is None:
         return tuple(resolved)
     keep: Final = frozenset(current) & frozenset(resolved)
-    return tuple(effort for effort in REASONING_EFFORT_CAPABILITY_ORDER if effort in keep)
+    return tuple(effort for effort in REASONING_EFFORT_ADVERTISEMENT_ORDER if effort in keep)
diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py
index e3d28a0097f..4a6c4a5bbb5 100644
--- a/litellm/types/llms/openai.py
+++ b/litellm/types/llms/openai.py
@@ -1840,7 +1840,7 @@ ResponsesAPIStreamingResponse = Annotated[
 ]
 
 
-REASONING_EFFORT = Literal["none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra"]
+REASONING_EFFORT = Literal["none", "minimal", "low", "medium", "high", "xhigh", "max"]
 
 
 class OpenAIRealtimeStreamSession(TypedDict, total=False):
diff --git a/litellm/types/router.py b/litellm/types/router.py
index fd32b70405b..d4c735387a5 100644
--- a/litellm/types/router.py
+++ b/litellm/types/router.py
@@ -641,19 +641,6 @@ class ModelGroupInfo(BaseModel):
     supported_openai_params: list[str] | None = Field(default=[])
     configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None
 
-    @field_validator("supported_reasoning_efforts", mode="before")
-    @classmethod
-    def _accept_only_well_formed_reasoning_efforts(cls, value: object) -> tuple[str, ...] | None:
-        """Deployment model_info reaches this model through a **kwargs splat, so an operator can put
-        any shape under this key. Anything that is not a level or a list of levels resolves to None
-        and lets the computed intersection stand, because raising here fails the whole
-        /model_group/info response rather than the one group that carries the bad value."""
-        if isinstance(value, str):
-            return (value,)
-        if isinstance(value, (list, tuple)) and all(isinstance(level, str) for level in value):
-            return tuple(value)
-        return None
-
     def __init__(self, **data) -> None:
         for field_name, field_type in get_type_hints(self.__class__).items():
             if field_type is bool and data.get(field_name) is None:
diff --git a/litellm/types/utils.py b/litellm/types/utils.py
index 6a5e4c24dce..4d59650f410 100644
--- a/litellm/types/utils.py
+++ b/litellm/types/utils.py
@@ -164,7 +164,6 @@ class ProviderSpecificModelInfo(TypedDict, total=False):
     supports_low_reasoning_effort: bool | None
     supports_xhigh_reasoning_effort: bool | None
     supports_max_reasoning_effort: bool | None
-    supports_ultra_reasoning_effort: bool | None  # writable-ok: Pydantic warns on ReadOnly TypedDict fields
     supports_output_config: bool | None
     supports_image_size: bool | None
     bedrock_output_config_effort_ceiling: Literal["low", "medium", "high", "max", "xhigh"] | None
diff --git a/litellm/utils.py b/litellm/utils.py
index a74cf23a4ae..5b2ef93edb3 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -5764,7 +5764,6 @@ def _get_model_info_helper(
                 supports_low_reasoning_effort=_model_info.get("supports_low_reasoning_effort", None),
                 supports_xhigh_reasoning_effort=_model_info.get("supports_xhigh_reasoning_effort", None),
                 supports_max_reasoning_effort=_model_info.get("supports_max_reasoning_effort", None),
-                supports_ultra_reasoning_effort=_model_info.get("supports_ultra_reasoning_effort", None),
                 bedrock_output_config_effort_ceiling=_model_info.get("bedrock_output_config_effort_ceiling", None),
                 bedrock_converse_supports_strict_tools=_model_info.get("bedrock_converse_supports_strict_tools", None),
                 supports_computer_use=_model_info.get("supports_computer_use", None),
diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py
index 5c76fe5e7da..b358e6fa28c 100644
--- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py
+++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py
@@ -1585,16 +1585,16 @@ def test_map_reasoning_effort_adds_summary_detailed(monkeypatch):
         assert result_dict["summary"] == "custom_summary"
         print("✓ Dict input is passed through without modification")
 
-        # Test 5: levels this bridge does not enumerate (max, ultra, future ones) pass through so the
-        # provider can judge them, instead of being silently dropped before the request is built
+        # Test 5: every REASONING_EFFORT level reaches the provider, and anything else (a typo, an
+        # unshipped level, "default") is dropped so the request still succeeds at the provider default
         from litellm.types.llms.openai import Reasoning
 
-        for effort in ("max", "ultra", "unknown_value"):
+        for effort in ("max", "xhigh", "none"):
             result_passthrough = handler._map_reasoning_effort(effort)
             assert result_passthrough == Reasoning(effort=effort)
-        assert handler._map_reasoning_effort("") is None
-        assert handler._map_reasoning_effort("default") is None
-        print("✓ Unenumerated reasoning_effort levels pass through to the provider")
+        for dropped in ("ultra", "hgih", "unknown_value", "", "default"):
+            assert handler._map_reasoning_effort(dropped) is None
+        print("✓ Enumerated levels pass through and unknown ones are dropped")
 
         print(
             "✓ All reasoning_effort behaviors work correctly with flag/env var control"
diff --git a/tests/test_litellm/llms/openai/test_gpt5_transformation.py b/tests/test_litellm/llms/openai/test_gpt5_transformation.py
index eaae3d8c2c5..a35b75a6106 100644
--- a/tests/test_litellm/llms/openai/test_gpt5_transformation.py
+++ b/tests/test_litellm/llms/openai/test_gpt5_transformation.py
@@ -1334,7 +1334,7 @@ def test_gpt5_6_never_advertises_reasoning_effort_max(model: str):
     gpt-5.6 entry asserts supports_max_reasoning_effort and the advertised set stops at xhigh."""
     from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts
 
-    resolved = resolve_supported_reasoning_efforts(litellm.get_model_info(model))
+    resolved = resolve_supported_reasoning_efforts(litellm.get_model_info(model), deployment_is_mapped=True)
     assert resolved is not None
     assert "max" not in resolved
     assert "xhigh" in resolved
@@ -1355,28 +1355,16 @@ def test_gpt5_6_keeps_reasoning_effort_max_on_the_responses_api(
     assert params["reasoning"] == {"effort": "max"}
 
 
-def test_gpt5_6_never_advertises_reasoning_effort_ultra():
-    """ultra is plumbed as an opt-in level but no map entry asserts it: OpenAI's model guidance
-    documents effort values only up to max, and /v1/responses answers ultra with a 400. A verified
-    supports_ultra_reasoning_effort flag lights it up with no code change."""
-    from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts
-
-    resolved = resolve_supported_reasoning_efforts(litellm.get_model_info("gpt-5.6"))
-    assert resolved is not None
-    assert "ultra" not in resolved
-
-
-@pytest.mark.parametrize("effort", ["max", "ultra"])
-def test_gpt5_forwards_levels_the_chat_gate_does_not_own(config: OpenAIConfig, effort: str):
-    """Only xhigh is gated on this surface. max and ultra reach the provider (or the responses
-    bridge) and are answered there, which is what happened before per-group capabilities existed."""
+def test_gpt5_forwards_levels_the_chat_gate_does_not_own(config: OpenAIConfig):
+    """Only xhigh is gated on this surface. max reaches the provider (or the responses bridge) and
+    is answered there, which is what happened before per-group capabilities existed."""
     params = config.map_openai_params(
-        non_default_params={"reasoning_effort": effort},
+        non_default_params={"reasoning_effort": "max"},
         optional_params={},
         model="gpt-5.1",
         drop_params=False,
     )
-    assert params["reasoning_effort"] == effort
+    assert params["reasoning_effort"] == "max"
 
 
 def test_gpt5_rejects_xhigh_for_models_without_the_flag(config: OpenAIConfig):
diff --git a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py
index a906b0e638f..791d64c6428 100644
--- a/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py
+++ b/tests/test_litellm/proxy/response_api_endpoints/test_endpoints.py
@@ -1352,7 +1352,7 @@ class TestParseCursorModelVariant:
             ("gemini-3.0-pro-thinking-low", "gemini-3.0-pro", "low"),
             ("claude-opus-5-fast", "claude-opus-5", None),
             ("gpt-5.6-sol", "gpt-5.6-sol", None),
-            ("gpt-5.6-thinking-ultra-fast", "gpt-5.6", "ultra"),
+            ("foo-thinking-ultra-fast", "foo-thinking-ultra", None),
             ("gpt-5.6-thinking-max", "gpt-5.6", "max"),
             ("foo-thinking-mega-fast", "foo-thinking-mega", None),
             ("-thinking-high", "-thinking-high", None),
diff --git a/tests/test_litellm/router_utils/test_reasoning_effort_capability.py b/tests/test_litellm/router_utils/test_reasoning_effort_capability.py
index f3d71d19c47..6cd5b956b05 100644
--- a/tests/test_litellm/router_utils/test_reasoning_effort_capability.py
+++ b/tests/test_litellm/router_utils/test_reasoning_effort_capability.py
@@ -1,38 +1,51 @@
 import pytest
 
 from litellm.router_utils.reasoning_effort_capability import (
+    deployment_is_catalog_mapped,
     intersect_supported_reasoning_efforts,
     resolve_supported_reasoning_efforts,
 )
 
 
+class TestDeploymentIsCatalogMapped:
+    def test_a_mode_the_catalog_supplied_marks_the_deployment_mapped(self):
+        assert deployment_is_catalog_mapped({"mode": "chat"}, {}) is True
+
+    def test_a_deployment_the_catalog_never_described_is_not_mapped(self):
+        assert deployment_is_catalog_mapped(None, {}) is False
+        assert deployment_is_catalog_mapped({"max_input_tokens": 200000}, {}) is False
+
+    def test_a_mode_the_operator_wrote_does_not_make_the_deployment_mapped(self):
+        # Every deployment is registered in the cost map under its own id, so an operator-written
+        # mode reads back identically to one the catalog supplied and would otherwise let an
+        # off-map deployment empty the levels its mapped siblings agree on.
+        assert deployment_is_catalog_mapped({"mode": "chat"}, {"mode": "chat", "id": "abc"}) is False
+
+
+class TestProvenanceSeparatesUnknownFromNonReasoning:
+    def test_an_off_map_deployment_resolves_to_unknown(self):
+        # get_model_info answers supports_reasoning None both for a deployment the map never
+        # described and for a mapped non-reasoning model, so reading an unset flag as () would let
+        # one custom deployment empty every level its mapped siblings agree on.
+        assert resolve_supported_reasoning_efforts({}, deployment_is_mapped=False) is None
+        assert resolve_supported_reasoning_efforts({"supports_reasoning": None}, deployment_is_mapped=False) is None
+
+    def test_a_mapped_deployment_the_map_calls_non_reasoning_supports_no_efforts(self):
+        assert resolve_supported_reasoning_efforts({}, deployment_is_mapped=True) == ()
+        assert resolve_supported_reasoning_efforts({"supports_reasoning": None}, deployment_is_mapped=True) == ()
+
+    def test_an_explicit_false_supports_no_efforts_off_the_map_too(self):
+        # The operator's own escape hatch: saying so on an off-map deployment must still empty the
+        # group, since nothing else can tell the resolver that model takes no effort level.
+        assert resolve_supported_reasoning_efforts({"supports_reasoning": False}, deployment_is_mapped=False) == ()
+
+
 class TestResolveSupportedReasoningEfforts:
-    def test_no_metadata_resolves_to_unknown(self):
-        assert resolve_supported_reasoning_efforts({}) is None
-
-    def test_non_reasoning_model_supports_no_efforts(self):
-        assert resolve_supported_reasoning_efforts({"supports_reasoning": False}) == ()
-        assert resolve_supported_reasoning_efforts({"mode": "chat", "supports_reasoning": None}) == ()
-
-    def test_unset_flag_on_an_entry_with_no_mode_resolves_to_unknown(self):
-        # The router registers a deployment absent from the model map under a synthesized entry, and
-        # get_model_info then answers with supports_reasoning None just as it does for a mapped
-        # non-reasoning model. Only the missing mode tells them apart, and reading the synthesized
-        # one as () would let one custom model empty every level its mapped siblings agree on.
-        assert resolve_supported_reasoning_efforts({"supports_reasoning": None}) is None
-        assert resolve_supported_reasoning_efforts({"mode": None, "supports_reasoning": None}) is None
-
-    def test_reasoning_model_with_no_flags_gets_the_opt_out_levels_only(self):
-        # The kimi shape: supports_reasoning true, zero effort flags. medium/high are unconditional,
-        # none/minimal/low are opt-out so absence means supported, xhigh/max are opt-in so absence
-        # means unsupported.
-        assert resolve_supported_reasoning_efforts({"supports_reasoning": True}) == (
-            "none",
-            "minimal",
-            "low",
-            "medium",
-            "high",
-        )
+    def test_a_reasoning_model_with_no_flags_at_all_resolves_to_unknown(self):
+        # 689 of the map's 854 reasoning entries carry no effort flag, and the o-series, xai and
+        # bedrock nova entries among them accept neither none nor minimal, so composing a set out of
+        # the opt-out defaults alone would advertise levels those providers reject.
+        assert resolve_supported_reasoning_efforts({"supports_reasoning": True}, deployment_is_mapped=True) is None
 
     def test_explicit_false_removes_an_opt_out_level(self):
         # The gpt-5.5-pro shape from the model map: only medium/high/xhigh are accepted upstream.
@@ -43,7 +56,8 @@ class TestResolveSupportedReasoningEfforts:
                 "supports_minimal_reasoning_effort": False,
                 "supports_low_reasoning_effort": False,
                 "supports_xhigh_reasoning_effort": True,
-            }
+            },
+            deployment_is_mapped=True,
         )
         assert resolved == ("medium", "high", "xhigh")
 
@@ -54,47 +68,100 @@ class TestResolveSupportedReasoningEfforts:
                 "supports_reasoning": True,
                 "supports_xhigh_reasoning_effort": True,
                 "supports_max_reasoning_effort": True,
-            }
+            },
+            deployment_is_mapped=True,
         )
         assert resolved == ("none", "minimal", "low", "medium", "high", "xhigh", "max")
 
-    def test_ultra_is_opt_in(self):
-        without_flag = resolve_supported_reasoning_efforts({"supports_reasoning": True})
-        with_flag = resolve_supported_reasoning_efforts(
-            {"supports_reasoning": True, "supports_ultra_reasoning_effort": True}
-        )
-        assert without_flag is not None and "ultra" not in without_flag
-        assert with_flag is not None and with_flag[-1] == "ultra"
-
     def test_opt_in_flag_set_false_stays_excluded(self):
         resolved = resolve_supported_reasoning_efforts(
-            {"supports_reasoning": True, "supports_xhigh_reasoning_effort": False}
+            {
+                "supports_reasoning": True,
+                "supports_minimal_reasoning_effort": True,
+                "supports_xhigh_reasoning_effort": False,
+            },
+            deployment_is_mapped=True,
         )
-        assert resolved is not None
-        assert "xhigh" not in resolved
+        assert resolved == ("none", "minimal", "low", "medium", "high")
+
+
+class TestBareModelNameFallback:
+    def test_a_prefixed_entry_inherits_the_flags_of_its_unprefixed_twin(self):
+        """azure/gpt-5-mini carries no effort flag while gpt-5-mini carries three, and the request
+        path resolves capability flags through that same twin (#20885). Reading only the prefixed
+        entry would answer unknown for a model the map fully describes."""
+        from litellm.utils import _get_model_info_helper
+
+        model_info = dict(_get_model_info_helper(model="gpt-5-mini", custom_llm_provider="azure"))
+
+        assert resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=True) == (
+            "minimal",
+            "low",
+            "medium",
+            "high",
+        )
+
+    def test_the_prefixed_entry_wins_over_its_twin_per_flag(self):
+        resolved = resolve_supported_reasoning_efforts(
+            {
+                "supports_reasoning": True,
+                "litellm_provider": "azure",
+                "key": "azure/gpt-5-mini",
+                "supports_xhigh_reasoning_effort": True,
+            },
+            deployment_is_mapped=True,
+        )
+        assert resolved == ("minimal", "low", "medium", "high", "xhigh")
 
 
 class TestNoneLevelPolarity:
     def test_none_stays_opt_out_off_azure(self):
         resolved = resolve_supported_reasoning_efforts(
-            {"supports_reasoning": True, "litellm_provider": "openai", "key": "gpt-5-mini"}
+            {
+                "supports_reasoning": True,
+                "litellm_provider": "openai",
+                "key": "openai/some-reasoner",
+                "supports_max_reasoning_effort": True,
+            },
+            deployment_is_mapped=True,
         )
         assert resolved is not None and "none" in resolved
 
-    def test_azure_without_the_flag_does_not_advertise_none(self):
-        resolved = resolve_supported_reasoning_efforts(
-            {"supports_reasoning": True, "litellm_provider": "azure", "key": "azure/unmapped-deployment"}
-        )
-        assert resolved == ("minimal", "low", "medium", "high")
-
-    def test_azure_with_the_flag_advertises_none(self):
+    def test_none_stays_opt_out_on_an_azure_model_outside_the_gpt_5_family(self):
+        """AzureOpenAIGPT5Config is selected by is_model_gpt_5_model, so an azure o-series or
+        anthropic deployment never reaches the gate that refuses none and must keep the level."""
         resolved = resolve_supported_reasoning_efforts(
             {
                 "supports_reasoning": True,
                 "litellm_provider": "azure",
+                "key": "azure/o3",
+                "supports_max_reasoning_effort": True,
+            },
+            deployment_is_mapped=True,
+        )
+        assert resolved is not None and "none" in resolved
+
+    def test_azure_gpt_5_without_the_flag_does_not_advertise_none(self):
+        resolved = resolve_supported_reasoning_efforts(
+            {
+                "supports_reasoning": True,
+                "litellm_provider": "azure",
+                "key": "azure/gpt-5-turbo",
+                "supports_minimal_reasoning_effort": True,
+            },
+            deployment_is_mapped=True,
+        )
+        assert resolved == ("minimal", "low", "medium", "high")
+
+    def test_azure_gpt_5_with_the_flag_advertises_none(self):
+        resolved = resolve_supported_reasoning_efforts(
+            {
+                "supports_reasoning": True,
+                "litellm_provider": "azure",
+                "key": "azure/gpt-5-turbo",
                 "supports_none_reasoning_effort": True,
-                "key": "azure/unmapped-deployment",
-            }
+            },
+            deployment_is_mapped=True,
         )
         assert resolved is not None and "none" in resolved
 
@@ -109,7 +176,7 @@ class TestNoneLevelPolarity:
         from litellm.utils import _get_model_info_helper
 
         model_info = dict(_get_model_info_helper(model=model_key.split("/", 1)[1], custom_llm_provider="azure"))
-        resolved = resolve_supported_reasoning_efforts(model_info)
+        resolved = resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=True)
 
         assert resolved is not None
         gate_accepts_none = AzureOpenAIGPT5Config._supports_reasoning_effort_level(model_key, "none")
diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py
index b8a5a454613..93b04cc2330 100644
--- a/tests/test_litellm/test_router.py
+++ b/tests/test_litellm/test_router.py
@@ -8994,6 +8994,7 @@ def test_model_group_info_reasoning_efforts_empty_on_a_mapped_non_reasoning_depl
                 "litellm_provider": "anthropic",
                 "mode": "chat",
                 "supports_reasoning": True,
+                "supports_max_reasoning_effort": True,
             }
         return {"key": model_name, "litellm_provider": "openai", "mode": "chat", "supports_reasoning": None}
 
@@ -9032,6 +9033,7 @@ def test_model_group_info_reasoning_efforts_ignore_a_value_declared_in_model_inf
             "litellm_provider": "openai",
             "mode": "chat",
             "supports_reasoning": True,
+            "supports_none_reasoning_effort": True,
         }
         if model_id == "first-deployment":
             info["supported_reasoning_efforts"] = ("high",)
@@ -9047,26 +9049,39 @@ def test_model_group_info_reasoning_efforts_ignore_a_value_declared_in_model_inf
     assert result.supported_reasoning_efforts == ("none", "minimal", "low", "medium", "high")
 
 
-@pytest.mark.parametrize(
-    "configured, expected",
-    [
-        ("high", ("high",)),
-        (["low", "high"], ("low", "high")),
-        (17, None),
-        ({"effort": "high"}, None),
-        ([1, 2], None),
-    ],
-)
-def test_model_group_info_tolerates_any_configured_reasoning_efforts_shape(configured, expected):
-    """Deployment model_info is splatted into ModelGroupInfo, so this key arrives with whatever an
-    operator wrote in the config. A shape pydantic cannot validate used to fail the whole
-    /model_group/info response, not just the group carrying it."""
-    from litellm.types.router import ModelGroupInfo
+def test_model_group_info_reasoning_efforts_ignore_a_mode_the_operator_declared():
+    """A deployment is registered in the cost map under its own id with whatever model_info the
+    operator wrote, so a mode they set themselves reads back exactly like one the map supplied. Only
+    a mode the map supplied marks the deployment as known, or an off-map deployment carrying any
+    mode empties the group it sits in."""
+    from litellm.router_utils.reasoning_effort_capability import resolve_supported_reasoning_efforts
 
-    info = ModelGroupInfo(
-        model_group="g",
-        providers=["openai"],
-        supported_reasoning_efforts=configured,
+    mapped_model = "openai/gpt-5.6-sol"
+    expected = resolve_supported_reasoning_efforts(
+        litellm.get_model_info(model=mapped_model),
+        deployment_is_mapped=True,
+    )
+    assert expected
+
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "smart-group",
+                "litellm_params": {"model": mapped_model, "api_key": "sk-fake"},
+                "model_info": {"id": "mapped-deployment"},
+            },
+            {
+                "model_name": "smart-group",
+                "litellm_params": {"model": "openai/a-model-the-map-never-heard-of", "api_key": "sk-fake"},
+                "model_info": {"id": "off-map-deployment", "mode": "chat"},
+            },
+        ]
     )
 
-    assert info.supported_reasoning_efforts == expected
+    result = router._set_model_group_info(
+        model_group="smart-group",
+        user_facing_model_group_name="smart-group",
+    )
+
+    assert result is not None
+    assert result.supported_reasoning_efforts == expected
diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py
index af69d117e04..27cf9067914 100644
--- a/tests/test_litellm/test_utils.py
+++ b/tests/test_litellm/test_utils.py
@@ -997,7 +997,6 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
                 "supports_none_reasoning_effort": {"type": "boolean"},
                 "supports_xhigh_reasoning_effort": {"type": "boolean"},
                 "supports_max_reasoning_effort": {"type": "boolean"},
-                "supports_ultra_reasoning_effort": {"type": "boolean"},
                 "supports_adaptive_thinking": {"type": "boolean"},
                 "supports_legacy_thinking": {"type": "boolean"},
                 "thinking_always_on": {"type": "boolean"},
diff --git a/ui/litellm-dashboard/src/components/add_model/TierModelEffortRows.test.tsx b/ui/litellm-dashboard/src/components/add_model/TierModelEffortRows.test.tsx
new file mode 100644
index 00000000000..218a18b805e
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/add_model/TierModelEffortRows.test.tsx
@@ -0,0 +1,78 @@
+import { describe, expect, it } from "vitest";
+import { tierEffortRows } from "./TierModelEffortRows";
+
+describe("tierEffortRows", () => {
+  it("offers the levels the proxy reports for the model", () => {
+    const rows = tierEffortRows({
+      models: ["gpt-5-mini"],
+      effortOptionsByModel: { "gpt-5-mini": ["low", "medium", "high"] },
+      paramsByModel: undefined,
+    });
+
+    expect(rows).toEqual([{ model: "gpt-5-mini", effort: undefined, options: ["low", "medium", "high"] }]);
+  });
+
+  it("keeps a row whose model reports no level but already has an effort stored, so it can be cleared", () => {
+    const rows = tierEffortRows({
+      models: ["off-map-model"],
+      effortOptionsByModel: {},
+      paramsByModel: { "off-map-model": { reasoning_effort: "high" } },
+    });
+
+    expect(rows).toEqual([{ model: "off-map-model", effort: "high", options: ["high"] }]);
+  });
+
+  it("drops a row whose model reports no level and has nothing stored", () => {
+    const rows = tierEffortRows({
+      models: ["plain-chat-model"],
+      effortOptionsByModel: { "plain-chat-model": [] },
+      paramsByModel: { "plain-chat-model": { temperature: 0.5 } },
+    });
+
+    expect(rows).toEqual([]);
+  });
+
+  it("lists a stored level the model no longer reports without duplicating the ones it does", () => {
+    const rows = tierEffortRows({
+      models: ["gpt-5-mini"],
+      effortOptionsByModel: { "gpt-5-mini": ["low", "high"] },
+      paramsByModel: { "gpt-5-mini": { reasoning_effort: "xhigh" } },
+    });
+
+    expect(rows).toEqual([{ model: "gpt-5-mini", effort: "xhigh", options: ["low", "high", "xhigh"] }]);
+  });
+
+  it("does not repeat a stored level the model already reports", () => {
+    const rows = tierEffortRows({
+      models: ["gpt-5-mini"],
+      effortOptionsByModel: { "gpt-5-mini": ["low", "high"] },
+      paramsByModel: { "gpt-5-mini": { reasoning_effort: "high" } },
+    });
+
+    expect(rows).toEqual([{ model: "gpt-5-mini", effort: "high", options: ["low", "high"] }]);
+  });
+
+  it.each([
+    ["an unset key", {}],
+    ["an explicit null", { reasoning_effort: null }],
+    ["an empty string", { reasoning_effort: "" }],
+  ])("reads %s as no stored effort", (_label, params) => {
+    const rows = tierEffortRows({
+      models: ["gpt-5-mini"],
+      effortOptionsByModel: { "gpt-5-mini": ["low"] },
+      paramsByModel: { "gpt-5-mini": params },
+    });
+
+    expect(rows[0].effort).toBeUndefined();
+  });
+
+  it("renders a non-string stored value as a string so the select can show and clear it", () => {
+    const rows = tierEffortRows({
+      models: ["gpt-5-mini"],
+      effortOptionsByModel: { "gpt-5-mini": ["low"] },
+      paramsByModel: { "gpt-5-mini": { reasoning_effort: 3 } },
+    });
+
+    expect(rows).toEqual([{ model: "gpt-5-mini", effort: "3", options: ["low", "3"] }]);
+  });
+});
diff --git a/ui/litellm-dashboard/src/components/add_model/TierModelEffortRows.tsx b/ui/litellm-dashboard/src/components/add_model/TierModelEffortRows.tsx
index 0c93b3228dc..ec9705b9451 100644
--- a/ui/litellm-dashboard/src/components/add_model/TierModelEffortRows.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/TierModelEffortRows.tsx
@@ -8,7 +8,8 @@ const PROVIDER_DEFAULT = "__provider_default__";
 
 const storedEffort = (params: TierModelParams | undefined): ReasoningEffort | undefined => {
   const stored = params?.reasoning_effort;
-  return typeof stored === "string" && stored ? stored : undefined;
+  if (stored === undefined || stored === null || stored === "") return undefined;
+  return typeof stored === "string" ? stored : String(stored);
 };
 
 interface TierModelEffortRowsProps {
@@ -19,6 +20,31 @@ interface TierModelEffortRowsProps {
   onEffortChange: (model: string, effort: ReasoningEffort | undefined) => void;
 }
 
+export interface TierEffortRow {
+  model: string;
+  effort: ReasoningEffort | undefined;
+  options: string[];
+}
+
+/**
+ * A stored effort outside the model's supported set (hand-authored, or capabilities changed since
+ * it was saved) is listed anyway, so the row renders with its value selected and can be cleared.
+ * Only a model with no supported level and nothing stored drops out.
+ */
+export const tierEffortRows = ({
+  models,
+  effortOptionsByModel,
+  paramsByModel,
+}: Pick<TierModelEffortRowsProps, "models" | "effortOptionsByModel" | "paramsByModel">): TierEffortRow[] =>
+  models
+    .map((model) => {
+      const effort = storedEffort(paramsByModel?.[model]);
+      const supported = effortOptionsByModel[model] ?? [];
+      const listed = effort !== undefined && !supported.includes(effort) ? [...supported, effort] : supported;
+      return { model, effort, options: Array.from(new Set(listed)) };
+    })
+    .filter(({ options }) => options.length > 0);
+
 const TierModelEffortRows: React.FC<TierModelEffortRowsProps> = ({
   tierLabel,
   models,
@@ -26,16 +52,7 @@ const TierModelEffortRows: React.FC<TierModelEffortRowsProps> = ({
   paramsByModel,
   onEffortChange,
 }) => {
-  const rows = models
-    .map((model) => {
-      const effort = storedEffort(paramsByModel?.[model]);
-      const supported = effortOptionsByModel[model] ?? [];
-      // A stored effort outside the supported set (hand-authored, or capabilities changed since it
-      // was saved) stays listed so it renders and can be cleared.
-      const options = effort !== undefined && !supported.includes(effort) ? [...supported, effort] : supported;
-      return { model, effort, options };
-    })
-    .filter(({ model, options }) => options.length > 0 || Object.keys(paramsByModel?.[model] ?? {}).length > 0);
+  const rows = tierEffortRows({ models, effortOptionsByModel, paramsByModel });
   if (rows.length === 0) return null;
   return (
     <div className="mt-2 space-y-1">
diff --git a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.test.tsx b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.test.tsx
index bd691c7f629..998924ce36d 100644
--- a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.test.tsx
+++ b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.test.tsx
@@ -1,6 +1,6 @@
 import { beforeEach, describe, expect, it, vi } from "vitest";
-import { modelAvailableCall } from "@/components/networking";
-import { fetchAvailableModelsForTeam } from "./fetch_models";
+import { modelAvailableCall, modelHubCall } from "@/components/networking";
+import { fetchAvailableModels, fetchAvailableModelsForTeam } from "./fetch_models";
 
 vi.mock("@/components/networking", () => ({
   modelAvailableCall: vi.fn(),
@@ -8,6 +8,7 @@ vi.mock("@/components/networking", () => ({
 }));
 
 const modelAvailableCallMock = vi.mocked(modelAvailableCall);
+const modelHubCallMock = vi.mocked(modelHubCall);
 
 describe("fetchAvailableModelsForTeam", () => {
   beforeEach(() => {
@@ -31,3 +32,33 @@ describe("fetchAvailableModelsForTeam", () => {
     expect(await fetchAvailableModelsForTeam("token", "team-123")).toEqual([]);
   });
 });
+
+describe("fetchAvailableModels", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("carries the reasoning capabilities the model hub reports for each group", async () => {
+    modelHubCallMock.mockResolvedValue({
+      data: [
+        { model_group: "smart", mode: "chat", supports_reasoning: true, supported_reasoning_efforts: ["low", "high"] },
+        { model_group: "plain", mode: "chat", supports_reasoning: false },
+      ],
+    });
+
+    expect(await fetchAvailableModels("token")).toEqual([
+      { model_group: "plain", mode: "chat" },
+      { model_group: "smart", mode: "chat", supports_reasoning: true, supported_reasoning_efforts: ["low", "high"] },
+    ]);
+  });
+
+  it.each([
+    ["an error payload in place of the list", { data: { error: "no access" } }],
+    ["a missing data key", {}],
+    ["no body at all", undefined],
+  ])("returns an empty list on %s rather than throwing", async (_label, response) => {
+    modelHubCallMock.mockResolvedValue(response);
+
+    expect(await fetchAvailableModels("token")).toEqual([]);
+  });
+});
diff --git a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx
index 96b87887d43..1f812cf0377 100644
--- a/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx
+++ b/ui/litellm-dashboard/src/components/llm_calls/fetch_models.tsx
@@ -44,7 +44,8 @@ export const fetchAvailableModelsForTeam = async (accessToken: string, teamId: s
 export const fetchAvailableModels = async (accessToken: string): Promise<ModelGroup[]> => {
   try {
     const fetchedModels = await modelHubCall(accessToken);
-    const models: ModelGroup[] = (fetchedModels?.data ?? [])
+    const fetchedData: unknown = fetchedModels?.data;
+    const models: ModelGroup[] = (Array.isArray(fetchedData) ? fetchedData : [])
       .map(toModelGroup)
       .filter((model: ModelGroup) => model.model_group !== "")
       .sort((a: ModelGroup, b: ModelGroup) => a.model_group.localeCompare(b.model_group));

From b71b574af410f436954f9e6fcb28b44eff6c1d34 Mon Sep 17 00:00:00 2001
From: Tin Chi Lo <tin@berri.ai>
Date: Mon, 24 Aug 2026 15:12:33 -0400
Subject: [PATCH 024/105] ci: raise three unit shard job timeouts to satisfy
 the startup safety gate

---
 .github/workflows/test-unit.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml
index a7c67f2b35d..2dfca3d308f 100644
--- a/.github/workflows/test-unit.yml
+++ b/.github/workflows/test-unit.yml
@@ -211,7 +211,7 @@ jobs:
             workers: 2
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: proxy-extras
             artifact-name: proxy-extras
@@ -219,7 +219,7 @@ jobs:
             workers: 2
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: enterprise-package
             artifact-name: enterprise-package
@@ -227,7 +227,7 @@ jobs:
             workers: 4
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 55
+            job-timeout-minutes: 60
 
           - shard: responses-caching-types
             artifact-name: responses-caching-types

From 9dabd72f2d7f13c148a6e3129a0a676030670a3a Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 12:14:17 +0000
Subject: [PATCH 025/105] refactor(repositories): type prisma table access with
 one generic protocol

Every repository handed its `.table` back untyped, so a dozen modules had
each grown a private `_PrismaTableActions` Protocol to paper over it. They
had drifted: some declared `update` as returning the row, others the row or
None, and none agreed on whether `find_many` was covariant

Replace all of them with a single `TableActions[RowT_co]` in
`litellm/repositories/prisma_protocols.py`, keyed to the prisma row each
repository is bound to. Query inputs stay `Mapping[str, object]` so callers
keep passing plain dicts, and `find_many` returns `Sequence` so the row type
stays covariant

Typing the nullable returns honestly surfaced paths that were already
crashing. A team admin could never edit or delete a memory entry owned by
their team: the write-auth check fed a raw prisma row to a helper that
expects the domain model, so `members_with_roles` arrived as plain dicts and
the request died as a 500 instead of applying the edit. Non-admin members hit
the same 500 in place of the 403 they were owed, so refusal and breakage were
indistinguishable. `/v2/model/info?user_models_only=true` dereferenced a
missing user row rather than returning the 400 the route already had, three
team routes dereferenced a team deleted between the read and the write, and
the agent registry dereferenced a missing agent instead of naming it

basedpyright drops 2,132 errors, 1,454 of them reportAny and 73
reportExplicitAny. The dashboard's generated types pick up `string[]` where
they had `unknown[]` for a team's members, admins and models
---
 basedpyright-code-budget.json                 |  32 +-
 .../proxy/common_utils/check_batch_cost.py    |   4 +-
 litellm/integrations/prometheus.py            |   5 +-
 litellm/models/team.py                        |   6 +-
 litellm/proxy/_experimental/mcp_server/db.py  |  99 ++---
 .../proxy/agent_endpoints/agent_registry.py   |  84 +++--
 .../claude_code_marketplace.py                |   4 +-
 litellm/proxy/auth/auth_checks.py             |  18 +-
 litellm/proxy/auth/user_api_key_auth.py       |  17 +-
 .../proxy/common_utils/config_sync_pubsub.py  |   8 +-
 .../expired_ui_session_key_cleanup_manager.py |  12 +-
 .../common_utils/key_rotation_manager.py      |  12 +-
 .../proxy/common_utils/reset_budget_job.py    |   6 +-
 .../proxy/container_endpoints/ownership.py    |  37 +-
 .../proxy/credential_endpoints/endpoints.py   |  13 +-
 litellm/proxy/db/tool_registry_writer.py      |  30 +-
 .../proxy/guardrails/guardrail_endpoints.py   |  22 +-
 .../proxy/guardrails/guardrail_registry.py    |  24 +-
 litellm/proxy/guardrails/usage_endpoints.py   |  38 +-
 litellm/proxy/guardrails/usage_tracking.py    |  16 +-
 .../access_group_endpoints.py                 |   4 +-
 .../budget_management_endpoints.py            |   2 +-
 .../cache_settings_endpoints.py               |   3 +-
 .../common_daily_activity.py                  |   8 +-
 .../config_override_endpoints.py              |   3 +-
 .../internal_user_endpoints.py                | 121 +++----
 .../jwt_key_mapping_endpoints.py              |   3 +
 .../key_management_endpoints.py               | 199 +++++------
 ...model_access_group_management_endpoints.py |  21 +-
 .../model_management_endpoints.py             |  88 +++--
 .../organization_endpoints.py                 |  43 ++-
 .../scim/scim_transformations.py              |   3 +-
 .../management_endpoints/scim/scim_v2.py      |   6 +
 .../tag_management_endpoints.py               |   4 +-
 .../team_callback_endpoints.py                |   3 +
 .../management_endpoints/team_endpoints.py    | 337 +++++++++---------
 litellm/proxy/management_endpoints/ui_sso.py  |  63 +---
 .../object_permission_utils.py                |  14 +-
 litellm/proxy/management_helpers/utils.py     |  26 +-
 litellm/proxy/memory/memory_endpoints.py      |  85 ++---
 .../openai_files_endpoints/common_utils.py    |  24 +-
 .../managed_id_rewriter.py                    |  16 +-
 .../pass_through_endpoints.py                 |  11 +-
 .../proxy/policy_engine/policy_registry.py    |  68 ++--
 .../policy_engine/policy_resolve_endpoints.py |  56 +--
 litellm/proxy/prompts/prompt_endpoints.py     |   8 +-
 litellm/proxy/proxy_server.py                 | 107 +++---
 .../spend_tracking/cloudzero_endpoints.py     |  32 +-
 .../spend_management_endpoints.py             |  55 +--
 .../proxy/spend_tracking/vantage_endpoints.py |  30 +-
 .../proxy_setting_endpoints.py                |  58 ++-
 litellm/proxy/utils.py                        |  70 ++--
 .../proxy/vector_store_endpoints/endpoints.py |  13 +-
 .../management_endpoints.py                   |  24 +-
 litellm/repositories/base_repository.py       |  31 +-
 litellm/repositories/budget_repository.py     |   8 +-
 litellm/repositories/config_repository.py     |   2 +-
 .../repositories/credentials_repository.py    |  53 ++-
 litellm/repositories/model_repository.py      |  42 +--
 .../object_permission_repository.py           |   8 +-
 .../repositories/organization_repository.py   |   8 +-
 litellm/repositories/prisma_protocols.py      |  87 +++++
 litellm/repositories/project_repository.py    |   8 +-
 litellm/repositories/table_repositories.py    | 116 +++---
 litellm/repositories/team_repository.py       |   8 +-
 .../repositories/user_banner_repository.py    |   7 +-
 litellm/repositories/user_repository.py       |   8 +-
 .../verification_token_repository.py          |  22 +-
 .../responses/file_search/emulated_handler.py |  56 +--
 .../custom_tools.py                           |  10 +-
 .../handler.py                                |   8 +-
 .../session_handler.py                        |   6 +-
 .../streaming_iterator.py                     |  23 +-
 .../transformation.py                         |  82 +++--
 litellm/responses/main.py                     |   4 +-
 .../responses/mcp/chat_completions_handler.py |  22 +-
 .../responses/mcp/mcp_streaming_iterator.py   |  14 +-
 litellm/responses/mcp/request_context.py      |  23 +-
 litellm/responses/sse_output_recovery.py      |  53 ++-
 litellm/responses/streaming_iterator.py       |  42 ++-
 litellm/responses/utils.py                    |  56 +--
 .../vector_stores/vector_store_registry.py    |  13 +-
 ruff-strict-budget.json                       |  12 +-
 .../proxy_unit_tests/test_jwt_key_mapping.py  |  28 ++
 .../agent_endpoints/test_agent_registry.py    |  57 +++
 .../test_claude_code_marketplace.py           |  24 ++
 .../common_utils/test_reset_budget_job.py     |   2 +-
 .../proxy/db/mcp_server/test_db.py            |  33 +-
 .../guardrails/test_guardrail_registry.py     |  22 ++
 .../scim/test_scim_v2_endpoints.py            |  43 +++
 .../test_internal_user_endpoints.py           |   2 +-
 .../test_key_management_endpoints.py          |  39 ++
 .../test_model_management_endpoints.py        |  55 +++
 .../test_organization_endpoints.py            |  26 ++
 .../test_team_callback_endpoints.py           |  38 ++
 .../test_team_endpoints.py                    |  96 ++++-
 .../proxy/memory/test_memory_endpoints.py     |  72 +++-
 .../prompts/test_prompt_endpoints_crud.py     |  55 +++
 tests/test_litellm/proxy/test_proxy_server.py |  33 ++
 .../test_vector_store_endpoints.py            |  51 +++
 type-discipline-budget.json                   |  12 +-
 ui/litellm-dashboard/src/lib/http/schema.d.ts |  30 +-
 102 files changed, 2320 insertions(+), 1325 deletions(-)

diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json
index 664e1669834..7e57539d1dd 100644
--- a/basedpyright-code-budget.json
+++ b/basedpyright-code-budget.json
@@ -1,18 +1,18 @@
 {
   "reportAny": {
-    "limit": 19955
+    "limit": 18501
   },
   "reportArgumentType": {
-    "limit": 2566
+    "limit": 2564
   },
   "reportAssignmentType": {
     "limit": 320
   },
   "reportAttributeAccessIssue": {
-    "limit": 488
+    "limit": 483
   },
   "reportCallIssue": {
-    "limit": 114
+    "limit": 113
   },
   "reportConstantRedefinition": {
     "limit": 40
@@ -24,7 +24,7 @@
     "limit": 19
   },
   "reportExplicitAny": {
-    "limit": 6049
+    "limit": 5976
   },
   "reportFunctionMemberAccess": {
     "limit": 7
@@ -45,7 +45,7 @@
     "limit": 35
   },
   "reportInvalidTypeForm": {
-    "limit": 35
+    "limit": 34
   },
   "reportInvalidTypeVarUse": {
     "limit": 2
@@ -54,10 +54,10 @@
     "limit": 0
   },
   "reportMissingParameterType": {
-    "limit": 5663
+    "limit": 5661
   },
   "reportMissingTypeArgument": {
-    "limit": 15555
+    "limit": 15504
   },
   "reportMissingTypeStubs": {
     "limit": 40
@@ -72,7 +72,7 @@
     "limit": 0
   },
   "reportOptionalMemberAccess": {
-    "limit": 1061
+    "limit": 1058
   },
   "reportOptionalOperand": {
     "limit": 0
@@ -90,7 +90,7 @@
     "limit": 8
   },
   "reportReturnType": {
-    "limit": 213
+    "limit": 212
   },
   "reportTypedDictNotRequiredAccess": {
     "limit": 26
@@ -99,31 +99,31 @@
     "limit": 0
   },
   "reportUnknownArgumentType": {
-    "limit": 44655
+    "limit": 44527
   },
   "reportUnknownLambdaType": {
     "limit": 109
   },
   "reportUnknownMemberType": {
-    "limit": 39011
+    "limit": 38827
   },
   "reportUnknownParameterType": {
-    "limit": 19885
+    "limit": 19848
   },
   "reportUnknownVariableType": {
-    "limit": 30569
+    "limit": 30384
   },
   "reportUnnecessaryCast": {
     "limit": 117
   },
   "reportUnnecessaryComparison": {
-    "limit": 699
+    "limit": 697
   },
   "reportUnnecessaryContains": {
     "limit": 5
   },
   "reportUnnecessaryIsInstance": {
-    "limit": 836
+    "limit": 833
   },
   "reportUntypedBaseClass": {
     "limit": 0
diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
index 76e92538aaa..aee3295d1da 100644
--- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
+++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
@@ -14,6 +14,8 @@ from litellm.constants import (
 )
 
 if TYPE_CHECKING:
+    from prisma import models as prisma_models
+
     from litellm.integrations.prometheus import PrometheusLogger
     from litellm.proxy._types import LiteLLM_ManagedObjectTable
     from litellm.proxy.utils import PrismaClient, ProxyLogging
@@ -351,7 +353,7 @@ class CheckBatchCost:
         return isinstance(error, (NotFoundError, openai.NotFoundError)) and output_file_id in str(error)
 
     async def _finalize_unbilled_terminal_job(
-        self, job: "LiteLLM_ManagedObjectTable", response: "LiteLLMBatch"
+        self, job: "prisma_models.LiteLLM_ManagedObjectTable", response: "LiteLLMBatch"
     ) -> None:
         """Persist a terminal batch that has nothing billable, converting any raw
         provider file ids to managed ids, and take it out of the poll page."""
diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py
index f9195db1d67..beda1a29075 100644
--- a/litellm/integrations/prometheus.py
+++ b/litellm/integrations/prometheus.py
@@ -96,7 +96,10 @@ class _PaginatedPrismaTable(Protocol[_TableRowT]):
 
 def _paginated_table(repository: BaseRepository[_TableRowT]) -> _PaginatedPrismaTable[_TableRowT]:
     """View a repository's prisma table through the pagination surface budget metrics need."""
-    return repository.table
+    return cast(
+        _PaginatedPrismaTable[_TableRowT],
+        repository.table,  # cast-ok: prisma rows carry the budget columns the domain model declares
+    )
 
 
 class _OrgBudgetRow(Protocol):
diff --git a/litellm/models/team.py b/litellm/models/team.py
index 544e2cf5bbc..da526515e6e 100644
--- a/litellm/models/team.py
+++ b/litellm/models/team.py
@@ -64,8 +64,8 @@ class TeamBase(LiteLLMPydanticObjectBase):
     team_alias: str | None = None
     team_id: str | None = None
     organization_id: str | None = None
-    admins: list = []
-    members: list = []
+    admins: list[str] = []
+    members: list[str] = []
     members_with_roles: list[Member] = []
     team_member_permissions: list[str] | None = None
     metadata: dict | None = None
@@ -75,7 +75,7 @@ class TeamBase(LiteLLMPydanticObjectBase):
     soft_budget: float | None = None
     budget_duration: str | None = None
     budget_limits: list[BudgetLimitEntry] | None = None
-    models: list = []
+    models: list[str] = []
     blocked: bool = False
     router_settings: dict | None = None
     access_group_ids: list[str] | None = None
diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py
index 28638ed9c77..4aa08020527 100644
--- a/litellm/proxy/_experimental/mcp_server/db.py
+++ b/litellm/proxy/_experimental/mcp_server/db.py
@@ -4,7 +4,7 @@ import hashlib
 import json
 from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
 from datetime import datetime, timedelta, timezone
-from typing import TYPE_CHECKING, Any, Final, Protocol, TypedDict, TypeVar, cast
+from typing import TYPE_CHECKING, Any, Final, Protocol, TypedDict, cast
 
 from litellm._logging import verbose_proxy_logger
 from litellm._uuid import uuid
@@ -13,7 +13,6 @@ from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
 from litellm.proxy._experimental.mcp_server.oauth_utils import build_upstream_oauth2_token_request
 from litellm.proxy._types import (
     LiteLLM_MCPServerTable,
-    LiteLLM_ObjectPermissionTable,
     MCPApprovalStatus,
     MCPEnvVar,
     MCPEnvVarScope,
@@ -30,6 +29,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import (
 )
 from litellm.proxy.utils import PrismaClient
 from litellm.repositories.object_permission_repository import ObjectPermissionRepository
+from litellm.repositories.prisma_protocols import TableActions
 from litellm.repositories.table_repositories import (
     MCPServerOAuthClientRepository,
     MCPServerRepository,
@@ -48,34 +48,9 @@ if TYPE_CHECKING:
 
     from litellm.types.mcp_server.mcp_server_manager import MCPServer
 
-_RowT = TypeVar("_RowT")
-
-
-class _TableActions(Protocol[_RowT]):
-    async def find_unique(
-        self, where: Mapping[str, object], include: Mapping[str, object] | None = None
-    ) -> _RowT | None: ...
-
-    async def find_many(
-        self,
-        take: int | None = None,
-        where: Mapping[str, object] | None = None,
-        order: Mapping[str, object] | None = None,
-    ) -> list[_RowT]: ...
-
-    async def create(self, data: Mapping[str, object]) -> _RowT: ...
-
-    async def upsert(self, where: Mapping[str, object], data: Mapping[str, object]) -> _RowT: ...
-
-    async def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> _RowT | None: ...
-
-    async def delete(self, where: Mapping[str, object]) -> _RowT | None: ...
-
-    async def delete_many(self, where: Mapping[str, object] | None = None) -> int: ...
-
 
 class _UserEnvVarsTransactionClient(Protocol):
-    litellm_mcpuserenvvars: "_TableActions[prisma_db_models.LiteLLM_MCPUserEnvVars]"
+    litellm_mcpuserenvvars: "TableActions[prisma_db_models.LiteLLM_MCPUserEnvVars]"
 
     async def execute_raw(self, query: str, *args: object) -> int: ...
 
@@ -473,15 +448,15 @@ def _credentials_blob_to_mutable_dict(blob: str | Mapping[str, object]) -> dict[
 
 def _mcp_server_table_actions(
     prisma_client: PrismaClient,
-) -> "_TableActions[prisma_db_models.LiteLLM_MCPServerTable]":
-    table: Final[_TableActions[prisma_db_models.LiteLLM_MCPServerTable]] = MCPServerRepository(prisma_client).table
+) -> "TableActions[prisma_db_models.LiteLLM_MCPServerTable]":
+    table: Final[TableActions[prisma_db_models.LiteLLM_MCPServerTable]] = MCPServerRepository(prisma_client).table
     return table
 
 
 def _verification_token_table_actions(
     prisma_client: PrismaClient,
-) -> "_TableActions[prisma_db_models.LiteLLM_VerificationToken]":
-    table: Final[_TableActions[prisma_db_models.LiteLLM_VerificationToken]] = VerificationTokenRepository(
+) -> "TableActions[prisma_db_models.LiteLLM_VerificationToken]":
+    table: Final[TableActions[prisma_db_models.LiteLLM_VerificationToken]] = VerificationTokenRepository(
         prisma_client
     ).table
     return table
@@ -489,15 +464,15 @@ def _verification_token_table_actions(
 
 def _team_table_actions(
     prisma_client: PrismaClient,
-) -> "_TableActions[prisma_db_models.LiteLLM_TeamTable]":
-    table: Final[_TableActions[prisma_db_models.LiteLLM_TeamTable]] = TeamRepository(prisma_client).table
+) -> "TableActions[prisma_db_models.LiteLLM_TeamTable]":
+    table: Final[TableActions[prisma_db_models.LiteLLM_TeamTable]] = TeamRepository(prisma_client).table
     return table
 
 
 def _oauth_client_table_actions(
     prisma_client: PrismaClient,
-) -> "_TableActions[prisma_db_models.LiteLLM_MCPServerOAuthClient]":
-    table: Final[_TableActions[prisma_db_models.LiteLLM_MCPServerOAuthClient]] = MCPServerOAuthClientRepository(
+) -> "TableActions[prisma_db_models.LiteLLM_MCPServerOAuthClient]":
+    table: Final[TableActions[prisma_db_models.LiteLLM_MCPServerOAuthClient]] = MCPServerOAuthClientRepository(
         prisma_client
     ).table
     return table
@@ -511,7 +486,7 @@ def _db_transaction_manager(prisma_client: PrismaClient) -> _UserEnvVarsTransact
 async def _db_find_mcp_server_rows(
     prisma_client: PrismaClient,
     where: "prisma_db_types.LiteLLM_MCPServerTableWhereInput | None" = None,
-) -> "list[prisma_db_models.LiteLLM_MCPServerTable]":
+) -> "Sequence[prisma_db_models.LiteLLM_MCPServerTable]":
     return await _mcp_server_table_actions(prisma_client).find_many(where=where)
 
 
@@ -526,17 +501,19 @@ async def _db_update_mcp_server_row(
     server_id: str,
     data: "prisma_db_types.LiteLLM_MCPServerTableUpdateInput",
 ) -> "prisma_db_models.LiteLLM_MCPServerTable":
-    row: Final[prisma_db_models.LiteLLM_MCPServerTable] = await MCPServerRepository(prisma_client).table.update(
+    row: Final[prisma_db_models.LiteLLM_MCPServerTable | None] = await _mcp_server_table_actions(prisma_client).update(
         where={"server_id": server_id},
         data=data,
     )
+    if row is None:
+        raise ValueError(f"MCP server not found, passed server_id={server_id}")
     return row
 
 
 def _user_credential_actions(
     prisma_client: PrismaClient,
-) -> "_TableActions[prisma_db_models.LiteLLM_MCPUserCredentials]":
-    table: Final[_TableActions[prisma_db_models.LiteLLM_MCPUserCredentials]] = MCPUserCredentialsRepository(
+) -> "TableActions[prisma_db_models.LiteLLM_MCPUserCredentials]":
+    table: Final[TableActions[prisma_db_models.LiteLLM_MCPUserCredentials]] = MCPUserCredentialsRepository(
         prisma_client
     ).table
     return table
@@ -544,8 +521,8 @@ def _user_credential_actions(
 
 def _user_env_var_actions(
     prisma_client: PrismaClient,
-) -> "_TableActions[prisma_db_models.LiteLLM_MCPUserEnvVars]":
-    table: Final[_TableActions[prisma_db_models.LiteLLM_MCPUserEnvVars]] = prisma_client.db.litellm_mcpuserenvvars
+) -> "TableActions[prisma_db_models.LiteLLM_MCPUserEnvVars]":
+    table: Final[TableActions[prisma_db_models.LiteLLM_MCPUserEnvVars]] = prisma_client.db.litellm_mcpuserenvvars
     return table
 
 
@@ -560,7 +537,7 @@ async def _db_find_user_credential_row(
 async def _db_find_user_credential_rows(
     prisma_client: PrismaClient,
     where: "prisma_db_types.LiteLLM_MCPUserCredentialsWhereInput | None" = None,
-) -> "list[prisma_db_models.LiteLLM_MCPUserCredentials]":
+) -> "Sequence[prisma_db_models.LiteLLM_MCPUserCredentials]":
     return await _user_credential_actions(prisma_client).find_many(where=where)
 
 
@@ -583,7 +560,7 @@ async def _db_upsert_user_credential_row(
 async def _db_find_user_env_var_rows(
     prisma_client: PrismaClient,
     where: "prisma_db_types.LiteLLM_MCPUserEnvVarsWhereInput | None" = None,
-) -> "list[prisma_db_models.LiteLLM_MCPUserEnvVars]":
+) -> "Sequence[prisma_db_models.LiteLLM_MCPUserEnvVars]":
     return await _user_env_var_actions(prisma_client).find_many(where=where)
 
 
@@ -658,7 +635,7 @@ async def get_mcp_servers(prisma_client: PrismaClient, server_ids: Iterable[str]
     """
     Returns the matching mcp servers from the db with the server_ids
     """
-    _mcp_servers: Final[list[prisma_db_models.LiteLLM_MCPServerTable]] = await _mcp_server_table_actions(
+    _mcp_servers: Final[Sequence[prisma_db_models.LiteLLM_MCPServerTable]] = await _mcp_server_table_actions(
         prisma_client
     ).find_many(
         where={
@@ -745,13 +722,13 @@ async def get_all_mcp_servers_for_user(
 
 async def get_objectpermissions_for_mcp_server(
     prisma_client: PrismaClient, mcp_server_id: str
-) -> list[LiteLLM_ObjectPermissionTable]:
+) -> "Sequence[prisma_db_models.LiteLLM_ObjectPermissionTable]":
     """
     Get all the object permissions records and the associated team and verficiationtoken records that have access to the mcp server
     """
-    object_permission_records: Final[list[LiteLLM_ObjectPermissionTable]] = await ObjectPermissionRepository(
-        prisma_client
-    ).table.find_many(
+    object_permission_records: Final[
+        Sequence[prisma_db_models.LiteLLM_ObjectPermissionTable]
+    ] = await ObjectPermissionRepository(prisma_client).table.find_many(
         where={
             "mcp_servers": {"has": mcp_server_id},
         },
@@ -766,19 +743,19 @@ async def get_objectpermissions_for_mcp_server(
 
 async def get_virtualkeys_for_mcp_server(
     prisma_client: PrismaClient, server_id: str
-) -> "list[prisma_db_models.LiteLLM_VerificationToken]":
+) -> "Sequence[prisma_db_models.LiteLLM_VerificationToken]":
     """
     Get all the virtual keys that have access to the mcp server
     """
-    virtual_keys: Final[list[prisma_db_models.LiteLLM_VerificationToken] | None] = await VerificationTokenRepository(
-        prisma_client
-    ).table.find_many(
+    virtual_keys: Final[
+        Sequence[prisma_db_models.LiteLLM_VerificationToken] | None
+    ] = await VerificationTokenRepository(prisma_client).table.find_many(
         where={
             "mcp_servers": {"has": server_id},
         },
     )
 
-    if virtual_keys is None:
+    if virtual_keys is None:  # pyright: ignore[reportUnnecessaryComparison]  # unreachable per seam types; kept as-is
         return []
     return virtual_keys
 
@@ -860,7 +837,7 @@ async def delete_mcp_server(
                 invalidate_token_cache = global_mcp_server_manager.invalidate_user_oauth_token_cache
             for user_id in credential_user_ids:
                 await invalidate_token_cache(user_id, server_id)
-    return deleted_server
+    return deleted_server  # pyright: ignore[reportReturnType]  # prisma row, not domain LiteLLM_MCPServerTable
 
 
 async def create_mcp_server(
@@ -880,7 +857,7 @@ async def create_mcp_server(
     data_dict["updated_by"] = touched_by
 
     new_mcp_server: Final[LiteLLM_MCPServerTable] = await MCPServerRepository(prisma_client).table.create(
-        data=data_dict
+        data=data_dict,  # pyright: ignore[reportAssignmentType]  # prisma row, not domain LiteLLM_MCPServerTable
     )
 
     _decrypt_env_vars_on_returned_row(new_mcp_server)
@@ -982,7 +959,7 @@ async def update_mcp_server(
     data: UpdateMCPServerRequest,
     touched_by: str,
     fields_set: set[str] | None = None,
-) -> LiteLLM_MCPServerTable:
+) -> LiteLLM_MCPServerTable | None:
     """
     Update a new mcp server record in the db
     """
@@ -1093,9 +1070,9 @@ async def update_mcp_server(
 
         data_dict["credentials"] = Json(None)
 
-    updated_mcp_server: Final[LiteLLM_MCPServerTable] = await MCPServerRepository(prisma_client).table.update(
+    updated_mcp_server: Final[LiteLLM_MCPServerTable | None] = await MCPServerRepository(prisma_client).table.update(
         where={"server_id": data.server_id},
-        data=data_dict,
+        data=data_dict,  # pyright: ignore[reportAssignmentType]  # prisma row, not domain LiteLLM_MCPServerTable
     )
 
     _decrypt_env_vars_on_returned_row(updated_mcp_server)
@@ -1181,7 +1158,7 @@ async def rotate_mcp_server_credentials_master_key(prisma_client: PrismaClient,
         )
         updated += 1
 
-    oauth_clients: Final[list[prisma_db_models.LiteLLM_MCPServerOAuthClient]] = await _oauth_client_table_actions(
+    oauth_clients: Final[Sequence[prisma_db_models.LiteLLM_MCPServerOAuthClient]] = await _oauth_client_table_actions(
         prisma_client
     ).find_many()
     oauth_updated = 0
@@ -1914,7 +1891,7 @@ async def get_mcp_submissions(
     along with a summary count breakdown by approval_status.
     Mirrors get_guardrail_submissions() from guardrail_endpoints.py.
     """
-    rows: Final[list[prisma_db_models.LiteLLM_MCPServerTable]] = await _mcp_server_table_actions(
+    rows: Final[Sequence[prisma_db_models.LiteLLM_MCPServerTable]] = await _mcp_server_table_actions(
         prisma_client
     ).find_many(
         where={"submitted_at": {"not": None}},
diff --git a/litellm/proxy/agent_endpoints/agent_registry.py b/litellm/proxy/agent_endpoints/agent_registry.py
index 64de6827679..fa33a307438 100644
--- a/litellm/proxy/agent_endpoints/agent_registry.py
+++ b/litellm/proxy/agent_endpoints/agent_registry.py
@@ -4,7 +4,7 @@ import json
 from collections.abc import Iterator, Mapping, Sequence
 from datetime import datetime, timezone
 from types import MappingProxyType
-from typing import Any, Final, NamedTuple, Protocol, TypedDict
+from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, TypedDict
 
 import litellm
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
@@ -12,9 +12,13 @@ from litellm.proxy.management_helpers.object_permission_utils import (
     handle_update_object_permission_common,
 )
 from litellm.proxy.utils import PrismaClient
+from litellm.repositories.prisma_protocols import TableActions
 from litellm.repositories.table_repositories import AgentsRepository, ObjectPermissionRepository
 from litellm.types.agents import AgentConfig, AgentResponse, PatchAgentRequest
 
+if TYPE_CHECKING:
+    from prisma import models as prisma_models
+
 
 class AgentObjectPermissionRecord(Protocol):
     def model_dump(self) -> dict[str, object]: ...
@@ -42,11 +46,20 @@ class AgentRecordDump(TypedDict):
 
 
 class AgentRecord(Protocol):
-    agent_id: str
-    agent_name: str
-    object_permission_id: str | None
-    object_permission: AgentObjectPermissionRecord | None
-    spend: float
+    @property
+    def agent_id(self) -> str: ...
+
+    @property
+    def agent_name(self) -> str: ...
+
+    @property
+    def object_permission_id(self) -> str | None: ...
+
+    @property
+    def object_permission(self) -> AgentObjectPermissionRecord | None: ...
+
+    @property
+    def spend(self) -> float: ...
 
     def model_dump(self) -> AgentRecordDump: ...
 
@@ -57,50 +70,47 @@ class AgentTableClient(Protocol):
     async def create(
         self,
         data: Mapping[str, object],
-        include: Mapping[str, bool] | None = None,
+        include: Mapping[str, object] | None = None,
     ) -> AgentRecord: ...
 
     async def find_unique(
         self,
         where: Mapping[str, object],
-        include: Mapping[str, bool] | None = None,
+        include: Mapping[str, object] | None = None,
     ) -> AgentRecord | None: ...
 
     async def find_many(
         self,
         where: Mapping[str, object] | None = None,
         order: Mapping[str, str] | None = None,
-        include: Mapping[str, bool] | None = None,
+        include: Mapping[str, object] | None = None,
     ) -> Sequence[AgentRecord]: ...
 
     async def update(
         self,
-        where: Mapping[str, object],
         data: Mapping[str, object],
-        include: Mapping[str, bool] | None = None,
-    ) -> AgentRecord: ...
+        where: Mapping[str, object],
+        include: Mapping[str, object] | None = None,
+    ) -> AgentRecord | None: ...
 
-    async def delete(self, where: Mapping[str, object]) -> AgentRecord: ...
+    async def delete(
+        self,
+        where: Mapping[str, object],
+        include: Mapping[str, object] | None = None,
+    ) -> AgentRecord | None: ...
 
 
 def agents_table(prisma_client: PrismaClient) -> AgentTableClient:
-    table: Final[AgentTableClient] = AgentsRepository(prisma_client).table
+    table: Final[AgentTableClient] = AgentsRepository(prisma_client).table  # pyright: ignore[reportAssignmentType]  # prisma rows type model_dump() as dict[str, Any]
     return table
 
 
-class ObjectPermissionGrantRecord(Protocol):
-    object_permission_id: str
-    agents: list[str] | None
-
-
-class ObjectPermissionTableClient(Protocol):
-    async def find_many(self, where: Mapping[str, object]) -> Sequence[ObjectPermissionGrantRecord]: ...
-
-    async def update_many(self, where: Mapping[str, object], data: Mapping[str, object]) -> int: ...
-
-
-def object_permission_table(prisma_client: PrismaClient) -> ObjectPermissionTableClient:
-    table: Final[ObjectPermissionTableClient] = ObjectPermissionRepository(prisma_client).table
+def object_permission_table(
+    prisma_client: PrismaClient,
+) -> "TableActions[prisma_models.LiteLLM_ObjectPermissionTable]":
+    table: Final[TableActions[prisma_models.LiteLLM_ObjectPermissionTable]] = ObjectPermissionRepository(
+        prisma_client
+    ).table
     return table
 
 
@@ -222,7 +232,9 @@ class AgentRegistry:
         self.load_agents_from_config(agent_config if agent_config is not None else self.config_agents)
         return self.agent_list
 
-    async def migrate_legacy_grant_ids(self, table: ObjectPermissionTableClient) -> GrantMigrationResult:
+    async def migrate_legacy_grant_ids(
+        self, table: "TableActions[prisma_models.LiteLLM_ObjectPermissionTable]"
+    ) -> GrantMigrationResult:
         """
         Rewrite object_permission.agents rows holding a legacy full-entry hash to the
         stable name-derived id.
@@ -360,6 +372,8 @@ class AgentRegistry:
         """
         try:
             deleted_agent: Final = await agents_table(prisma_client).delete(where={"agent_id": agent_id})
+            if deleted_agent is None:
+                raise ValueError(f"Agent not found, passed agent_id={agent_id}")
             return dict(deleted_agent)
         except Exception as e:
             raise Exception(f"Error deleting agent from DB: {e}")
@@ -386,12 +400,12 @@ class AgentRegistry:
             The patched agent
         """
         try:
-            existing_agent = await AgentsRepository(prisma_client).table.find_unique(where={"agent_id": agent_id})
-            if existing_agent is not None:
-                existing_agent = dict(existing_agent)
-
-            if existing_agent is None:
+            existing_row: Final = await AgentsRepository(prisma_client).table.find_unique(
+                where={"agent_id": agent_id}  # mutable-ok: prisma filters are plain dicts
+            )
+            if existing_row is None:
                 raise Exception(f"Agent with ID {agent_id} not found")
+            existing_agent: Final = dict(existing_row)
 
             augment_agent: Final = {**existing_agent, **agent}
             update_data: Final[dict[str, Any]] = {}
@@ -436,6 +450,8 @@ class AgentRegistry:
                 },
                 include={"object_permission": True},
             )
+            if patched_agent is None:
+                raise ValueError(f"Agent not found, passed agent_id={agent_id}")
             patched_agent_dict: Final = patched_agent.model_dump()
             if patched_agent.object_permission is not None:
                 try:
@@ -523,6 +539,8 @@ class AgentRegistry:
                 include={"object_permission": True},
             )
 
+            if updated_agent is None:
+                raise ValueError(f"Agent not found, passed agent_id={agent_id}")
             updated_agent_dict: Final = updated_agent.model_dump()
             if updated_agent.object_permission is not None:
                 try:
diff --git a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py
index 09ba2c93ea0..65bc46edfaf 100644
--- a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py
+++ b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py
@@ -543,7 +543,7 @@ async def update_plugin(
 
         manifest: Final[Mapping[str, object]] = _build_plugin_manifest(plugin_name, request)
 
-        plugin: Final[_PluginRecord] = await ClaudeCodePluginRepository(prisma_client).table.update(
+        plugin: Final[_PluginRecord | None] = await ClaudeCodePluginRepository(prisma_client).table.update(
             where={"name": plugin_name},  # mutable-ok: prisma query arguments must be plain dicts
             data={  # mutable-ok: prisma query arguments must be plain dicts
                 "version": request.version,
@@ -553,6 +553,8 @@ async def update_plugin(
                 "updated_at": datetime.now(timezone.utc),
             },
         )
+        if plugin is None:
+            raise _error_response(404, f"Plugin '{plugin_name}' not found")
 
         verbose_proxy_logger.info("Plugin %s updated successfully", plugin_name)
 
diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py
index e7b98b3cc7f..3e8537b070f 100644
--- a/litellm/proxy/auth/auth_checks.py
+++ b/litellm/proxy/auth/auth_checks.py
@@ -156,7 +156,12 @@ class _PrismaVectorStoreRow(Protocol):
 
 class _PrismaUserRow(Protocol):
     user_id: str
-    organization_memberships: Sequence[LiteLLM_OrganizationMembershipTable | None] | None
+
+    @property
+    def organization_memberships(self) -> Sequence[_PrismaModelDumpRow | None] | None: ...
+
+    @organization_memberships.setter
+    def organization_memberships(self, value: Sequence[_PrismaModelDumpRow] | None) -> None: ...
 
     def __iter__(self) -> Iterator[tuple[str, object]]: ...
 
@@ -214,9 +219,14 @@ def _user_table(repo: _PrismaTableHolder[_PrismaUserRow]) -> _PrismaAuthTable[_P
     return repo.table
 
 
+class _VectorStorePermissionsRow(Protocol):
+    @property
+    def vector_stores(self) -> Sequence[str] | None: ...
+
+
 def _object_permission_table(
-    repo: _PrismaTableHolder[LiteLLM_ObjectPermissionTable],
-) -> _PrismaAuthTable[LiteLLM_ObjectPermissionTable]:
+    repo: _PrismaTableHolder[_VectorStorePermissionsRow],
+) -> _PrismaAuthTable[_VectorStorePermissionsRow]:
     return repo.table
 
 
@@ -5277,7 +5287,7 @@ async def vector_store_access_check(
 def _can_object_call_vector_stores(
     object_type: Literal["key", "team", "org"],
     vector_store_ids_to_run: list[str],
-    object_permissions: LiteLLM_ObjectPermissionTable | None,
+    object_permissions: _VectorStorePermissionsRow | None,
 ):
     """
     Raises ProxyException if the object (key, team, org) cannot access the specific vector store.
diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py
index 658d176f6a7..035265d55f4 100644
--- a/litellm/proxy/auth/user_api_key_auth.py
+++ b/litellm/proxy/auth/user_api_key_auth.py
@@ -196,6 +196,15 @@ class _UserModelBudgetLimiter(Protocol):
     ) -> bool: ...
 
 
+class _TokenTeamModels(Protocol):
+    @property
+    def team_models(self) -> list[str]: ...
+
+
+def _token_team_models(valid_token: _TokenTeamModels) -> list[str]:
+    return valid_token.team_models
+
+
 async def _read_user_model_max_budget(
     user_id: str | None,
     prisma_client: PrismaClient | None,
@@ -1991,7 +2000,7 @@ async def _user_api_key_auth_builder(
                                 include={"litellm_budget_table": True},
                             )
                             if _db_member is not None:
-                                team_member_info = LiteLLM_TeamMembership(**_db_member.dict())
+                                team_member_info = LiteLLM_TeamMembership(**_db_member.model_dump())
                                 await user_api_key_cache.async_set_cache(
                                     key=_cache_key,
                                     value=team_member_info,
@@ -2143,6 +2152,7 @@ async def _user_api_key_auth_builder(
                             proxy_logging_obj=proxy_logging_obj,
                         )
                 except HTTPException:
+                    token_team_models: Final = _token_team_models(valid_token)
                     _team_obj = LiteLLM_TeamTableCachedObj(
                         team_id=valid_token.team_id,
                         max_budget=valid_token.team_max_budget,
@@ -2151,7 +2161,7 @@ async def _user_api_key_auth_builder(
                         tpm_limit=valid_token.team_tpm_limit,
                         rpm_limit=valid_token.team_rpm_limit,
                         blocked=valid_token.team_blocked,
-                        models=valid_token.team_models,
+                        models=token_team_models,
                         metadata=valid_token.team_metadata,
                         object_permission_id=valid_token.team_object_permission_id,
                         object_permission=await _resolve_object_permission_for_unresolvable_team(
@@ -2295,6 +2305,7 @@ def _team_obj_from_token(valid_token: UserAPIKeyAuth) -> LiteLLM_TeamTableCached
     UserAPIKeyAuth. Only called when valid_token.team_id is known to be
     non-None (the caller gates on it)."""
     assert valid_token.team_id is not None
+    token_team_models: Final = _token_team_models(valid_token)
     return LiteLLM_TeamTableCachedObj(
         team_id=valid_token.team_id,
         max_budget=valid_token.team_max_budget,
@@ -2303,7 +2314,7 @@ def _team_obj_from_token(valid_token: UserAPIKeyAuth) -> LiteLLM_TeamTableCached
         tpm_limit=valid_token.team_tpm_limit,
         rpm_limit=valid_token.team_rpm_limit,
         blocked=valid_token.team_blocked,
-        models=valid_token.team_models,
+        models=token_team_models,
         metadata=valid_token.team_metadata,
         object_permission_id=valid_token.team_object_permission_id,
     )
diff --git a/litellm/proxy/common_utils/config_sync_pubsub.py b/litellm/proxy/common_utils/config_sync_pubsub.py
index d5317fc0e02..6d781babe63 100644
--- a/litellm/proxy/common_utils/config_sync_pubsub.py
+++ b/litellm/proxy/common_utils/config_sync_pubsub.py
@@ -7,6 +7,7 @@ from dataclasses import asdict, dataclass
 from typing import TYPE_CHECKING, Final, Protocol, cast  # noqa: TID251  # untyped prisma/redis boundary needs cast
 
 from litellm._logging import verbose_proxy_logger
+from litellm.repositories.prisma_protocols import RowT_co, TableActions
 
 if TYPE_CHECKING:
     from litellm.caching.redis_cache import RedisCache
@@ -163,13 +164,14 @@ class _PublishOnWriteActions:
 
 
 def wrap_table_actions_for_config_sync(
-    actions: object,
+    actions: "TableActions[RowT_co]",
     table_name: str,
     publish: Callable[[str], Awaitable[None]] = publish_config_change_for_object_type,
-) -> object:
+) -> "TableActions[RowT_co]":
     if table_name not in _CONFIG_SYNCED_TABLE_NAMES:
         return actions
-    return _PublishOnWriteActions(actions=actions, object_type=table_name, publish=publish)
+    wrapped: Final = _PublishOnWriteActions(actions=actions, object_type=table_name, publish=publish)
+    return cast("TableActions[RowT_co]", wrapped)  # cast-ok: dynamic write-through proxy keeps the wrapped row type
 
 
 class ConfigSyncSubscriber:
diff --git a/litellm/proxy/common_utils/expired_ui_session_key_cleanup_manager.py b/litellm/proxy/common_utils/expired_ui_session_key_cleanup_manager.py
index e314aec497f..58183eec689 100644
--- a/litellm/proxy/common_utils/expired_ui_session_key_cleanup_manager.py
+++ b/litellm/proxy/common_utils/expired_ui_session_key_cleanup_manager.py
@@ -4,8 +4,9 @@ Expired UI session key cleanup manager.
 Deletes expired virtual keys created for LiteLLM dashboard sessions.
 """
 
+from collections.abc import Sequence
 from datetime import datetime, timezone
-from typing import Any, Final
+from typing import Any, Final, Protocol
 
 from litellm._logging import verbose_proxy_logger
 from litellm.constants import (
@@ -14,7 +15,7 @@ from litellm.constants import (
     LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME,
     UI_SESSION_TOKEN_TEAM_ID,
 )
-from litellm.proxy._types import KeyRequest, LiteLLM_VerificationToken, UserAPIKeyAuth
+from litellm.proxy._types import KeyRequest, UserAPIKeyAuth
 from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
 from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks
 from litellm.proxy.management_endpoints.key_management_endpoints import (
@@ -26,6 +27,11 @@ from litellm.repositories.verification_token_repository import (
 )
 
 
+class _ExpiredSessionKeyRow(Protocol):
+    @property
+    def token(self) -> str | None: ...
+
+
 class ExpiredUISessionKeyCleanupManager:
     """
     Cleans up expired UI session keys.
@@ -138,7 +144,7 @@ class ExpiredUISessionKeyCleanupManager:
 
         return len(tokens)
 
-    async def _find_expired_ui_session_keys(self) -> list[LiteLLM_VerificationToken]:
+    async def _find_expired_ui_session_keys(self) -> Sequence[_ExpiredSessionKeyRow]:
         """
         Find expired LiteLLM dashboard session keys.
         """
diff --git a/litellm/proxy/common_utils/key_rotation_manager.py b/litellm/proxy/common_utils/key_rotation_manager.py
index 839ff28c354..352d024e20e 100644
--- a/litellm/proxy/common_utils/key_rotation_manager.py
+++ b/litellm/proxy/common_utils/key_rotation_manager.py
@@ -4,8 +4,9 @@ Key Rotation Manager - Automated key rotation based on rotation schedules
 Handles finding keys that need rotation based on their individual schedules.
 """
 
+from collections.abc import Sequence
 from datetime import datetime, timezone
-from typing import Final
+from typing import TYPE_CHECKING, Final
 
 from litellm._logging import verbose_proxy_logger
 from litellm.constants import (
@@ -31,6 +32,9 @@ from litellm.repositories.verification_token_repository import (
     VerificationTokenRepository,
 )
 
+if TYPE_CHECKING:
+    from prisma import models as prisma_models
+
 
 class KeyRotationManager:
     """
@@ -106,7 +110,7 @@ class KeyRotationManager:
                     cronjob_id=KEY_ROTATION_JOB_NAME,
                 )
 
-    async def _find_keys_needing_rotation(self) -> list[LiteLLM_VerificationToken]:
+    async def _find_keys_needing_rotation(self) -> "Sequence[prisma_models.LiteLLM_VerificationToken]":
         """
         Find keys that are due for rotation based on their key_rotation_at timestamp.
 
@@ -156,7 +160,7 @@ class KeyRotationManager:
         # Check if the rotation time has passed
         return now >= key.key_rotation_at
 
-    async def _rotate_key(self, key: LiteLLM_VerificationToken):
+    async def _rotate_key(self, key: "prisma_models.LiteLLM_VerificationToken"):
         """
         Rotate a single key using existing regenerate_key_fn and call the rotation hook
         """
@@ -197,7 +201,7 @@ class KeyRotationManager:
         if isinstance(response, GenerateKeyResponse):
             await KeyManagementEventHooks.async_key_rotated_hook(
                 data=regenerate_request,
-                existing_key_row=key,
+                existing_key_row=key,  # pyright: ignore[reportArgumentType]  # prisma row, hook wants the domain model
                 response=response,
                 user_api_key_dict=system_user,
                 litellm_changed_by=LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME,
diff --git a/litellm/proxy/common_utils/reset_budget_job.py b/litellm/proxy/common_utils/reset_budget_job.py
index 8fcb184b26a..68999c93823 100644
--- a/litellm/proxy/common_utils/reset_budget_job.py
+++ b/litellm/proxy/common_utils/reset_budget_job.py
@@ -37,7 +37,7 @@ from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManage
 from litellm.proxy.db.exception_handler import call_with_db_reconnect_retry
 from litellm.proxy.utils import PrismaClient, ProxyLogging
 from litellm.repositories.organization_repository import OrganizationRepository
-from litellm.repositories.prisma_protocols import ReadOnlyTable, SpendLinkedTable
+from litellm.repositories.prisma_protocols import SpendLinkedTable
 from litellm.repositories.table_repositories import (
     EndUserRepository,
     TagRepository,
@@ -675,7 +675,7 @@ class ResetBudgetJob:
         rely on the default budget (litellm.max_end_user_budget_id) applied
         in-memory during auth checks.
         """
-        table: Final[ReadOnlyTable] = EndUserRepository(self.prisma_client).table
+        table: Final = EndUserRepository(self.prisma_client).table
         rows: Final = await self._with_db_retry(
             lambda: table.find_many(
                 where={
@@ -685,7 +685,7 @@ class ResetBudgetJob:
             ),
             reason="reset_budget_read_endusers_without_budget_id_failure",
         )
-        return [LiteLLM_EndUserTable.model_validate(row.dict()) for row in rows]
+        return [LiteLLM_EndUserTable.model_validate(row.model_dump()) for row in rows]
 
     async def _write_key_reset_updates(self, updated_keys: list[LiteLLM_VerificationToken]) -> None:
         """
diff --git a/litellm/proxy/container_endpoints/ownership.py b/litellm/proxy/container_endpoints/ownership.py
index a559ab49cfa..e3088771c82 100644
--- a/litellm/proxy/container_endpoints/ownership.py
+++ b/litellm/proxy/container_endpoints/ownership.py
@@ -1,7 +1,7 @@
 import json
 from collections.abc import Mapping, Sequence
 from collections.abc import Set as AbstractSet
-from typing import TYPE_CHECKING, Any, Final, Protocol
+from typing import TYPE_CHECKING, Any, Final
 
 from fastapi import HTTPException
 
@@ -18,28 +18,11 @@ from litellm.repositories.table_repositories import ManagedObjectRepository
 from litellm.responses.utils import ResponsesAPIRequestUtils
 
 if TYPE_CHECKING:
+    from prisma import models as prisma_models
+
     from litellm.proxy.utils import PrismaClient
 
 
-class _ManagedObjectRow(Protocol):
-    model_object_id: str
-    unified_object_id: str | None
-    file_purpose: str | None
-    created_by: str | None
-
-
-class _ManagedObjectTable(Protocol):
-    async def find_unique(self, *, where: Mapping[str, str]) -> _ManagedObjectRow | None: ...
-
-    async def find_first(self, *, where: Mapping[str, str]) -> _ManagedObjectRow | None: ...
-
-    async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_ManagedObjectRow]: ...
-
-    async def create(self, *, data: Mapping[str, str]) -> _ManagedObjectRow: ...
-
-    async def update(self, *, where: Mapping[str, str], data: Mapping[str, str]) -> _ManagedObjectRow | None: ...
-
-
 CONTAINER_OBJECT_PURPOSE: Final = "container"
 
 # 60s LRU/TTL cache absorbs every container access check before it reaches
@@ -220,7 +203,7 @@ async def record_container_owner(
         verbose_proxy_logger.warning("Skipping container ownership tracking because prisma_client is None")
         return response
 
-    table: Final[_ManagedObjectTable] = ManagedObjectRepository(prisma_client).table
+    table: Final = ManagedObjectRepository(prisma_client).table
     existing: Final = await table.find_unique(where={"model_object_id": model_object_id})
     if existing is not None:
         if getattr(existing, "file_purpose", None) != CONTAINER_OBJECT_PURPOSE:
@@ -272,8 +255,8 @@ async def _get_container_owner(original_container_id: str, custom_llm_provider:
     if prisma_client is None:
         return None
 
-    table: Final[_ManagedObjectTable] = ManagedObjectRepository(prisma_client).table
-    row: Final[_ManagedObjectRow | None] = await table.find_first(
+    table: Final = ManagedObjectRepository(prisma_client).table
+    row: Final[prisma_models.LiteLLM_ManagedObjectTable | None] = await table.find_first(
         where={
             "model_object_id": model_object_id,
             "file_purpose": CONTAINER_OBJECT_PURPOSE,
@@ -309,8 +292,8 @@ async def _get_stored_container_id(original_container_id: str, custom_llm_provid
     if prisma_client is None:
         return None
 
-    table: Final[_ManagedObjectTable] = ManagedObjectRepository(prisma_client).table
-    row: Final[_ManagedObjectRow | None] = await table.find_first(
+    table: Final = ManagedObjectRepository(prisma_client).table
+    row: Final[prisma_models.LiteLLM_ManagedObjectTable | None] = await table.find_first(
         where={
             "model_object_id": model_object_id,
             "file_purpose": CONTAINER_OBJECT_PURPOSE,
@@ -394,8 +377,8 @@ async def _get_allowed_container_ids(
     if prisma_client is None:
         return set()
 
-    table: Final[_ManagedObjectTable] = ManagedObjectRepository(prisma_client).table
-    rows: Final[Sequence[_ManagedObjectRow]] = await table.find_many(
+    table: Final = ManagedObjectRepository(prisma_client).table
+    rows: Final[Sequence[prisma_models.LiteLLM_ManagedObjectTable]] = await table.find_many(
         where={
             "file_purpose": CONTAINER_OBJECT_PURPOSE,
             "created_by": {"in": owner_scopes},
diff --git a/litellm/proxy/credential_endpoints/endpoints.py b/litellm/proxy/credential_endpoints/endpoints.py
index 3b3e9692eda..dc193cb8523 100644
--- a/litellm/proxy/credential_endpoints/endpoints.py
+++ b/litellm/proxy/credential_endpoints/endpoints.py
@@ -2,7 +2,10 @@
 CRUD endpoints for storing reusable credentials.
 """
 
-from typing import Final
+from typing import (
+    Final,
+    cast,  # noqa: TID251  # jsonify_object in proxy/utils.py is annotated with a bare dict
+)
 
 from fastapi import APIRouter, Depends, HTTPException, Path, Request, Response
 
@@ -88,7 +91,9 @@ async def create_credential(
         )
         encrypted_credential: Final = CredentialHelperUtils.encrypt_credential_values(processed_credential)
         credentials_dict: Final = encrypted_credential.model_dump()
-        credentials_dict_jsonified: Final = jsonify_object(credentials_dict)
+        credentials_dict_jsonified: Final = cast(  # cast-ok: deep-copies a model_dump, so keys are str
+            "dict[str, object]", jsonify_object(credentials_dict)
+        )
         await CredentialsRepository(prisma_client).create(
             data={
                 **credentials_dict_jsonified,
@@ -310,7 +315,9 @@ async def update_credential(
         if db_credential is None:
             raise HTTPException(status_code=404, detail="Credential not found in DB.")
         merged_credential: Final = update_db_credential(db_credential, credential)
-        credential_object_jsonified: Final = jsonify_object(merged_credential.model_dump())
+        credential_object_jsonified: Final = cast(  # cast-ok: deep-copies a model_dump, so keys are str
+            "dict[str, object]", jsonify_object(merged_credential.model_dump())
+        )
         await credentials_repository.update_by_name(
             credential_name,
             data={
diff --git a/litellm/proxy/db/tool_registry_writer.py b/litellm/proxy/db/tool_registry_writer.py
index 187a18be845..367552e783e 100644
--- a/litellm/proxy/db/tool_registry_writer.py
+++ b/litellm/proxy/db/tool_registry_writer.py
@@ -6,14 +6,15 @@ Admins use the management endpoints to read and update input_policy / output_pol
 """
 
 import uuid
-from collections.abc import Mapping, Sequence
+from collections.abc import Mapping
 from datetime import datetime, timezone
-from typing import TYPE_CHECKING, Any, Final, Protocol, TypeVar
+from typing import TYPE_CHECKING, Any, Final
 
 from litellm._logging import verbose_proxy_logger
 from litellm.proxy._types import ToolDiscoveryQueueItem
 from litellm.proxy.db.exception_handler import call_with_db_reconnect_retry
 from litellm.repositories.object_permission_repository import ObjectPermissionRepository
+from litellm.repositories.prisma_protocols import TableActions
 from litellm.repositories.table_repositories import ToolRepository
 from litellm.types.tool_management import (
     LiteLLM_ToolTableRow,
@@ -25,33 +26,16 @@ if TYPE_CHECKING:
 
     from litellm.proxy.utils import PrismaClient
 
-_RowT_co: Final = TypeVar("_RowT_co", covariant=True)
 
-
-class _TableActions(Protocol[_RowT_co]):
-    async def find_unique(self, where: Mapping[str, object]) -> _RowT_co | None: ...
-
-    async def find_many(
-        self,
-        where: Mapping[str, object] | None = None,
-        order: Mapping[str, object] | None = None,
-        include: Mapping[str, object] | None = None,
-    ) -> Sequence[_RowT_co]: ...
-
-    async def upsert(self, where: Mapping[str, object], data: Mapping[str, object]) -> _RowT_co: ...
-
-    async def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> _RowT_co | None: ...
-
-
-def _tool_table_actions(prisma_client: "PrismaClient") -> "_TableActions[prisma_db_models.LiteLLM_ToolTable]":
-    table: Final[_TableActions[prisma_db_models.LiteLLM_ToolTable]] = ToolRepository(prisma_client).table
+def _tool_table_actions(prisma_client: "PrismaClient") -> "TableActions[prisma_db_models.LiteLLM_ToolTable]":
+    table: Final[TableActions[prisma_db_models.LiteLLM_ToolTable]] = ToolRepository(prisma_client).table
     return table
 
 
 def _object_permission_table_actions(
     prisma_client: "PrismaClient",
-) -> "_TableActions[prisma_db_models.LiteLLM_ObjectPermissionTable]":
-    table: Final[_TableActions[prisma_db_models.LiteLLM_ObjectPermissionTable]] = ObjectPermissionRepository(
+) -> "TableActions[prisma_db_models.LiteLLM_ObjectPermissionTable]":
+    table: Final[TableActions[prisma_db_models.LiteLLM_ObjectPermissionTable]] = ObjectPermissionRepository(
         prisma_client
     ).table
     return table
diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py
index e50a3a5a1e7..20efbe06ecc 100644
--- a/litellm/proxy/guardrails/guardrail_endpoints.py
+++ b/litellm/proxy/guardrails/guardrail_endpoints.py
@@ -29,6 +29,7 @@ from litellm.proxy.guardrails.guardrail_hooks.custom_code.sandbox import (
 from litellm.proxy.guardrails.guardrail_registry import GuardrailRegistry
 from litellm.proxy.guardrails.usage_endpoints import router as guardrails_usage_router
 from litellm.proxy.management_endpoints.common_utils import _user_has_admin_view
+from litellm.repositories.prisma_protocols import TableActions
 from litellm.repositories.table_repositories import GuardrailsRepository
 from litellm.types.guardrails import (
     PII_ENTITY_CATEGORIES_MAP,
@@ -65,29 +66,12 @@ router: Final = APIRouter()
 GUARDRAIL_REGISTRY: Final = GuardrailRegistry()
 
 
-class _GuardrailsTableActions(Protocol):
-    async def create(self, data: Mapping[str, object]) -> "LiteLLM_GuardrailsTable": ...
-
-    async def delete(self, where: Mapping[str, object]) -> "LiteLLM_GuardrailsTable | None": ...
-
-    async def find_unique(self, where: Mapping[str, object]) -> "LiteLLM_GuardrailsTable | None": ...
-
-    async def find_many(
-        self, where: Mapping[str, object], order: Mapping[str, str]
-    ) -> "Sequence[LiteLLM_GuardrailsTable]": ...
-
-    async def update(
-        self, where: Mapping[str, object], data: Mapping[str, object]
-    ) -> "LiteLLM_GuardrailsTable | None": ...
-
-
 def _as_str_object_mapping(mapping: Mapping[str, object]) -> Mapping[str, object]:
     return mapping
 
 
-def _guardrails_table(prisma_client: "PrismaClient") -> _GuardrailsTableActions:
-    table: Final[_GuardrailsTableActions] = GuardrailsRepository(prisma_client).table
-    return table
+def _guardrails_table(prisma_client: "PrismaClient") -> "TableActions[LiteLLM_GuardrailsTable]":
+    return GuardrailsRepository(prisma_client).table
 
 
 async def _create_guardrail_row(prisma_client: "PrismaClient", data: Mapping[str, object]) -> "LiteLLM_GuardrailsTable":
diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py
index d29ec555a80..987e7d778c7 100644
--- a/litellm/proxy/guardrails/guardrail_registry.py
+++ b/litellm/proxy/guardrails/guardrail_registry.py
@@ -3,12 +3,12 @@
 import asyncio
 import importlib
 import os
-from collections.abc import Callable, Iterator, Mapping, Sequence
+from collections.abc import Callable, Iterator, Mapping
 from datetime import datetime, timezone
 from itertools import chain, count
-from typing import Final, Literal, Optional, Protocol, cast
+from typing import TYPE_CHECKING, Final, Literal, Optional, Protocol, cast
 
-from pydantic import BaseModel, ValidationError
+from pydantic import ValidationError
 
 import litellm
 from litellm import Router
@@ -39,6 +39,7 @@ from litellm.proxy.guardrails.guardrail_hooks.tool_permission import (
 )
 from litellm.proxy.types_utils.utils import get_instance_fn
 from litellm.proxy.utils import PrismaClient
+from litellm.repositories.prisma_protocols import TableActions
 from litellm.repositories.table_repositories import GuardrailsRepository
 from litellm.secret_managers.main import get_secret
 from litellm.types.guardrails import (
@@ -61,6 +62,9 @@ from .guardrail_initializers import (
     initialize_tool_permission,
 )
 
+if TYPE_CHECKING:
+    from prisma import models as prisma_models
+
 
 class _GuardrailRowLike(Protocol):
     @property
@@ -68,15 +72,7 @@ class _GuardrailRowLike(Protocol):
     def __iter__(self) -> Iterator[tuple[str, object]]: ...
 
 
-class _GuardrailTableActions(Protocol):
-    async def create(self, *, data: Mapping[str, object]) -> _GuardrailRowLike: ...
-    async def delete(self, *, where: Mapping[str, str]) -> object: ...
-    async def update(self, *, where: Mapping[str, str], data: Mapping[str, object]) -> _GuardrailRowLike: ...
-    async def find_many(self, *, where: Mapping[str, str], order: Mapping[str, str]) -> Sequence[BaseModel]: ...
-    async def find_unique(self, *, where: Mapping[str, str]) -> BaseModel | None: ...
-
-
-def _guardrail_table(prisma_client: PrismaClient) -> _GuardrailTableActions:
+def _guardrail_table(prisma_client: PrismaClient) -> "TableActions[prisma_models.LiteLLM_GuardrailsTable]":
     """Typed view of the guardrails table actions exposed by the Prisma repository."""
     return GuardrailsRepository(prisma_client).table
 
@@ -347,7 +343,7 @@ class GuardrailRegistry:
             guardrail_info: Final[str] = safe_dumps(guardrail.get("guardrail_info", {}))
 
             # Update in DB
-            updated_guardrail: Final[_GuardrailRowLike] = await _guardrail_table(prisma_client).update(
+            updated_guardrail: Final[_GuardrailRowLike | None] = await _guardrail_table(prisma_client).update(
                 where={"guardrail_id": guardrail_id},
                 data={
                     "guardrail_name": guardrail_name,
@@ -356,6 +352,8 @@ class GuardrailRegistry:
                     "updated_at": datetime.now(timezone.utc),
                 },
             )
+            if updated_guardrail is None:
+                raise ValueError(f"Guardrail not found, passed guardrail_id={guardrail_id}")
 
             # Convert to dict and return
             return dict(updated_guardrail)
diff --git a/litellm/proxy/guardrails/usage_endpoints.py b/litellm/proxy/guardrails/usage_endpoints.py
index 9d0d84dc2b1..7a0edbddca8 100644
--- a/litellm/proxy/guardrails/usage_endpoints.py
+++ b/litellm/proxy/guardrails/usage_endpoints.py
@@ -17,6 +17,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict
 from litellm._logging import verbose_proxy_logger
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.repositories.prisma_protocols import TableActions
 from litellm.repositories.table_repositories import (
     DailyGuardrailMetricsRepository,
     DailyGuardrailUsageUnitsRepository,
@@ -30,13 +31,6 @@ from litellm.repositories.table_repositories import (
 if TYPE_CHECKING:
     from prisma import models as prisma_models
     from prisma import types as prisma_types
-    from prisma.actions import (
-        LiteLLM_DailyGuardrailMetricsActions,
-        LiteLLM_DailyGuardrailUsageUnitsActions,
-        LiteLLM_DailyPolicyMetricsActions,
-        LiteLLM_GuardrailsTableActions,
-        LiteLLM_PolicyTableActions,
-    )
 
     from litellm.proxy.utils import PrismaClient
     from litellm.types.guardrails import Guardrail
@@ -85,8 +79,8 @@ def _resolve_usage_window(start_date: str | None, end_date: str | None) -> tuple
 
 def _guardrails_table(
     prisma_client: "PrismaClient",
-) -> "LiteLLM_GuardrailsTableActions[prisma_models.LiteLLM_GuardrailsTable]":
-    guardrails_table: LiteLLM_GuardrailsTableActions[prisma_models.LiteLLM_GuardrailsTable] = GuardrailsRepository(
+) -> "TableActions[prisma_models.LiteLLM_GuardrailsTable]":
+    guardrails_table: Final[TableActions[prisma_models.LiteLLM_GuardrailsTable]] = GuardrailsRepository(
         prisma_client
     ).table
     return guardrails_table
@@ -94,28 +88,26 @@ def _guardrails_table(
 
 def _policies_table(
     prisma_client: "PrismaClient",
-) -> "LiteLLM_PolicyTableActions[prisma_models.LiteLLM_PolicyTable]":
-    policies_table: Final[LiteLLM_PolicyTableActions[prisma_models.LiteLLM_PolicyTable]] = PolicyRepository(
-        prisma_client
-    ).table
+) -> "TableActions[prisma_models.LiteLLM_PolicyTable]":
+    policies_table: Final[TableActions[prisma_models.LiteLLM_PolicyTable]] = PolicyRepository(prisma_client).table
     return policies_table
 
 
 def _daily_guardrail_metrics_table(
     prisma_client: "PrismaClient",
-) -> "LiteLLM_DailyGuardrailMetricsActions[prisma_models.LiteLLM_DailyGuardrailMetrics]":
-    metrics_table: Final[LiteLLM_DailyGuardrailMetricsActions[prisma_models.LiteLLM_DailyGuardrailMetrics]] = (
-        DailyGuardrailMetricsRepository(prisma_client).table
-    )
+) -> "TableActions[prisma_models.LiteLLM_DailyGuardrailMetrics]":
+    metrics_table: Final[TableActions[prisma_models.LiteLLM_DailyGuardrailMetrics]] = DailyGuardrailMetricsRepository(
+        prisma_client
+    ).table
     return metrics_table
 
 
 def _daily_policy_metrics_table(
     prisma_client: "PrismaClient",
-) -> "LiteLLM_DailyPolicyMetricsActions[prisma_models.LiteLLM_DailyPolicyMetrics]":
-    metrics_table: Final[LiteLLM_DailyPolicyMetricsActions[prisma_models.LiteLLM_DailyPolicyMetrics]] = (
-        DailyPolicyMetricsRepository(prisma_client).table
-    )
+) -> "TableActions[prisma_models.LiteLLM_DailyPolicyMetrics]":
+    metrics_table: Final[TableActions[prisma_models.LiteLLM_DailyPolicyMetrics]] = DailyPolicyMetricsRepository(
+        prisma_client
+    ).table
     return metrics_table
 
 
@@ -135,8 +127,8 @@ async def _find_daily_policy_metrics(
 
 def _daily_guardrail_usage_units_table(
     prisma_client: "PrismaClient",
-) -> "LiteLLM_DailyGuardrailUsageUnitsActions[prisma_models.LiteLLM_DailyGuardrailUsageUnits]":
-    units_table: Final[LiteLLM_DailyGuardrailUsageUnitsActions[prisma_models.LiteLLM_DailyGuardrailUsageUnits]] = (
+) -> "TableActions[prisma_models.LiteLLM_DailyGuardrailUsageUnits]":
+    units_table: Final[TableActions[prisma_models.LiteLLM_DailyGuardrailUsageUnits]] = (
         DailyGuardrailUsageUnitsRepository(prisma_client).table
     )
     return units_table
diff --git a/litellm/proxy/guardrails/usage_tracking.py b/litellm/proxy/guardrails/usage_tracking.py
index 820f6438aaf..b8ae09afc00 100644
--- a/litellm/proxy/guardrails/usage_tracking.py
+++ b/litellm/proxy/guardrails/usage_tracking.py
@@ -14,6 +14,8 @@ from operator import itemgetter
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, NamedTuple, TypeVar
 
+from typing_extensions import ReadOnly, TypedDict
+
 from litellm._logging import verbose_proxy_logger
 from litellm.proxy._types import DB_RETRY_SAFE_ERROR_TYPES
 from litellm.proxy.utils import PrismaClient
@@ -47,6 +49,18 @@ class _MetricsKey(NamedTuple):
     date: str
 
 
+class _UsageUnitCompoundKey(TypedDict):
+    guardrail_id: ReadOnly[str]
+    date: ReadOnly[str]
+    team_id: ReadOnly[str]
+    api_key: ReadOnly[str]
+    usage_unit: ReadOnly[str]
+
+
+class _UsageUnitWhereUnique(TypedDict):
+    guardrail_id_date_team_id_api_key_usage_unit: ReadOnly[_UsageUnitCompoundKey]
+
+
 class PendingRollups:
     """Rollup rows whose connection-error retries exhausted, held for the next flush."""
 
@@ -229,7 +243,7 @@ async def _upsert_usage_unit_row(prisma_client: PrismaClient, key: _UsageUnitKey
         "usage_unit": key.usage_unit,
         "units": units,
     }
-    where: Final[prisma_types.LiteLLM_DailyGuardrailUsageUnitsWhereUniqueInput] = {
+    where: Final[_UsageUnitWhereUnique] = {
         "guardrail_id_date_team_id_api_key_usage_unit": {
             "guardrail_id": key.guardrail_id,
             "date": key.date,
diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py
index 2271501d480..b12b764d144 100644
--- a/litellm/proxy/management_endpoints/access_group_endpoints.py
+++ b/litellm/proxy/management_endpoints/access_group_endpoints.py
@@ -390,7 +390,7 @@ async def list_access_groups(
     _require_admin_view(user_api_key_dict)
     prisma_client: Final = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value)
 
-    table: Final[_AccessGroupTable] = AccessGroupRepository(prisma_client).table
+    table: Final = AccessGroupRepository(prisma_client).table
     records: Final = await table.find_many(order={"created_at": "desc"})
     return [_record_to_response(r) for r in records]
 
@@ -406,7 +406,7 @@ async def get_access_group(
     _require_admin_view(user_api_key_dict)
     prisma_client: Final = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value)
 
-    table: Final[_AccessGroupTable] = AccessGroupRepository(prisma_client).table
+    table: Final = AccessGroupRepository(prisma_client).table
     record: Final = await table.find_unique(where={"access_group_id": access_group_id})
     if record is None:
         raise HTTPException(
diff --git a/litellm/proxy/management_endpoints/budget_management_endpoints.py b/litellm/proxy/management_endpoints/budget_management_endpoints.py
index 8c6195388c5..17a845d4300 100644
--- a/litellm/proxy/management_endpoints/budget_management_endpoints.py
+++ b/litellm/proxy/management_endpoints/budget_management_endpoints.py
@@ -93,7 +93,7 @@ async def new_budget(
         budget_obj.budget_reset_at = get_budget_reset_time(budget_duration=budget_obj.budget_duration)
 
     budget_obj_json: Final = budget_obj.model_dump(exclude_none=True)
-    budget_obj_jsonified: Final = jsonify_object(budget_obj_json)  # json dump any dictionaries
+    budget_obj_jsonified: Final[dict[str, object]] = jsonify_object(budget_obj_json)  # mutable-ok: prisma create input
     try:
         response: Final = await BudgetRepository(prisma_client).table.create(
             data={
diff --git a/litellm/proxy/management_endpoints/cache_settings_endpoints.py b/litellm/proxy/management_endpoints/cache_settings_endpoints.py
index 385073edc90..77ff77c9a88 100644
--- a/litellm/proxy/management_endpoints/cache_settings_endpoints.py
+++ b/litellm/proxy/management_endpoints/cache_settings_endpoints.py
@@ -43,7 +43,8 @@ router: Final = APIRouter()
 
 
 class _CacheConfigRow(Protocol):
-    cache_settings: str | Mapping[str, object] | None
+    @property
+    def cache_settings(self) -> str | Mapping[str, object] | None: ...
 
 
 class _CacheConfigTable(Protocol):
diff --git a/litellm/proxy/management_endpoints/common_daily_activity.py b/litellm/proxy/management_endpoints/common_daily_activity.py
index 3d2fa798e03..d3968bf323b 100644
--- a/litellm/proxy/management_endpoints/common_daily_activity.py
+++ b/litellm/proxy/management_endpoints/common_daily_activity.py
@@ -441,7 +441,7 @@ async def get_api_key_metadata(
     This ensures that key_alias and team_id are preserved in historical activity logs
     even after a key is deleted or regenerated.
     """
-    key_records: list[PrismaVerificationToken] = await VerificationTokenRepository(prisma_client).table.find_many(
+    key_records: Sequence[PrismaVerificationToken] = await VerificationTokenRepository(prisma_client).table.find_many(
         where={"token": {"in": list(api_keys)}}
     )
     result: Final[dict[str, _KeyMetadataDict]] = {
@@ -452,9 +452,9 @@ async def get_api_key_metadata(
     missing_keys: Final = api_keys - set(result.keys())
     if missing_keys:
         try:
-            deleted_key_records: Final[list[PrismaDeletedVerificationToken]] = await DeletedVerificationTokenRepository(
-                prisma_client
-            ).table.find_many(
+            deleted_key_records: Final[
+                Sequence[PrismaDeletedVerificationToken]
+            ] = await DeletedVerificationTokenRepository(prisma_client).table.find_many(
                 where={"token": {"in": list(missing_keys)}},
                 order={"deleted_at": "desc"},
             )
diff --git a/litellm/proxy/management_endpoints/config_override_endpoints.py b/litellm/proxy/management_endpoints/config_override_endpoints.py
index dde0751d98d..7f4faddf178 100644
--- a/litellm/proxy/management_endpoints/config_override_endpoints.py
+++ b/litellm/proxy/management_endpoints/config_override_endpoints.py
@@ -46,7 +46,8 @@ router: Final = APIRouter()
 
 
 class _ConfigOverrideRow(Protocol):
-    config_value: str | Mapping[str, object] | None
+    @property
+    def config_value(self) -> str | Mapping[str, object] | None: ...
 
 
 class _ConfigOverridesTableClient(Protocol):
diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py
index c2f5b8eeb8b..9a98bdbb6b1 100644
--- a/litellm/proxy/management_endpoints/internal_user_endpoints.py
+++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py
@@ -15,9 +15,9 @@ These are members of a Team on LiteLLM
 import asyncio
 import json
 import traceback
-from collections.abc import Mapping, Sequence
+from collections.abc import Awaitable, Mapping, Sequence
 from datetime import datetime, timezone
-from typing import Any, Final, Literal, Protocol, cast
+from typing import Any, Final, Literal, cast
 
 import fastapi
 from fastapi import APIRouter, Depends, Header, HTTPException, Request, status
@@ -58,6 +58,7 @@ from litellm.proxy.management_helpers.object_permission_utils import (
 from litellm.proxy.management_helpers.utils import management_endpoint_wrapper
 from litellm.proxy.utils import handle_exception_on_proxy, hash_password
 from litellm.repositories.organization_repository import OrganizationRepository
+from litellm.repositories.prisma_protocols import TableActions
 from litellm.repositories.table_repositories import (
     InvitationLinkRepository,
     OrganizationMembershipRepository,
@@ -86,15 +87,6 @@ from litellm.types.proxy.management_endpoints.scim_v2 import (
 if TYPE_CHECKING:
     from prisma import models as prisma_models
     from prisma import types as prisma_types
-    from prisma.actions import (
-        LiteLLM_InvitationLinkActions,
-        LiteLLM_OrganizationMembershipActions,
-        LiteLLM_OrganizationTableActions,
-        LiteLLM_TeamMembershipActions,
-        LiteLLM_TeamTableActions,
-        LiteLLM_UserTableActions,
-        LiteLLM_VerificationTokenActions,
-    )
 
     from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
     from litellm.proxy.proxy_server import PrismaClient
@@ -105,31 +97,31 @@ router: Final = APIRouter()
 
 def _user_table(
     prisma_client: "PrismaClient | None",
-) -> "LiteLLM_UserTableActions[prisma_models.LiteLLM_UserTable]":
-    user_table: Final[LiteLLM_UserTableActions[prisma_models.LiteLLM_UserTable]] = UserRepository(prisma_client).table
+) -> "TableActions[prisma_models.LiteLLM_UserTable]":
+    user_table: Final[TableActions[prisma_models.LiteLLM_UserTable]] = UserRepository(prisma_client).table
     return user_table
 
 
 def _team_table(
     prisma_client: "PrismaClient | None",
-) -> "LiteLLM_TeamTableActions[prisma_models.LiteLLM_TeamTable]":
-    team_table: Final[LiteLLM_TeamTableActions[prisma_models.LiteLLM_TeamTable]] = TeamRepository(prisma_client).table
+) -> "TableActions[prisma_models.LiteLLM_TeamTable]":
+    team_table: Final[TableActions[prisma_models.LiteLLM_TeamTable]] = TeamRepository(prisma_client).table
     return team_table
 
 
 def _verification_token_table(
     prisma_client: "PrismaClient | None",
-) -> "LiteLLM_VerificationTokenActions[prisma_models.LiteLLM_VerificationToken]":
-    token_table: Final[LiteLLM_VerificationTokenActions[prisma_models.LiteLLM_VerificationToken]] = (
-        VerificationTokenRepository(prisma_client).table
-    )
+) -> "TableActions[prisma_models.LiteLLM_VerificationToken]":
+    token_table: Final[TableActions[prisma_models.LiteLLM_VerificationToken]] = VerificationTokenRepository(
+        prisma_client
+    ).table
     return token_table
 
 
 def _organization_membership_table(
     prisma_client: "PrismaClient | None",
-) -> "LiteLLM_OrganizationMembershipActions[prisma_models.LiteLLM_OrganizationMembership]":
-    membership_table: Final[LiteLLM_OrganizationMembershipActions[prisma_models.LiteLLM_OrganizationMembership]] = (
+) -> "TableActions[prisma_models.LiteLLM_OrganizationMembership]":
+    membership_table: Final[TableActions[prisma_models.LiteLLM_OrganizationMembership]] = (
         OrganizationMembershipRepository(prisma_client).table
     )
     return membership_table
@@ -137,8 +129,8 @@ def _organization_membership_table(
 
 def _invitation_link_table(
     prisma_client: "PrismaClient | None",
-) -> "LiteLLM_InvitationLinkActions[prisma_models.LiteLLM_InvitationLink]":
-    invitation_table: LiteLLM_InvitationLinkActions[prisma_models.LiteLLM_InvitationLink] = InvitationLinkRepository(
+) -> "TableActions[prisma_models.LiteLLM_InvitationLink]":
+    invitation_table: Final[TableActions[prisma_models.LiteLLM_InvitationLink]] = InvitationLinkRepository(
         prisma_client
     ).table
     return invitation_table
@@ -146,19 +138,19 @@ def _invitation_link_table(
 
 def _organization_table(
     prisma_client: "PrismaClient | None",
-) -> "LiteLLM_OrganizationTableActions[prisma_models.LiteLLM_OrganizationTable]":
-    organization_table: Final[LiteLLM_OrganizationTableActions[prisma_models.LiteLLM_OrganizationTable]] = (
-        OrganizationRepository(prisma_client).table
-    )
+) -> "TableActions[prisma_models.LiteLLM_OrganizationTable]":
+    organization_table: Final[TableActions[prisma_models.LiteLLM_OrganizationTable]] = OrganizationRepository(
+        prisma_client
+    ).table
     return organization_table
 
 
 def _team_membership_table(
     prisma_client: "PrismaClient | None",
-) -> "LiteLLM_TeamMembershipActions[prisma_models.LiteLLM_TeamMembership]":
-    team_membership_table: Final[LiteLLM_TeamMembershipActions[prisma_models.LiteLLM_TeamMembership]] = (
-        TeamMembershipRepository(prisma_client).table
-    )
+) -> "TableActions[prisma_models.LiteLLM_TeamMembership]":
+    team_membership_table: Final[TableActions[prisma_models.LiteLLM_TeamMembership]] = TeamMembershipRepository(
+        prisma_client
+    ).table
     return team_membership_table
 
 
@@ -294,7 +286,7 @@ async def _add_user_to_organizations(
         organization_member_add,
     )
 
-    tasks: Final = []
+    tasks: Final[list[Awaitable[object]]] = []
     for organization_id in organizations:
         tasks.append(
             organization_member_add(
@@ -406,7 +398,7 @@ async def add_new_user_to_default_team(
     teams: list[str] | list[NewUserRequestTeam],
     prisma_client: "PrismaClient",
 ):
-    tasks: Final = []
+    tasks: Final[list[Awaitable[object]]] = []
     for team in teams:
         user_role: Literal["user", "admin"] = "user"
         max_budget_in_team: float | None = None
@@ -1479,7 +1471,8 @@ async def _update_single_user_helper(
             # Create new user if not found
             non_default_values["user_id"] = str(uuid.uuid4())
             non_default_values["user_email"] = user_request.user_email
-            response = await prisma_client.insert_data(data=non_default_values, table_name="user")
+            inserted_user_row: Final = await prisma_client.insert_data(data=non_default_values, table_name="user")
+            response = inserted_user_row  # pyright: ignore[reportAssignmentType]  # insert_data returns a prisma row
 
     if response is not None:
         await _schedule_user_update_audit_log(
@@ -1795,7 +1788,9 @@ async def bulk_user_update(
 
         # Apply update transformations (reuse existing logic)
         data_json: Final[dict] = data.user_updates.model_dump(exclude_unset=True)
-        non_default_values: Final = _update_internal_user_params(data_json=data_json, data=data.user_updates)
+        non_default_values: Final[dict[str, object]] = _update_internal_user_params(
+            data_json=data_json, data=data.user_updates
+        )
 
         # Remove user identification fields since we're updating by user_id
         non_default_values.pop("user_id", None)
@@ -2149,7 +2144,7 @@ async def get_users(
         _validate_sort_params(sort_by, sort_order) if sort_by is not None and isinstance(sort_by, str) else None
     )
 
-    users: Sequence[prisma_models.LiteLLM_UserTable] | None = await UserRepository(prisma_client).table.find_many(
+    users: Final[Sequence[prisma_models.LiteLLM_UserTable]] = await UserRepository(prisma_client).table.find_many(
         where=where_conditions,
         skip=skip,
         take=page_size,
@@ -2160,10 +2155,7 @@ async def get_users(
     total_count: Final[int] = await UserRepository(prisma_client).table.count(where=where_conditions)
 
     # Get key count for each user
-    if users is not None:
-        user_key_counts = await get_user_key_counts(prisma_client, [user.user_id for user in users])
-    else:
-        user_key_counts = {}
+    user_key_counts: Final = await get_user_key_counts(prisma_client, [user.user_id for user in users])
 
     verbose_proxy_logger.debug("Total count of users: %s", total_count)
 
@@ -2172,17 +2164,14 @@ async def get_users(
 
     # Prepare response
     user_list: list[LiteLLM_UserTableWithKeyCount] = []
-    if users is not None:
-        for user in users:
-            user_dump = user.model_dump()
-            user_dump["metadata"] = _redact_scim_enterprise_metadata(user_dump.get("metadata"))
-            user_list.append(
-                LiteLLM_UserTableWithKeyCount.model_validate(
-                    {**user_dump, "key_count": user_key_counts.get(user.user_id, 0)}
-                )
+    for user in users:
+        user_dump = user.model_dump()
+        user_dump["metadata"] = _redact_scim_enterprise_metadata(user_dump.get("metadata"))
+        user_list.append(
+            LiteLLM_UserTableWithKeyCount.model_validate(
+                {**user_dump, "key_count": user_key_counts.get(user.user_id, 0)}
             )
-    else:
-        user_list = []
+        )
 
     return {
         "users": user_list,
@@ -2193,13 +2182,6 @@ async def get_users(
     }
 
 
-class _DeleteTeamRow(Protocol):
-    team_id: str
-    members_with_roles: object
-
-    def model_dump(self) -> Mapping[str, object]: ...
-
-
 @router.post(
     "/user/delete",
     tags=["Internal User management"],
@@ -2258,9 +2240,9 @@ async def delete_user(
     # loop an org-admin of org-A could delete users in org-B by supplying
     # {"user_ids": [victim_in_org_B], "organization_id": "org-A"}.
     caller_is_proxy_admin: Final = user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value
-    caller_admin_org_ids: set = set()
+    caller_admin_org_ids: set[str] = set()
     if not caller_is_proxy_admin:
-        caller_memberships: Final = (
+        caller_memberships: Final[Sequence[prisma_models.LiteLLM_OrganizationMembership]] = (
             await _organization_membership_table(prisma_client).find_many(
                 where={
                     "user_id": user_api_key_dict.user_id,
@@ -2279,7 +2261,7 @@ async def delete_user(
 
     # Batch-fetch target memberships once before the per-user loop. Avoids
     # an N+1 DB call when delete_user is called with a large user_ids list.
-    target_org_ids_by_user: Final[dict[str, set]] = {}
+    target_org_ids_by_user: Final[dict[str, set[str]]] = {}
     if not caller_is_proxy_admin:
         all_target_memberships: Final = await _organization_membership_table(prisma_client).find_many(
             where={"user_id": {"in": data.user_ids}}
@@ -2319,7 +2301,7 @@ async def delete_user(
         # we do this after the first for loop, since first for loop is for validation. we only want this inserted after validation passes
         if is_audit_logging_enabled():
             # make an audit log for each team deleted
-            _user_row = user_row.json(exclude_none=True)
+            _user_row = user_row.model_dump_json(exclude_none=True)
 
             asyncio.create_task(
                 create_audit_log_for_update(
@@ -2342,10 +2324,10 @@ async def delete_user(
             )
 
         ## CLEANUP MEMBERS_WITH_ROLES
-        fetch_all_teams: Sequence[_DeleteTeamRow] = await TeamRepository(prisma_client).table.find_many(
-            where={"team_id": {"in": user_row.teams}}
-        )
-        teams_to_update = []
+        fetch_all_teams: Sequence[prisma_models.LiteLLM_TeamTable] = await TeamRepository(
+            prisma_client
+        ).table.find_many(where={"team_id": {"in": user_row.teams}})
+        teams_to_update: list[tuple[str, str]] = []
         for team in fetch_all_teams:
             removed_team_members, new_team_members = _cleanup_members_with_roles(
                 existing_team_row=LiteLLM_TeamTable.model_validate(team.model_dump()),
@@ -2357,15 +2339,14 @@ async def delete_user(
             )
             if removed_team_members:
                 _db_new_team_members: list[dict] = [m.model_dump() for m in new_team_members]
-                team.members_with_roles = json.dumps(_db_new_team_members)
-                teams_to_update.append(team)
+                teams_to_update.append((team.team_id, json.dumps(_db_new_team_members)))
 
         ## update teams
 
-        for team in teams_to_update:
+        for team_id, members_with_roles in teams_to_update:
             await TeamRepository(prisma_client).table.update(
-                where={"team_id": team.team_id},
-                data={"members_with_roles": team.members_with_roles},
+                where={"team_id": team_id},
+                data={"members_with_roles": members_with_roles},
             )
     # End of Audit logging
 
diff --git a/litellm/proxy/management_endpoints/jwt_key_mapping_endpoints.py b/litellm/proxy/management_endpoints/jwt_key_mapping_endpoints.py
index 41e52f05c01..9f561eadfbd 100644
--- a/litellm/proxy/management_endpoints/jwt_key_mapping_endpoints.py
+++ b/litellm/proxy/management_endpoints/jwt_key_mapping_endpoints.py
@@ -122,6 +122,9 @@ async def update_jwt_key_mapping(
             where={"id": data.id}, data=update_data
         )
 
+        if updated_mapping is None:
+            raise HTTPException(status_code=404, detail="Mapping not found")
+
         # Invalidate new cache key if claim fields changed
         cache_key = f"jwt_key_mapping:{updated_mapping.jwt_claim_name}:{updated_mapping.jwt_claim_value}"
         await user_api_key_cache.async_delete_cache(cache_key)
diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py
index 54f567b7aa2..f6a4448e7c9 100644
--- a/litellm/proxy/management_endpoints/key_management_endpoints.py
+++ b/litellm/proxy/management_endpoints/key_management_endpoints.py
@@ -123,6 +123,7 @@ from litellm.repositories.budget_repository import BudgetRepository
 from litellm.repositories.config_repository import ConfigParam, ConfigRepository
 from litellm.repositories.credentials_repository import CredentialsRepository
 from litellm.repositories.model_repository import ModelRepository
+from litellm.repositories.prisma_protocols import TableActions
 from litellm.repositories.table_repositories import (
     DeletedVerificationTokenRepository,
     DeprecatedVerificationTokenRepository,
@@ -151,65 +152,22 @@ from litellm.types.utils import (
 
 if TYPE_CHECKING:
     from prisma import Prisma
+    from prisma import models as prisma_models
 
-_PrismaRowT = TypeVar("_PrismaRowT")
 _RepositoryModelT = TypeVar("_RepositoryModelT", bound=BaseModel)
 
 
-class _PrismaTableActions(Protocol[_PrismaRowT]):
-    """Typed view of the Prisma table actions a repository exposes through its untyped ``table``."""
-
-    async def find_unique(
-        self,
-        *,
-        where: Mapping[str, object],
-        include: Mapping[str, object] | None = None,
-    ) -> _PrismaRowT | None: ...
-
-    async def find_first(
-        self,
-        *,
-        where: Mapping[str, object],
-        include: Mapping[str, object] | None = None,
-    ) -> _PrismaRowT | None: ...
-
-    async def find_many(
-        self,
-        *,
-        where: Mapping[str, object] | None = None,
-        include: Mapping[str, object] | None = None,
-        order: Mapping[str, object] | None = None,
-        skip: int | None = None,
-        take: int | None = None,
-    ) -> list[_PrismaRowT]: ...
-
-    async def count(self, *, where: Mapping[str, object] | None = None) -> int: ...
-
-    async def create(self, *, data: Mapping[str, object]) -> _PrismaRowT: ...
-
-    async def create_many(self, *, data: Sequence[Mapping[str, object]]) -> int: ...
-
-    async def delete_many(self, *, where: Mapping[str, object] | None = None) -> int: ...
-
-    async def update(
-        self,
-        *,
-        where: Mapping[str, object],
-        data: Mapping[str, object],
-    ) -> _PrismaRowT | None: ...
-
-    async def upsert(
-        self,
-        *,
-        where: Mapping[str, object],
-        data: Mapping[str, object],
-    ) -> _PrismaRowT: ...
-
-
 class _UserRowLike(Protocol):
-    user_id: str | None
-    user_email: str | None
-    user_alias: str | None
+    """Read-only view of the user columns ``/key/list`` expands keys with."""
+
+    @property
+    def user_id(self) -> str | None: ...
+
+    @property
+    def user_email(self) -> str | None: ...
+
+    @property
+    def user_alias(self) -> str | None: ...
 
     def model_dump(self) -> Mapping[str, object]: ...
 
@@ -217,46 +175,56 @@ class _UserRowLike(Protocol):
 
 
 class _TxTables(Protocol):
-    litellm_proxymodeltable: _PrismaTableActions[object]
+    litellm_proxymodeltable: TableActions[object]
 
 
-class _TableSource(Protocol[_PrismaRowT]):
-    """Repository view that exposes its untyped Prisma ``table`` with a concrete row type."""
+class _ConfigTableActions(Protocol):
+    """Config table surface this module needs; the shared repository seam exposes no ``update``."""
 
-    @property
-    def table(self) -> _PrismaTableActions[_PrismaRowT]: ...
+    async def find_many(self) -> Sequence[ConfigParam]: ...
 
-
-def _table_of(source: _TableSource[_PrismaRowT]) -> _PrismaTableActions[_PrismaRowT]:
-    return source.table
+    async def update(
+        self,
+        *,
+        where: Mapping[str, object],
+        data: Mapping[str, object],
+    ) -> ConfigParam | None: ...
 
 
 def _prisma_table(
     repository: BaseRepository[_RepositoryModelT],
-) -> _PrismaTableActions[_RepositoryModelT]:
-    return _table_of(repository)
+) -> TableActions[_RepositoryModelT]:
+    return cast(  # cast-ok: callers read only the field names the prisma row and repository model share
+        "TableActions[_RepositoryModelT]", repository.table
+    )
 
 
 def _deleted_verification_token_table(
     prisma_client: PrismaClient,
-) -> _PrismaTableActions[LiteLLM_DeletedVerificationToken]:
-    return _table_of(DeletedVerificationTokenRepository(prisma_client))
+) -> "TableActions[prisma_models.LiteLLM_DeletedVerificationToken]":
+    return DeletedVerificationTokenRepository(prisma_client).table
 
 
-def _deprecated_verification_token_table(prisma_client: PrismaClient) -> _PrismaTableActions[object]:
-    return _table_of(DeprecatedVerificationTokenRepository(prisma_client))
+def _deprecated_verification_token_table(
+    prisma_client: PrismaClient,
+) -> "TableActions[prisma_models.LiteLLM_DeprecatedVerificationToken]":
+    return DeprecatedVerificationTokenRepository(prisma_client).table
 
 
-def _user_table(prisma_client: PrismaClient) -> _PrismaTableActions[_UserRowLike]:
-    return _table_of(UserRepository(prisma_client))
+def _user_table(prisma_client: PrismaClient) -> TableActions[_UserRowLike]:
+    return UserRepository(prisma_client).table
 
 
-def _credentials_table(prisma_client: PrismaClient) -> _PrismaTableActions[CredentialItem]:
-    return _table_of(CredentialsRepository(prisma_client))
+def _credentials_table(prisma_client: PrismaClient) -> TableActions[CredentialItem]:
+    return cast(  # cast-ok: the rotation loop reads and rewrites these rows through CredentialItem names only
+        "TableActions[CredentialItem]", CredentialsRepository(prisma_client).table
+    )
 
 
-def _config_table(prisma_client: PrismaClient) -> _PrismaTableActions[ConfigParam]:
-    return _table_of(ConfigRepository(prisma_client))
+def _config_table(prisma_client: PrismaClient) -> _ConfigTableActions:
+    return cast(  # cast-ok: ConfigRepository.table hides the write actions this module needs on that same object
+        "_ConfigTableActions", ConfigRepository(prisma_client).table
+    )
 
 
 async def _check_custom_key_allowed(custom_key_value: str | None) -> None:
@@ -1046,7 +1014,7 @@ async def _common_key_generation_helper(
         )
         new_budget: Final = prisma_client.jsonify_object(budget_row.json(exclude_none=True))
 
-        _budget: Final[LiteLLM_BudgetTable] = await BudgetRepository(prisma_client).table.create(
+        _budget: Final[prisma_models.LiteLLM_BudgetTable] = await BudgetRepository(prisma_client).table.create(
             data={
                 **new_budget,
                 "created_by": user_api_key_dict.user_id or litellm_proxy_admin_name,
@@ -1252,7 +1220,7 @@ async def _common_key_generation_helper(
 
 
 def _check_key_model_specific_limits(
-    keys: list[LiteLLM_VerificationToken],
+    keys: Sequence[LiteLLM_VerificationToken],
     data: GenerateKeyRequest | UpdateKeyRequest,
     entity_rpm_limit: int | None,
     entity_tpm_limit: int | None,
@@ -1323,7 +1291,7 @@ def _check_key_model_specific_limits(
 
 
 def _check_key_rpm_tpm_limits(
-    keys: list[LiteLLM_VerificationToken],
+    keys: Sequence[LiteLLM_VerificationToken],
     data: GenerateKeyRequest | UpdateKeyRequest,
     entity_rpm_limit: int | None,
     entity_tpm_limit: int | None,
@@ -1361,7 +1329,7 @@ def _check_key_rpm_tpm_limits(
 
 
 def check_team_key_model_specific_limits(
-    keys: list[LiteLLM_VerificationToken],
+    keys: Sequence[LiteLLM_VerificationToken],
     team_table: LiteLLM_TeamTableCachedObj,
     data: GenerateKeyRequest | UpdateKeyRequest,
 ) -> None:
@@ -1386,7 +1354,7 @@ def check_team_key_model_specific_limits(
 
 
 def check_team_key_rpm_tpm_limits(
-    keys: list[LiteLLM_VerificationToken],
+    keys: Sequence[LiteLLM_VerificationToken],
     team_table: LiteLLM_TeamTableCachedObj,
     data: GenerateKeyRequest | UpdateKeyRequest,
 ) -> None:
@@ -1494,7 +1462,7 @@ async def _check_project_key_limits(
 
 
 def check_org_key_model_specific_limits(
-    keys: list[LiteLLM_VerificationToken],
+    keys: Sequence[LiteLLM_VerificationToken],
     org_table: LiteLLM_OrganizationTable,
     data: GenerateKeyRequest | UpdateKeyRequest,
 ) -> None:
@@ -1527,7 +1495,7 @@ def check_org_key_model_specific_limits(
 
 
 def check_org_key_rpm_tpm_limits(
-    keys: list[LiteLLM_VerificationToken],
+    keys: Sequence[LiteLLM_VerificationToken],
     org_table: LiteLLM_OrganizationTable,
     data: GenerateKeyRequest | UpdateKeyRequest,
 ) -> None:
@@ -2242,9 +2210,9 @@ async def _get_and_validate_existing_key(
             code=status.HTTP_400_BAD_REQUEST,
         )
 
-    rows: list[LiteLLM_VerificationToken] = await _prisma_table(VerificationTokenRepository(prisma_client)).find_many(
-        where={"key_alias": key_alias}, take=2
-    )
+    rows: Sequence[LiteLLM_VerificationToken] = await _prisma_table(
+        VerificationTokenRepository(prisma_client)
+    ).find_many(where={"key_alias": key_alias}, take=2)
 
     if len(rows) == 0:
         raise ProxyException(
@@ -2407,7 +2375,10 @@ async def _process_single_key_update(
         )
 
     _data: Final = {**non_default_values, "token": update_key_request.key}
-    response: Final = await prisma_client.update_data(token=update_key_request.key, data=_data)
+    response: Final[Mapping[str, object] | None] = cast(  # cast-ok: every update_data branch returns a str-keyed dict
+        "Mapping[str, object] | None",
+        await prisma_client.update_data(token=update_key_request.key, data=_data),
+    )
 
     # Delete cache
     await _delete_cache_key_object(
@@ -3225,7 +3196,7 @@ async def bulk_update_team_keys(
         # `blocked` is Boolean? with no default; `/key/generate` writes NULL. Prisma's `NOT`
         # excludes NULLs, so explicitly OR `false` with `null` to include them.
         now: Final = datetime.now(timezone.utc)
-        existing_keys = await VerificationTokenRepository(prisma_client).table.find_many(
+        existing_keys = await _prisma_table(VerificationTokenRepository(prisma_client)).find_many(
             where={
                 "team_id": data.team_id,
                 "AND": [
@@ -3243,7 +3214,9 @@ async def bulk_update_team_keys(
                     "error": f"Team {data.team_id} has more than {MAX_BATCH_SIZE} keys. Use `key_ids` to update in batches of {MAX_BATCH_SIZE}."
                 },
             )
-        requested_tokens = [row.token for row in existing_keys]
+        requested_tokens = cast(  # cast-ok: token is the table's primary key, so a row read back always carries one
+            "list[str]", [row.token for row in existing_keys]
+        )
     else:
         if data.key_ids is None or len(data.key_ids) == 0:
             raise HTTPException(
@@ -3261,7 +3234,7 @@ async def bulk_update_team_keys(
             seen_hashes.add(h)
             requested_tokens.append(k)
             hashed_key_ids.append(h)
-        existing_keys = await VerificationTokenRepository(prisma_client).table.find_many(
+        existing_keys = await _prisma_table(VerificationTokenRepository(prisma_client)).find_many(
             where={"team_id": data.team_id, "token": {"in": hashed_key_ids}}
         )
 
@@ -3698,7 +3671,7 @@ async def info_key_fn(
         hashed_key: str | None = key
         if key is not None:
             hashed_key = _hash_token_if_needed(token=key)
-        key_info = await VerificationTokenRepository(prisma_client).table.find_unique(
+        key_info = await _prisma_table(VerificationTokenRepository(prisma_client)).find_unique(
             where={"token": hashed_key},
             include={"litellm_budget_table": True},
         )
@@ -3727,7 +3700,7 @@ async def info_key_fn(
             key_info = key_info.model_dump()
         except Exception:
             # if using pydantic v1
-            key_info = key_info.dict()
+            key_info = key_info.dict()  # pyright: ignore[reportDeprecated]  # deliberate pydantic v1 fallback
         key_token_hash: Final = key_info.pop("token")
 
         model_max_budget = key_info.get("model_max_budget") or {}
@@ -4012,7 +3985,10 @@ async def generate_key_helper_fn(
             if table_name is None or table_name == "user":  # do not auto-create users for `/key/generate`
                 ## CREATE USER (If necessary)
                 if query_type == "insert_data":
-                    user_row = await prisma_client.insert_data(data=user_data, table_name="user")
+                    user_row = cast(  # cast-ok: table_name="user" is the insert_data branch returning the user row
+                        "prisma_models.LiteLLM_UserTable | None",
+                        await prisma_client.insert_data(data=user_data, table_name="user"),
+                    )
 
                     if user_row is None:
                         raise Exception("Failed to create user")
@@ -4219,9 +4195,12 @@ async def delete_verification_tokens(
         if prisma_client:
             hashed_tokens: Final[list[str]] = [_hash_token_if_needed(token=key) for key in tokens]
             tokens = hashed_tokens
-            _keys_being_deleted: Final[list[LiteLLM_VerificationToken]] = await _prisma_table(
-                VerificationTokenRepository(prisma_client)
-            ).find_many(where={"token": {"in": hashed_tokens}})
+            _keys_being_deleted: Final[list[LiteLLM_VerificationToken]] = cast(  # cast-ok: find_many returns a list
+                "list[LiteLLM_VerificationToken]",
+                await _prisma_table(VerificationTokenRepository(prisma_client)).find_many(
+                    where={"token": {"in": hashed_tokens}}
+                ),
+            )
 
             if len(_keys_being_deleted) == 0:
                 raise HTTPException(
@@ -4297,7 +4276,7 @@ async def delete_verification_tokens(
 
 
 def _transform_verification_tokens_to_deleted_records(
-    keys: list[LiteLLM_VerificationToken],
+    keys: Sequence[LiteLLM_VerificationToken],
     user_api_key_dict: UserAPIKeyAuth,
     litellm_changed_by: str | None = None,
 ) -> list[dict[str, object]]:
@@ -4372,7 +4351,7 @@ async def _save_deleted_verification_token_records(
 
 
 async def _persist_deleted_verification_tokens(
-    keys: list[LiteLLM_VerificationToken],
+    keys: Sequence[LiteLLM_VerificationToken],
     prisma_client: PrismaClient,
     user_api_key_dict: UserAPIKeyAuth,
     litellm_changed_by: str | None = None,
@@ -4435,7 +4414,9 @@ async def _rotate_master_key(
     from litellm.proxy.proxy_server import proxy_config
 
     try:
-        models: list | None = await _prisma_table(ModelRepository(prisma_client)).find_many()
+        models: list | None = cast(  # cast-ok: find_many returns a real list, which TableActions widens to Sequence
+            "list[object]", await _prisma_table(ModelRepository(prisma_client)).find_many()
+        )
     except Exception:
         models = None
     # 2. process model table
@@ -5361,9 +5342,9 @@ async def validate_key_list_check(
 
     if key_hash:
         try:
-            key_info: Final[LiteLLM_VerificationToken] = await VerificationTokenRepository(
-                prisma_client
-            ).table.find_unique(
+            key_info: Final[LiteLLM_VerificationToken | None] = await _prisma_table(
+                VerificationTokenRepository(prisma_client)
+            ).find_unique(
                 where={"token": key_hash},
             )
         except Exception:
@@ -5373,6 +5354,13 @@ async def validate_key_list_check(
                 param="key_hash",
                 code=status.HTTP_403_FORBIDDEN,
             )
+        if key_info is None:
+            raise ProxyException(
+                message="Key Hash not found.",
+                type=ProxyErrorTypes.bad_request_error,
+                param="key_hash",
+                code=status.HTTP_403_FORBIDDEN,
+            )
         can_user_query_key_info: Final = await _can_user_query_key_info(
             user_api_key_dict=user_api_key_dict,
             key=key_hash,
@@ -5394,8 +5382,9 @@ async def _fetch_user_team_objects(
     if complete_user_info is None or not complete_user_info.teams:
         return []
 
-    teams: Final[list[BaseModel] | None] = await TeamRepository(prisma_client).table.find_many(
-        where={"team_id": {"in": complete_user_info.teams}}
+    teams: Final[Sequence[BaseModel] | None] = cast(  # cast-ok: the None guard below predates the non-optional seam
+        "Sequence[BaseModel] | None",
+        await TeamRepository(prisma_client).table.find_many(where={"team_id": {"in": complete_user_info.teams}}),
     )
     if teams is None:
         return []
@@ -6130,7 +6119,7 @@ async def _list_key_helper(
             key_dict = key.model_dump()
         except Exception:
             # Fallback for Pydantic v1 compatibility
-            key_dict = key.dict()
+            key_dict = key.dict()  # pyright: ignore[reportDeprecated]  # deliberate pydantic v1 fallback
         # Attach object_permission if object_permission_id is set (only for non-deleted keys)
         if not use_deleted_table:
             key_dict = await attach_object_permission_to_dict(key_dict, prisma_client)
@@ -6155,7 +6144,9 @@ async def _list_key_helper(
                 # Use deleted key type to preserve deleted_at, deleted_by, etc.
                 key_list.append(LiteLLM_DeletedVerificationToken.model_validate(key_dict))
             else:
-                key_list.append(UserAPIKeyAuth(**key_dict))  # Return full key object
+                key_list.append(
+                    UserAPIKeyAuth(**key_dict)  # pyright: ignore[reportAny]  # model_dump() is dict[str, Any]
+                )
         else:
             _token = key_dict.get("token")
             key_list.append(cast(str, _token))  # Return only the token
diff --git a/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py b/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py
index 8e8545a51cc..e1a7645e988 100644
--- a/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py
+++ b/litellm/proxy/management_endpoints/model_access_group_management_endpoints.py
@@ -40,17 +40,22 @@ router: Final = APIRouter()
 
 
 class _DeploymentRow(Protocol):
-    model_id: str
-    model_name: str
-    model_info: object
+    @property
+    def model_id(self) -> str: ...
+
+    @property
+    def model_name(self) -> str: ...
+
+    @property
+    def model_info(self) -> object: ...
 
 
 class _ModelTableClient(Protocol):
-    async def find_many(self, where: Mapping[str, object] | None = None) -> Sequence[_DeploymentRow]: ...
+    async def find_many(self, *, where: Mapping[str, object] | None = None) -> Sequence[_DeploymentRow]: ...
 
-    async def find_unique(self, where: Mapping[str, object]) -> _DeploymentRow | None: ...
+    async def find_unique(self, *, where: Mapping[str, object]) -> _DeploymentRow | None: ...
 
-    async def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> object: ...
+    async def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> object: ...
 
 
 def _model_table(prisma_client: PrismaClient) -> _ModelTableClient:
@@ -322,7 +327,9 @@ async def get_all_access_groups_from_db(
 
     for deployment in deployments:
         model_info = deployment.model_info or {}
-        access_groups = model_info.get("access_groups", [])
+        access_groups = model_info.get(  # pyright: ignore[reportAttributeAccessIssue]  # Json reads back as a dict
+            "access_groups", []
+        )
         model_name = deployment.model_name
 
         for access_group in access_groups:
diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py
index 217fc61a56c..3242c6f6084 100644
--- a/litellm/proxy/management_endpoints/model_management_endpoints.py
+++ b/litellm/proxy/management_endpoints/model_management_endpoints.py
@@ -16,7 +16,7 @@ import json
 from collections.abc import Awaitable, Mapping, Sequence
 from json import JSONDecodeError
 from types import MappingProxyType
-from typing import Final, Literal, Protocol, cast
+from typing import TYPE_CHECKING, Final, Literal, Protocol, cast
 
 from fastapi import APIRouter, Depends, Header, HTTPException, Request, status
 from pydantic import BaseModel, ConfigDict, Field, ValidationError
@@ -72,6 +72,7 @@ from litellm.proxy.spend_tracking.ptu_feature_flag import (
 )
 from litellm.proxy.utils import PrismaClient, ProxyLogging
 from litellm.repositories.model_repository import ModelRepository
+from litellm.repositories.prisma_protocols import TableActions
 from litellm.repositories.table_repositories import ModelTableRepository
 from litellm.repositories.team_repository import TeamRepository
 from litellm.router import Router
@@ -100,6 +101,9 @@ from litellm.types.router import (
 )
 from litellm.utils import get_utc_datetime
 
+if TYPE_CHECKING:
+    from prisma import models as prisma_models
+
 router: Final = APIRouter()
 
 
@@ -120,10 +124,14 @@ class UpdatePublicModelGroupsRequest(BaseModel):
 
 
 class _ProxyModelRow(Protocol):
-    model_id: str
-    model_name: str
-    litellm_params: Mapping[str, object]
-    model_info: Mapping[str, object] | None
+    @property
+    def model_id(self) -> str: ...
+
+    @property
+    def model_name(self) -> str: ...
+
+    @property
+    def model_info(self) -> object: ...
 
     def model_dump_json(self, *, exclude_none: bool = False) -> str: ...
 
@@ -133,7 +141,9 @@ class _ProxyModelTable(Protocol):
 
     def find_many(self, *, where: Mapping[str, object]) -> Awaitable[Sequence[_ProxyModelRow]]: ...
 
-    def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> Awaitable[_ProxyModelRow]: ...
+    def update(
+        self, *, where: Mapping[str, object], data: Mapping[str, object]
+    ) -> Awaitable[_ProxyModelRow | None]: ...
 
     def delete(self, *, where: Mapping[str, object]) -> Awaitable[_ProxyModelRow | None]: ...
 
@@ -144,41 +154,35 @@ class _TxModelTables(Protocol):
     litellm_proxymodeltable: _ProxyModelTable
 
 
+class _ExistingModelRow(Protocol):
+    @property
+    def litellm_params(self) -> Mapping[str, object]: ...
+
+    def model_dump_json(self, *, exclude_none: bool = False) -> str: ...
+
+
 class _TeamRow(Protocol):
-    models: Sequence[str]
+    @property
+    def models(self) -> Sequence[str]: ...
 
     def model_dump(self) -> Mapping[str, object]: ...
 
 
-class _TeamTable(Protocol):
+class _TeamLookupTable(Protocol):
     def find_unique(self, *, where: Mapping[str, object]) -> Awaitable[_TeamRow | None]: ...
 
+
+class _TeamTable(_TeamLookupTable, Protocol):
     def update(
         self, *, where: Mapping[str, object], data: Mapping[str, object], include: Mapping[str, bool]
     ) -> Awaitable[LiteLLM_TeamTable]: ...
 
 
-class _TeamIdRef(Protocol):
-    team_id: str
-
-
-class _ModelAliasRow(Protocol):
-    id: int
-    model_aliases: dict[str, str]
-    team: _TeamIdRef | None
-
-
-class _ModelAliasTable(Protocol):
-    def find_many(self, *, include: Mapping[str, bool]) -> Awaitable[Sequence[_ModelAliasRow]]: ...
-
-    def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> Awaitable[object]: ...
-
-
 def _proxy_model_table(prisma_client: PrismaClient) -> _ProxyModelTable:
     return ModelRepository(prisma_client).table
 
 
-def _repo_team_table(prisma_client: PrismaClient) -> _TeamTable:
+def _repo_team_table(prisma_client: PrismaClient) -> _TeamLookupTable:
     return TeamRepository(prisma_client).table
 
 
@@ -186,7 +190,7 @@ def _db_team_table(prisma_client: PrismaClient) -> _TeamTable:
     return prisma_client.db.litellm_teamtable
 
 
-def _model_alias_table(prisma_client: PrismaClient) -> _ModelAliasTable:
+def _model_alias_table(prisma_client: PrismaClient) -> "TableActions[prisma_models.LiteLLM_ModelTable]":
     return ModelTableRepository(prisma_client).table
 
 
@@ -677,6 +681,14 @@ async def patch_model(
             data=update_data,
         )
 
+        if updated_model is None:
+            raise ProxyException(
+                message=f"Model {model_id} not found on proxy.",
+                type=ProxyErrorTypes.not_found_error,
+                code=status.HTTP_404_NOT_FOUND,
+                param=None,
+            )
+
         # Clear cache and reload models (uses config setting or defaults to preserving config models for DB updates)
         live_before_reload: Final = live_model_ids_snapshot()
         reload_outcome: Final = await clear_cache()
@@ -811,7 +823,7 @@ async def _set_model_blocked_status(
             live_after=reload_outcome.live_after,
         )
 
-        return updated_model
+        return updated_model  # pyright: ignore[reportReturnType]  # prisma row, coerced by this route's response_model
 
     except Exception as e:
         verbose_proxy_logger.exception("Error in model %s: %s", action, e)
@@ -897,7 +909,7 @@ async def _add_model_to_db(
     prisma_client: PrismaClient,
     new_encryption_key: str | None = None,
     should_create_model_in_db: bool = True,
-) -> LiteLLM_ProxyModelTable | None:
+) -> "prisma_models.LiteLLM_ProxyModelTable | LiteLLM_ProxyModelTable | None":
     # encrypt litellm params #
     _litellm_params_dict: Final = model_params.litellm_params.dict(exclude_none=True)
     _original_litellm_model_name: Final = model_params.litellm_params.model
@@ -914,8 +926,9 @@ async def _add_model_to_db(
     }
     if model_params.model_info.id is not None:
         _data["model_id"] = model_params.model_info.id
+    _create_data: Final = cast("Mapping[str, object]", _data)  # cast-ok: str-keyed json payload built just above
     if should_create_model_in_db:
-        model_response = await ModelRepository(prisma_client).table.create(data=_data)
+        model_response = await ModelRepository(prisma_client).table.create(data=_create_data)
     else:
         model_response = LiteLLM_ProxyModelTable(**_data)
     return model_response
@@ -925,7 +938,7 @@ async def _add_team_model_to_db(
     model_params: Deployment,
     user_api_key_dict: UserAPIKeyAuth,
     prisma_client: PrismaClient,
-) -> LiteLLM_ProxyModelTable | None:
+) -> "prisma_models.LiteLLM_ProxyModelTable | LiteLLM_ProxyModelTable | None":
     """
     If 'team_id' is provided,
 
@@ -1638,7 +1651,9 @@ async def delete_team_model_alias(
     tasks: Final = []
     removed_model_aliases: Final[list[tuple[str, str]]] = []
     for team_model_alias in team_model_aliases:
-        model_aliases = team_model_alias.model_aliases  # {"alias": "public model name"}
+        model_aliases = cast(  # cast-ok: prisma types Json columns as `str`; the driver hands back the parsed dict
+            "dict[str, str]", team_model_alias.model_aliases
+        )
         id = team_model_alias.id
 
         if public_model_name in model_aliases.values():
@@ -1733,7 +1748,7 @@ async def add_new_model(
             existing_params=None,
         )
 
-        model_response: LiteLLM_ProxyModelTable | None = None
+        model_response: prisma_models.LiteLLM_ProxyModelTable | LiteLLM_ProxyModelTable | None = None
         # update DB
         incoming_model_info: Final = model_params.model_info.model_dump(exclude_none=True)
         _raise_if_ptu_cost_attribution_disabled(incoming_model_info)
@@ -1902,7 +1917,10 @@ async def update_model(
 
         # update DB
         if store_model_in_db is True:
-            _existing_litellm_params_dict: Final = dict(_existing_litellm_params.litellm_params)
+            existing_model_row: Final = cast(  # cast-ok: prisma types Json columns as `str`; the driver parses them
+                "_ExistingModelRow", _existing_litellm_params
+            )
+            _existing_litellm_params_dict: Final = dict(existing_model_row.litellm_params)
 
             if model_params.litellm_params is None:
                 raise Exception("litellm_params not provided")
@@ -1946,8 +1964,8 @@ async def update_model(
                     user_api_key_dict=user_api_key_dict,
                     table_name=LitellmTableNames.PROXY_MODEL_TABLE_NAME,
                     before_value=(
-                        _existing_litellm_params.model_dump_json(exclude_none=True)
-                        if isinstance(_existing_litellm_params, BaseModel)
+                        existing_model_row.model_dump_json(exclude_none=True)
+                        if isinstance(existing_model_row, BaseModel)
                         else None
                     ),
                     after_value=(
diff --git a/litellm/proxy/management_endpoints/organization_endpoints.py b/litellm/proxy/management_endpoints/organization_endpoints.py
index ffca858c0ce..9198aa35f3f 100644
--- a/litellm/proxy/management_endpoints/organization_endpoints.py
+++ b/litellm/proxy/management_endpoints/organization_endpoints.py
@@ -14,7 +14,14 @@ Endpoints for /organization operations
 #### ORGANIZATION MANAGEMENT ####
 
 from collections.abc import Mapping, Sequence
-from typing import TYPE_CHECKING, Annotated, Final, Protocol, overload
+from typing import (
+    TYPE_CHECKING,
+    Annotated,
+    Final,
+    Protocol,
+    cast,  # noqa: TID251  # prisma types Json columns as fields.Json but reads back plain python values
+    overload,
+)
 
 import fastapi
 from fastapi import APIRouter, Depends, HTTPException, Request, status
@@ -74,6 +81,11 @@ if TYPE_CHECKING:
 router: Final = APIRouter()
 
 
+class _ObjectPermissionRow(Protocol):
+    @property
+    def object_permission_id(self) -> str | None: ...
+
+
 class _UserTableClient(Protocol):
     async def find_unique(self, where: Mapping[str, object]) -> "PrismaUserTable | None": ...
 
@@ -681,7 +693,10 @@ async def update_organization(
         existing_metadata: Final = existing_organization_row.metadata or {}
         updated_metadata: Final = updated_organization_row_json.get("metadata", {})
         merged_metadata: Final[Mapping[str, object]] = _update_dictionary(
-            existing_dict=existing_metadata.copy(), new_dict=updated_metadata
+            existing_dict=cast(  # cast-ok: prisma de-serializes a Json column to the plain python dict it stores
+                "dict[str, object]", existing_metadata
+            ).copy(),
+            new_dict=updated_metadata,
         )
         updated_organization_row_json["metadata"] = merged_metadata
 
@@ -720,7 +735,7 @@ async def update_organization(
 
 async def handle_update_object_permission(
     data_json: dict[str, object],
-    existing_organization_row: LiteLLM_OrganizationTable,
+    existing_organization_row: _ObjectPermissionRow,
 ) -> dict[str, object]:
     """
     Handle the update of object permission for an organization.
@@ -1276,17 +1291,20 @@ async def find_member_if_email(user_email: str, prisma_client: PrismaClient) ->
     Find a member if the user_email is in LiteLLM_UserTable
     """
 
+    not_unique_user_email_error: Final = HTTPException(
+        status_code=400,
+        detail={
+            "error": f"Unique user not found for user_email={user_email}. Potential duplicate OR non-existent user_email in LiteLLM_UserTable. Use 'user_id' instead."
+        },
+    )
     try:
-        existing_user_email_row: Final[BaseModel] = await UserRepository(prisma_client).table.find_unique(
+        existing_user_email_row: Final = await UserRepository(prisma_client).table.find_unique(
             where={"user_email": user_email}
         )
     except Exception:
-        raise HTTPException(
-            status_code=400,
-            detail={
-                "error": f"Unique user not found for user_email={user_email}. Potential duplicate OR non-existent user_email in LiteLLM_UserTable. Use 'user_id' instead."
-            },
-        )
+        raise not_unique_user_email_error
+    if existing_user_email_row is None:
+        raise not_unique_user_email_error
     existing_user_email_row_pydantic: Final = LiteLLM_UserTable.model_validate(existing_user_email_row.model_dump())
     return existing_user_email_row_pydantic
 
@@ -1537,7 +1555,10 @@ async def add_member_to_organization(
             _returned_user = await prisma_client.insert_data(data=new_user_defaults, table_name="user")
             if _returned_user is not None:
                 user_object = LiteLLM_UserTable.model_validate(_returned_user.model_dump())
-        elif existing_user_email_row is not None and len(existing_user_email_row) > 1:
+        elif existing_user_email_row is not None and (
+            len(existing_user_email_row)  # pyright: ignore[reportArgumentType]  # find_unique yields a row, not a list
+            > 1
+        ):
             raise HTTPException(
                 status_code=400,
                 detail={"error": "Multiple users with this email found in db. Please use 'user_id' instead."},
diff --git a/litellm/proxy/management_endpoints/scim/scim_transformations.py b/litellm/proxy/management_endpoints/scim/scim_transformations.py
index 2d95d0bea29..49531a7a72d 100644
--- a/litellm/proxy/management_endpoints/scim/scim_transformations.py
+++ b/litellm/proxy/management_endpoints/scim/scim_transformations.py
@@ -33,7 +33,8 @@ class ScimTransformations:
 
         # Get user's teams/groups
         groups: Final = []
-        for team_id in user.teams or []:
+        team_ids: Final[list[str]] = user.teams or []  # mutable-ok: scim reads the user row's team ids
+        for team_id in team_ids:
             team = await TeamRepository(prisma_client).table.find_unique(where={"team_id": team_id})
             if team:
                 team_alias = getattr(team, "team_alias", team.team_id)
diff --git a/litellm/proxy/management_endpoints/scim/scim_v2.py b/litellm/proxy/management_endpoints/scim/scim_v2.py
index 7183e6cb402..4c16f3b4d7b 100644
--- a/litellm/proxy/management_endpoints/scim/scim_v2.py
+++ b/litellm/proxy/management_endpoints/scim/scim_v2.py
@@ -2761,6 +2761,12 @@ async def patch_group(
         if final_team:
             updated_team = final_team
 
+        if updated_team is None:
+            raise HTTPException(
+                status_code=404,
+                detail={"error": f"Group not found with ID: {group_id}"},  # mutable-ok: FastAPI detail contract
+            )
+
         # Convert to SCIM format and return
         scim_group: Final = await ScimTransformations.transform_litellm_team_to_scim_group(
             LiteLLM_TeamTable.model_validate(updated_team.model_dump())
diff --git a/litellm/proxy/management_endpoints/tag_management_endpoints.py b/litellm/proxy/management_endpoints/tag_management_endpoints.py
index 7aeb5039687..b74aa1a4e16 100644
--- a/litellm/proxy/management_endpoints/tag_management_endpoints.py
+++ b/litellm/proxy/management_endpoints/tag_management_endpoints.py
@@ -369,10 +369,10 @@ async def _add_tag_to_deployment(deployment: "Deployment", tag: str):
 
         # Prisma returns litellm_params as dict (already parsed from JSON)
         existing_params = db_model.litellm_params
-        if isinstance(existing_params, str):
+        if isinstance(existing_params, str):  # pyright: ignore[reportUnnecessaryIsInstance]  # prisma Json stub is str
             # If it's a string, parse it
             existing_params = json.loads(existing_params)
-        elif not isinstance(existing_params, dict):
+        elif not isinstance(existing_params, dict):  # pyright: ignore[reportUnnecessaryIsInstance]  # prisma Json stub
             raise Exception(f"Unexpected litellm_params type: {type(existing_params)}")
 
         # Add tag to tags array (preserve encryption of other fields)
diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py
index 14a2a8a98a5..9a2ec38d627 100644
--- a/litellm/proxy/management_endpoints/team_callback_endpoints.py
+++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py
@@ -352,6 +352,9 @@ async def add_team_callbacks(
             include={"object_permission": True},  # mutable-ok: prisma include takes a dict literal
         )
 
+        if new_team_row is None:
+            raise _callback_error(400, f"Team id = {team_id} does not exist. Please use a different team id.")
+
         # Without this a newly registered callback stays dormant for existing keys.
         await _refresh_cached_team(
             team_row=new_team_row,
diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py
index 01254d5c064..c8373fe6c30 100644
--- a/litellm/proxy/management_endpoints/team_endpoints.py
+++ b/litellm/proxy/management_endpoints/team_endpoints.py
@@ -16,7 +16,16 @@ import traceback
 from collections.abc import Mapping, Sequence
 from datetime import datetime, timezone
 from types import MappingProxyType
-from typing import Annotated, Final, NamedTuple, Protocol, TypedDict, TypeVar, cast
+from typing import (
+    TYPE_CHECKING,
+    Annotated,
+    Final,
+    NamedTuple,
+    Protocol,
+    TypedDict,
+    TypeVar,
+    cast,
+)
 
 import fastapi
 from fastapi import APIRouter, Depends, Header, HTTPException, Request, status
@@ -33,21 +42,17 @@ from litellm.proxy._types import (
     BudgetNewRequest,
     CommonProxyErrors,
     DeleteTeamRequest,
-    LiteLLM_AccessGroupTable,
     LiteLLM_AuditLogs,
-    LiteLLM_BudgetTableFull,
     LiteLLM_DeletedTeamTable,
     LiteLLM_ManagementEndpoint_MetadataFields,
     LiteLLM_ManagementEndpoint_MetadataFields_Premium,
     LiteLLM_ModelTable,
-    LiteLLM_OrganizationMembershipTable,
     LiteLLM_OrganizationTable,
     LiteLLM_OrganizationTableWithMembers,
     LiteLLM_TeamMembership,
     LiteLLM_TeamTable,
     LiteLLM_TeamTableCachedObj,
     LiteLLM_UserTable,
-    LiteLLM_VerificationToken,
     LitellmTableNames,
     LitellmUserRoles,
     Member,
@@ -138,6 +143,7 @@ from litellm.proxy.management_helpers.utils import (
 from litellm.proxy.utils import PrismaClient, ProxyLogging, handle_exception_on_proxy
 from litellm.repositories.budget_repository import BudgetRepository
 from litellm.repositories.organization_repository import OrganizationRepository
+from litellm.repositories.prisma_protocols import TableActions
 from litellm.repositories.table_repositories import (
     AccessGroupRepository,
     DeletedTeamRepository,
@@ -169,6 +175,10 @@ from litellm.types.proxy.management_endpoints.team_endpoints import (
     UpdateTeamMemberPermissionsRequest,
 )
 
+if TYPE_CHECKING:
+    from prisma import Prisma
+    from prisma import models as prisma_models
+
 router: Final = APIRouter()
 
 _DbRecordT = TypeVar("_DbRecordT")
@@ -183,95 +193,14 @@ class _TeamIdGroupRow(TypedDict):
     _count: _TeamIdKeyCount
 
 
-class _PrismaTableActions(Protocol[_DbRecordT]):
-    async def find_unique(
-        self,
-        where: Mapping[str, object],
-        include: Mapping[str, bool] | None = None,
-    ) -> _DbRecordT | None: ...
-
-    async def find_first(
-        self,
-        where: Mapping[str, object] | None = None,
-        order: Mapping[str, str] | None = None,
-    ) -> _DbRecordT | None: ...
-
-    async def find_many(
-        self,
-        where: Mapping[str, object] | None = None,
-        include: Mapping[str, bool] | None = None,
-        order: Mapping[str, str] | None = None,
-        skip: int | None = None,
-        take: int | None = None,
-        cursor: Mapping[str, object] | None = None,
-    ) -> list[_DbRecordT]: ...
-
-    async def create(
-        self,
-        data: Mapping[str, object],
-        include: Mapping[str, bool] | None = None,
-    ) -> _DbRecordT: ...
-
-    async def create_many(
-        self,
-        data: Sequence[Mapping[str, object]],
-        skip_duplicates: bool | None = None,
-    ) -> int: ...
-
-    async def update(
-        self,
-        where: Mapping[str, object],
-        data: Mapping[str, object],
-        include: Mapping[str, bool] | None = None,
-    ) -> _DbRecordT: ...
-
-    async def update_many(
-        self,
-        where: Mapping[str, object],
-        data: Mapping[str, object],
-    ) -> int: ...
-
-    async def upsert(
-        self,
-        where: Mapping[str, object],
-        data: Mapping[str, Mapping[str, object]],
-    ) -> _DbRecordT: ...
-
-    async def delete_many(
-        self,
-        where: Mapping[str, object] | None = None,
-    ) -> int: ...
-
-    async def count(
-        self,
-        where: Mapping[str, object] | None = None,
-    ) -> int: ...
-
-    async def group_by(
-        self,
-        by: Sequence[str],
-        where: Mapping[str, object] | None = None,
-        count: Mapping[str, bool] | None = None,
-    ) -> Sequence[_TeamIdGroupRow]: ...
-
-
-class _HasTableActions(Protocol[_DbRecordT]):
-    @property
-    def table(self) -> "_PrismaTableActions[_DbRecordT]": ...
-
-
-def _typed_table(
-    repo: "_HasTableActions[_DbRecordT]", record_type: type[_DbRecordT]
-) -> "_PrismaTableActions[_DbRecordT]":
-    return repo.table
-
-
 def _as_object(value: object) -> object:
     return value
 
 
-def _nullable(value: _DbRecordT | None) -> _DbRecordT | None:
-    return value
+def _as_list(rows: Sequence[_DbRecordT]) -> list[_DbRecordT]:  # mutable-ok: pydantic list[...] fields reject Sequence
+    return cast(  # cast-ok: prisma-client-py find_many returns a list; TableActions only widens it to Sequence
+        "list[_DbRecordT]", rows
+    )
 
 
 class _UserIdRow(Protocol):
@@ -279,33 +208,75 @@ class _UserIdRow(Protocol):
     def user_id(self) -> str | None: ...
 
 
-class _HasUserIdTable(Protocol):
-    @property
-    def table(self) -> "_PrismaTableActions[_UserIdRow]": ...
-
-
-def _user_id_rows_db(repo: "_HasUserIdTable") -> "_PrismaTableActions[_UserIdRow]":
+def _user_id_rows_db(repo: UserRepository) -> "TableActions[_UserIdRow]":
     return repo.table
 
 
-class _RawTeamRow(Protocol):
+class _ModelDumpRow(Protocol):
+    def model_dump(self) -> Mapping[str, object]: ...
+
+
+class _TeamIdRow(Protocol):
     @property
-    def members_with_roles(self) -> Sequence[Mapping[str, object]] | None: ...
+    def team_id(self) -> str: ...
 
 
-class _HasRawTeamTable(Protocol):
+class _CacheableTeamRow(_TeamIdRow, _ModelDumpRow, Protocol): ...
+
+
+class _ObjectPermissionRow(Protocol):
     @property
-    def table(self) -> "_PrismaTableActions[_RawTeamRow]": ...
+    def object_permission_id(self) -> str | None: ...
 
 
-def _raw_team_db(repo: "_HasRawTeamTable") -> "_PrismaTableActions[_RawTeamRow]":
-    return repo.table
+class _TeamAliasBudgetRow(Protocol):
+    @property
+    def team_alias(self) -> str | None: ...
+
+    @property
+    def budget_duration(self) -> str | None: ...
+
+
+class _TeamBudgetRow(_TeamAliasBudgetRow, Protocol):
+    metadata: Mapping[str, JsonValue] | None
+
+
+class _AuditableTeamRow(Protocol):
+    def json(self, *, exclude_none: bool = False) -> str: ...
+
+
+class _RawTeamRow(_TeamIdRow, _ModelDumpRow, _ObjectPermissionRow, _TeamBudgetRow, _AuditableTeamRow, Protocol):
+    @property
+    def members_with_roles(
+        self,
+    ) -> Sequence[dict[str, object]] | None: ...  # mutable-ok: prisma deserializes this JSON column into plain dicts
+
+    @property
+    def organization_id(self) -> str | None: ...
+
+    @property
+    def max_budget(self) -> float | None: ...
+
+    @property
+    def soft_budget(self) -> float | None: ...
+
+    @property
+    def model_id(self) -> int | None: ...
+
+
+def _raw_team_db(repo: TeamRepository) -> "TableActions[_RawTeamRow]":
+    return cast(  # cast-ok: prisma types Json columns as str; the client hands back the deserialized value
+        "TableActions[_RawTeamRow]", repo.table
+    )
+
+
+class _BudgetIdRow(Protocol):
+    @property
+    def budget_id(self) -> str: ...
 
 
 class _BudgetWriteCall(Protocol):
-    async def __call__(
-        self, budget_obj: BudgetNewRequest, user_api_key_dict: UserAPIKeyAuth
-    ) -> LiteLLM_BudgetTableFull: ...
+    async def __call__(self, budget_obj: BudgetNewRequest, user_api_key_dict: UserAPIKeyAuth) -> _BudgetIdRow: ...
 
 
 def _as_budget_write(fn: "_BudgetWriteCall") -> "_BudgetWriteCall":
@@ -330,7 +301,7 @@ class _TeamIdInFilter(TypedDict, total=False):
 
 class _TeamCreateTx(AccessGroupSyncTx, Protocol):
     @property
-    def litellm_teamtable(self) -> "_PrismaTableActions[LiteLLM_TeamTable]": ...
+    def litellm_teamtable(self) -> "TableActions[prisma_models.LiteLLM_TeamTable]": ...
 
 
 _STRIP_DELETED_TEAM_FROM_USERS_SQL: Final = """
@@ -340,46 +311,52 @@ UPDATE "LiteLLM_UserTable" SET teams = array_remove(teams, $1) WHERE $1 = ANY(te
 _INCLUDE_MODEL_TABLE: Final = MappingProxyType({"litellm_model_table": True})
 
 
-def _team_db(prisma_client: PrismaClient | None) -> "_PrismaTableActions[LiteLLM_TeamTable]":
-    return _typed_table(TeamRepository(prisma_client), LiteLLM_TeamTable)
+def _team_db(prisma_client: PrismaClient | None) -> "TableActions[prisma_models.LiteLLM_TeamTable]":
+    return TeamRepository(prisma_client).table
 
 
-def _team_membership_db(prisma_client: PrismaClient | None) -> "_PrismaTableActions[LiteLLM_TeamMembership]":
-    return _typed_table(TeamMembershipRepository(prisma_client), LiteLLM_TeamMembership)
+def _team_tx_db(tx: "Prisma") -> "TableActions[prisma_models.LiteLLM_TeamTable]":
+    return cast(  # cast-ok: generated actions type Json columns as str; TableActions widens inputs to Mapping
+        "TableActions[prisma_models.LiteLLM_TeamTable]", tx.litellm_teamtable
+    )
 
 
-def _user_db(prisma_client: PrismaClient | None) -> "_PrismaTableActions[LiteLLM_UserTable]":
-    return _typed_table(UserRepository(prisma_client), LiteLLM_UserTable)
+def _team_membership_db(prisma_client: PrismaClient | None) -> "TableActions[prisma_models.LiteLLM_TeamMembership]":
+    return TeamMembershipRepository(prisma_client).table
 
 
-def _model_db(prisma_client: PrismaClient | None) -> "_PrismaTableActions[LiteLLM_ModelTable]":
-    return _typed_table(ModelTableRepository(prisma_client), LiteLLM_ModelTable)
+def _user_db(prisma_client: PrismaClient | None) -> "TableActions[prisma_models.LiteLLM_UserTable]":
+    return UserRepository(prisma_client).table
 
 
-def _org_db(prisma_client: PrismaClient | None) -> "_PrismaTableActions[LiteLLM_OrganizationTable]":
-    return _typed_table(OrganizationRepository(prisma_client), LiteLLM_OrganizationTable)
+def _model_db(prisma_client: PrismaClient | None) -> "TableActions[prisma_models.LiteLLM_ModelTable]":
+    return ModelTableRepository(prisma_client).table
+
+
+def _org_db(prisma_client: PrismaClient | None) -> "TableActions[prisma_models.LiteLLM_OrganizationTable]":
+    return OrganizationRepository(prisma_client).table
 
 
 def _org_membership_db(
     prisma_client: PrismaClient | None,
-) -> "_PrismaTableActions[LiteLLM_OrganizationMembershipTable]":
-    return _typed_table(OrganizationMembershipRepository(prisma_client), LiteLLM_OrganizationMembershipTable)
+) -> "TableActions[prisma_models.LiteLLM_OrganizationMembership]":
+    return OrganizationMembershipRepository(prisma_client).table
 
 
-def _budget_db(prisma_client: PrismaClient | None) -> "_PrismaTableActions[LiteLLM_BudgetTableFull]":
-    return _typed_table(BudgetRepository(prisma_client), LiteLLM_BudgetTableFull)
+def _budget_db(prisma_client: PrismaClient | None) -> "TableActions[prisma_models.LiteLLM_BudgetTable]":
+    return BudgetRepository(prisma_client).table
 
 
-def _deleted_team_db(prisma_client: PrismaClient | None) -> "_PrismaTableActions[LiteLLM_DeletedTeamTable]":
-    return _typed_table(DeletedTeamRepository(prisma_client), LiteLLM_DeletedTeamTable)
+def _deleted_team_db(prisma_client: PrismaClient | None) -> "TableActions[prisma_models.LiteLLM_DeletedTeamTable]":
+    return DeletedTeamRepository(prisma_client).table
 
 
-def _access_group_db(prisma_client: PrismaClient | None) -> "_PrismaTableActions[LiteLLM_AccessGroupTable]":
-    return _typed_table(AccessGroupRepository(prisma_client), LiteLLM_AccessGroupTable)
+def _access_group_db(prisma_client: PrismaClient | None) -> "TableActions[prisma_models.LiteLLM_AccessGroupTable]":
+    return AccessGroupRepository(prisma_client).table
 
 
-def _tokens_db(prisma_client: PrismaClient | None) -> "_PrismaTableActions[LiteLLM_VerificationToken]":
-    return _typed_table(VerificationTokenRepository(prisma_client), LiteLLM_VerificationToken)
+def _tokens_db(prisma_client: PrismaClient | None) -> "TableActions[prisma_models.LiteLLM_VerificationToken]":
+    return VerificationTokenRepository(prisma_client).table
 
 
 def _sanitize_for_log(value: object) -> str:
@@ -392,7 +369,7 @@ def _sanitize_for_log(value: object) -> str:
 
 
 async def _refresh_cached_team(
-    team_row: LiteLLM_TeamTable,
+    team_row: _CacheableTeamRow,
     user_api_key_cache: UserApiKeyCache,
     proxy_logging_obj: ProxyLogging,
 ) -> None:
@@ -481,7 +458,7 @@ class TeamMemberBudgetHandler:
 
     @staticmethod
     async def create_team_member_budget_table(
-        data: NewTeamRequest | LiteLLM_TeamTable,
+        data: NewTeamRequest | _TeamAliasBudgetRow,
         new_team_data_json: dict,
         user_api_key_dict: UserAPIKeyAuth,
         team_member_budget: float | None = None,
@@ -532,7 +509,7 @@ class TeamMemberBudgetHandler:
 
     @staticmethod
     async def upsert_team_member_budget_table(
-        team_table: LiteLLM_TeamTable,
+        team_table: _TeamBudgetRow,
         user_api_key_dict: UserAPIKeyAuth,
         updated_kv: dict,
         team_member_budget: float | None = None,
@@ -603,7 +580,7 @@ class TeamMemberBudgetHandler:
 
     @staticmethod
     async def clear_team_member_budget_fields(
-        team_table: LiteLLM_TeamTable,
+        team_table: _TeamBudgetRow,
         user_api_key_dict: "UserAPIKeyAuth",
         updated_kv: dict,
         explicitly_set_fields: set,
@@ -1540,7 +1517,7 @@ async def new_team(
 
         tx: _TeamCreateTx
         async with prisma_client.db.tx() as tx:
-            team_row: Final[LiteLLM_TeamTable] = await tx.litellm_teamtable.create(
+            team_row: Final[prisma_models.LiteLLM_TeamTable] = await tx.litellm_teamtable.create(
                 data=team_creation_data,
                 include=_INCLUDE_MODEL_TABLE,
             )
@@ -1595,7 +1572,7 @@ async def new_team(
 
 
 async def _create_team_update_audit_log(
-    existing_team_row: LiteLLM_TeamTable,
+    existing_team_row: _AuditableTeamRow,
     updated_kv: dict,
     team_id: str,
     litellm_changed_by: str | None,
@@ -1718,11 +1695,11 @@ async def _auto_add_team_members_to_organization(
 
 async def fetch_and_validate_organization(
     organization_id: str,
-    existing_team_row: LiteLLM_TeamTable,
+    existing_team_row: _ModelDumpRow,
     llm_router: Router | None,
     prisma_client: PrismaClient,
     user_api_key_dict: UserAPIKeyAuth | None = None,
-) -> LiteLLM_OrganizationTable:
+) -> "prisma_models.LiteLLM_OrganizationTable":
     """
     Fetch and validate an organization for team update operations.
 
@@ -1996,7 +1973,9 @@ async def update_team(
         validate_budget_duration(data.budget_duration)
         validate_budget_duration(data.team_member_budget_duration)
 
-        existing_team_row = await _team_db(prisma_client).find_unique(where={"team_id": data.team_id})
+        existing_team_row = await _raw_team_db(TeamRepository(prisma_client)).find_unique(
+            where={"team_id": data.team_id}
+        )
 
         if existing_team_row is None:
             raise HTTPException(
@@ -2234,18 +2213,16 @@ async def update_team(
 
         updated_kv = prisma_client.jsonify_team_object(db_data=updated_kv)
         team_update_data: Final[Mapping[str, object]] = updated_kv
-        team_row: Final[LiteLLM_TeamTable | None] = _nullable(
-            await _team_db(prisma_client).update(
-                where={"team_id": data.team_id},
-                data=team_update_data,
-                # `object_permission` is included so `_refresh_cached_team`
-                # doesn't write a cached team with the relation nulled out —
-                # see team_model_add for the full rationale.
-                include={
-                    "litellm_model_table": True,
-                    "object_permission": True,
-                },
-            )
+        team_row: Final = await _team_db(prisma_client).update(
+            where={"team_id": data.team_id},
+            data=team_update_data,
+            # `object_permission` is included so `_refresh_cached_team`
+            # doesn't write a cached team with the relation nulled out.
+            # See team_model_add for the full rationale.
+            include={
+                "litellm_model_table": True,
+                "object_permission": True,
+            },
         )
 
         if team_row is None or team_row.team_id is None:
@@ -2375,7 +2352,7 @@ def _set_budget_reset_at(data: UpdateTeamRequest, updated_kv: dict) -> None:
         updated_kv["budget_limits"] = json.dumps(initialized_windows)
 
 
-async def handle_update_object_permission(data_json: dict, existing_team_row: LiteLLM_TeamTable) -> dict:
+async def handle_update_object_permission(data_json: dict, existing_team_row: _ObjectPermissionRow) -> dict:
     """
     Handle the update of object permission for a team.
 
@@ -2705,7 +2682,7 @@ async def _add_team_members_to_team(
     prisma_client: PrismaClient,
     user_api_key_dict: UserAPIKeyAuth,
     litellm_proxy_admin_name: str,
-) -> tuple[LiteLLM_TeamTable, list[LiteLLM_UserTable], list[LiteLLM_TeamMembership]]:
+) -> tuple["prisma_models.LiteLLM_TeamTable", list[LiteLLM_UserTable], list[LiteLLM_TeamMembership]]:
     """Add team members to the team.
 
     The members_with_roles reconciliation runs inside a transaction that locks
@@ -2750,7 +2727,7 @@ async def _write_members_with_roles_locked(
     complete_team_data: LiteLLM_TeamTable,
     prisma_client: PrismaClient,
     updated_users: list[LiteLLM_UserTable],
-) -> LiteLLM_TeamTable | None:
+) -> "prisma_models.LiteLLM_TeamTable | None":
     """Reconcile members_with_roles under the team row lock. None when the team row is gone.
 
     That read is at least as recent as the user and membership writes the caller
@@ -2772,7 +2749,7 @@ async def _write_members_with_roles_locked(
         )
 
         _db_team_members: Final = [m.model_dump() for m in complete_team_data.members_with_roles]
-        return await tx.litellm_teamtable.update(
+        return await _team_tx_db(tx).update(
             where={"team_id": data.team_id},
             data={"members_with_roles": json.dumps(_db_team_members)},
         )
@@ -3292,7 +3269,9 @@ async def team_member_delete(
     key_val: Final[Mapping[str, object]] = (
         {"user_id": {"in": sorted(removed_user_ids)}} if removed_user_ids else {"user_email": data.user_email}
     )
-    existing_user_rows: Final[Sequence[LiteLLM_UserTable]] = await _user_db(prisma_client).find_many(where=key_val)
+    existing_user_rows: Final[Sequence[prisma_models.LiteLLM_UserTable]] = await _user_db(prisma_client).find_many(
+        where=key_val
+    )
 
     # Also clean up any existing team membership rows for this user and team
     user_ids_to_delete: Final = removed_user_ids.union(
@@ -3303,7 +3282,7 @@ async def team_member_delete(
     ## DELETE KEYS CREATED BY USER FOR THIS TEAM
     # Fetch keys before deletion so their audit records can be persisted alongside the delete.
     # An empty user_ids_to_delete still resolves cleanly: prisma's "in": [] matches no rows.
-    keys_to_delete: Final[list[LiteLLM_VerificationToken]] = await _tokens_db(prisma_client).find_many(
+    keys_to_delete: Final = await _tokens_db(prisma_client).find_many(
         where={
             "user_id": {"in": sorted(user_ids_to_delete)},
             "team_id": data.team_id,
@@ -3313,7 +3292,7 @@ async def team_member_delete(
     # All four cleanups run on one connection so a failure between them leaves
     # no partial removal: either every write below lands, or none of them do.
     async with prisma_client.tx() as tx:
-        await tx.litellm_teamtable.update(
+        await _team_tx_db(tx).update(
             where={"team_id": data.team_id},
             data={"members_with_roles": json.dumps(_db_new_team_members)},
         )
@@ -3826,9 +3805,7 @@ async def delete_team(
         _persist_deleted_verification_tokens,
     )
 
-    keys_to_delete: list[LiteLLM_VerificationToken] = await _tokens_db(prisma_client).find_many(
-        where={"team_id": {"in": data.team_ids}}
-    )
+    keys_to_delete: Final = await _tokens_db(prisma_client).find_many(where={"team_id": {"in": data.team_ids}})
 
     if keys_to_delete:
         await _persist_deleted_verification_tokens(
@@ -3930,7 +3907,7 @@ async def _sweep_deleted_team_references(team_ids: Sequence[str], prisma_client:
 
 
 async def _invalidate_deleted_key_cache(
-    keys: Sequence[LiteLLM_VerificationToken],
+    keys: "Sequence[prisma_models.LiteLLM_VerificationToken]",
     user_api_key_cache: UserApiKeyCache,
     proxy_logging_obj: ProxyLogging,
 ) -> None:
@@ -4115,7 +4092,7 @@ async def _hydrate_member_emails(
     if not missing_user_ids:
         return tuple(members)
 
-    user_rows: Final[Sequence[LiteLLM_UserTable]] = await _user_db(prisma_client).find_many(
+    user_rows: Final[Sequence[prisma_models.LiteLLM_UserTable]] = await _user_db(prisma_client).find_many(
         where={  # mutable-ok: Prisma query filters are dict-shaped
             "user_id": {  # mutable-ok: Prisma query filters are dict-shaped
                 "in": sorted(missing_user_ids)
@@ -4126,7 +4103,7 @@ async def _hydrate_member_emails(
 
     return tuple(
         m.model_copy(update={"user_email": email_by_user_id[m.user_id]})  # mutable-ok: pydantic update payload
-        if not m.user_email and m.user_id in email_by_user_id
+        if not m.user_email and m.user_id is not None and m.user_id in email_by_user_id
         else m
         for m in members
     )
@@ -4711,7 +4688,7 @@ async def _build_team_list_where_conditions(
 
 async def _batch_resolve_access_group_resources(
     all_access_group_ids: list[str],
-) -> dict[str, LiteLLM_AccessGroupTable]:
+) -> "dict[str, prisma_models.LiteLLM_AccessGroupTable]":
     """
     Batch-fetch access groups in a single DB query and return them keyed by
     access_group_id. Missing/invalid groups are silently omitted.
@@ -4729,7 +4706,7 @@ async def _batch_resolve_access_group_resources(
 
 
 def _convert_teams_to_response_models(
-    teams: list,
+    teams: Sequence,
     use_deleted_table: bool,
     keys_count_by_team: dict[str, int] | None = None,
 ) -> list[TeamListItem | LiteLLM_TeamTable | LiteLLM_DeletedTeamTable]:
@@ -4763,7 +4740,7 @@ def _convert_teams_to_response_models(
 
 async def _get_keys_count_by_team(
     prisma_client: PrismaClient,
-    teams: Sequence[LiteLLM_TeamTable],
+    teams: Sequence[_TeamIdRow],
 ) -> dict[str, int]:
     """Aggregate virtual-key counts per team for the given page of teams.
 
@@ -4775,10 +4752,13 @@ async def _get_keys_count_by_team(
     if not page_team_ids:
         return {}
 
-    grouped: Final = await _tokens_db(prisma_client).group_by(
-        by=["team_id"],
-        where={"team_id": {"in": page_team_ids}},
-        count={"team_id": True},
+    grouped: Final = cast(  # cast-ok: prisma group_by returns one row per `by` key with `count=` nested under "_count"
+        "Sequence[_TeamIdGroupRow]",
+        await _tokens_db(prisma_client).group_by(
+            by=["team_id"],
+            where={"team_id": {"in": page_team_ids}},
+            count={"team_id": True},
+        ),
     )
     return {row["team_id"]: row.get("_count", {}).get("team_id", 0) for row in grouped if row.get("team_id")}
 
@@ -5168,7 +5148,7 @@ async def list_team(
                 _team_memberships.append(tm)
 
         # add all keys that belong to the team
-        keys = await _tokens_db(prisma_client).find_many(where={"team_id": team.team_id})
+        keys = _as_list(await _tokens_db(prisma_client).find_many(where={"team_id": team.team_id}))
 
         try:
             returned_responses.append(
@@ -5403,6 +5383,11 @@ async def team_model_add(
         data={"updated_at": datetime.now(timezone.utc)},
         include={"object_permission": True},
     )
+    if updated_team is None:
+        raise HTTPException(
+            status_code=404,
+            detail={"error": f"Team not found, passed team_id={data.team_id}"},
+        )
 
     await _refresh_cached_team(
         team_row=updated_team,
@@ -5485,6 +5470,11 @@ async def team_model_delete(
         data={"models": updated_models},
         include={"object_permission": True},
     )
+    if updated_team is None:
+        raise HTTPException(
+            status_code=404,
+            detail={"error": f"Team not found, passed team_id={data.team_id}"},
+        )
 
     await _refresh_cached_team(
         team_row=updated_team,
@@ -5619,8 +5609,13 @@ async def update_team_member_permissions(
         where={"team_id": data.team_id},
         data={"team_member_permissions": data.team_member_permissions},
     )
+    if updated_team is None:
+        raise HTTPException(
+            status_code=404,
+            detail={"error": f"Team not found, passed team_id={data.team_id}"},
+        )
 
-    return updated_team
+    return updated_team  # pyright: ignore[reportReturnType]  # prisma row, coerced by this route's response_model
 
 
 @router.post(
@@ -5685,7 +5680,9 @@ async def bulk_update_team_member_permissions(
     }
 
 
-async def _compute_and_batch_updates(prisma_client, teams: Sequence[LiteLLM_TeamTable], permissions_to_add: set) -> int:
+async def _compute_and_batch_updates(
+    prisma_client, teams: "Sequence[prisma_models.LiteLLM_TeamTable]", permissions_to_add: set
+) -> int:
     """Compute merged permissions and batch-write updates. Returns count of teams updated."""
     updates: Final = []
     for team in teams:
diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py
index 3c135650de9..0c8240b3298 100644
--- a/litellm/proxy/management_endpoints/ui_sso.py
+++ b/litellm/proxy/management_endpoints/ui_sso.py
@@ -29,7 +29,6 @@ from typing import (
     NoReturn,
     Optional,
     Protocol,
-    TypeVar,
     Union,
     cast,
     overload,
@@ -122,6 +121,7 @@ from litellm.proxy.utils import (
     get_custom_url,
     get_server_root_path,
 )
+from litellm.repositories.prisma_protocols import TableActions
 from litellm.repositories.table_repositories import SSOConfigRepository
 from litellm.repositories.team_repository import TeamRepository
 from litellm.repositories.user_repository import UserRepository
@@ -171,51 +171,16 @@ _CLI_SSO_SECRET_KEY_FRAGMENTS: Final = frozenset(
     }
 )
 
-_DbRecordT: Final = TypeVar("_DbRecordT", covariant=True)
-
-
-class _PrismaTableActions(Protocol[_DbRecordT]):
-    async def find_unique(
-        self,
-        where: Mapping[str, object],
-    ) -> _DbRecordT | None: ...
-
-    async def find_first(
-        self,
-        where: Mapping[str, object] | None = None,
-    ) -> _DbRecordT | None: ...
-
-    async def find_many(
-        self,
-        where: Mapping[str, object] | None = None,
-        include: Mapping[str, bool] | None = None,
-    ) -> Sequence[_DbRecordT]: ...
-
-    async def update(
-        self,
-        where: Mapping[str, object],
-        data: Mapping[str, object],
-    ) -> _DbRecordT: ...
-
-    async def update_many(
-        self,
-        where: Mapping[str, object],
-        data: Mapping[str, object],
-    ) -> int: ...
-
 
 class _UserMetadataRow(Protocol):
     @property
     def metadata(self) -> Mapping[str, object] | None: ...
 
 
-class _HasUserMetadataTable(Protocol):
-    @property
-    def table(self) -> "_PrismaTableActions[_UserMetadataRow]": ...
-
-
-def _user_meta_db(repo: "_HasUserMetadataTable") -> "_PrismaTableActions[_UserMetadataRow]":
-    return repo.table
+def _user_meta_db(repo: UserRepository) -> "TableActions[_UserMetadataRow]":
+    return cast(  # cast-ok: prisma types Json columns as str; the client hands back the deserialized value
+        "TableActions[_UserMetadataRow]", repo.table
+    )
 
 
 class _SsoConfigRow(Protocol):
@@ -223,25 +188,17 @@ class _SsoConfigRow(Protocol):
     def sso_settings(self) -> Mapping[str, object] | None: ...
 
 
-class _HasSsoConfigTable(Protocol):
-    @property
-    def table(self) -> "_PrismaTableActions[_SsoConfigRow]": ...
-
-
-def _sso_config_db(repo: "_HasSsoConfigTable") -> "_PrismaTableActions[_SsoConfigRow]":
-    return repo.table
+def _sso_config_db(repo: SSOConfigRepository) -> "TableActions[_SsoConfigRow]":
+    return cast(  # cast-ok: prisma types Json columns as str; the client hands back the deserialized value
+        "TableActions[_SsoConfigRow]", repo.table
+    )
 
 
 class _TeamDetailRow(Protocol):
     def model_dump(self) -> Mapping[str, object]: ...
 
 
-class _HasTeamDetailTable(Protocol):
-    @property
-    def table(self) -> "_PrismaTableActions[_TeamDetailRow]": ...
-
-
-def _team_detail_db(repo: "_HasTeamDetailTable") -> "_PrismaTableActions[_TeamDetailRow]":
+def _team_detail_db(repo: TeamRepository) -> "TableActions[_TeamDetailRow]":
     return repo.table
 
 
diff --git a/litellm/proxy/management_helpers/object_permission_utils.py b/litellm/proxy/management_helpers/object_permission_utils.py
index 33b84545915..fb64914f6f5 100644
--- a/litellm/proxy/management_helpers/object_permission_utils.py
+++ b/litellm/proxy/management_helpers/object_permission_utils.py
@@ -4,7 +4,7 @@ organizations, teams, and keys.
 """
 
 import json
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
 from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any, Final, Optional
 
@@ -19,6 +19,8 @@ from litellm.repositories.object_permission_repository import ObjectPermissionRe
 from litellm.repositories.table_repositories import MCPServerRepository
 
 if TYPE_CHECKING:
+    from prisma import models as prisma_models
+
     from litellm.proxy._types import (
         LiteLLM_ObjectPermissionTable,
         LiteLLM_TeamTableCachedObj,
@@ -26,7 +28,7 @@ if TYPE_CHECKING:
 
 
 async def attach_object_permission_to_dict(
-    data_dict: dict,
+    data_dict: dict[str, object],
     prisma_client: PrismaClient,
 ) -> dict:
     """
@@ -61,7 +63,7 @@ async def attach_object_permission_to_dict(
             try:
                 object_permission = object_permission.model_dump()
             except Exception:
-                object_permission = object_permission.dict()
+                object_permission = object_permission.dict()  # pyright: ignore[reportDeprecated]  # pydantic v1 fallback
             data_dict["object_permission"] = object_permission
     return data_dict
 
@@ -188,7 +190,9 @@ async def _set_object_permission(
         return data_json
 
     # Clean data: exclude None values and object_permission_id
-    clean_data: Final = {k: v for k, v in permission_data.items() if v is not None and k != "object_permission_id"}
+    clean_data: Final[dict[str, object]] = {
+        k: v for k, v in permission_data.items() if v is not None and k != "object_permission_id"
+    }
 
     # Serialize mcp_tool_permissions to JSON string for GraphQL compatibility
     if "mcp_tool_permissions" in clean_data:
@@ -224,7 +228,7 @@ def _mcp_server_identifier_matches(server: Any, identifier: str) -> bool:
 async def _get_db_mcp_servers_by_identifiers(
     identifiers: set[str],
     prisma_client: PrismaClient | None,
-) -> list[Any]:
+) -> "Sequence[prisma_models.LiteLLM_MCPServerTable]":
     if prisma_client is None or not identifiers:
         return []
 
diff --git a/litellm/proxy/management_helpers/utils.py b/litellm/proxy/management_helpers/utils.py
index cb30ce90c7f..229e4fdb9e7 100644
--- a/litellm/proxy/management_helpers/utils.py
+++ b/litellm/proxy/management_helpers/utils.py
@@ -86,6 +86,20 @@ class _PrismaTeamMembershipTable(Protocol):
     async def create(self, *, data: Mapping[str, object], include: Mapping[str, bool]) -> _PrismaRecord: ...
 
 
+def _user_table(prisma_client: PrismaClient) -> _PrismaUserTable:
+    table: Final[_PrismaUserTable] = UserRepository(prisma_client).table
+    return table
+
+
+async def _find_users_by_email(prisma_client: PrismaClient, user_email: str) -> Sequence[_PrismaUserRecord] | None:
+    rows: Final[Sequence[_PrismaUserRecord] | None] = await prisma_client.get_data(
+        key_val={"user_email": user_email},
+        table_name="user",
+        query_type="find_all",
+    )
+    return rows
+
+
 def get_new_internal_user_defaults(user_id: str, user_email: str | None = None) -> dict[str, object]:
     user_info: Final = litellm.default_internal_user_params or {}
 
@@ -309,8 +323,7 @@ async def _append_team_id_if_absent(prisma_client: PrismaClient, user_id: str, t
     number of teams a user belongs to). Teams added concurrently for a different
     team id are unaffected, since each update filters on its own team id.
     """
-    user_table: Final[_PrismaUserTable] = UserRepository(prisma_client).table
-    await user_table.update_many(
+    await _user_table(prisma_client).update_many(
         where={"user_id": user_id, "NOT": {"teams": {"has": team_id}}},
         data={"teams": {"push": [team_id]}},
     )
@@ -348,8 +361,7 @@ async def add_new_member(
         # Prisma only compiles an upsert down to INSERT ... ON CONFLICT when it
         # is non-empty, and falls back to a racy SELECT-then-INSERT when it is
         # not, so this re-states user_id as a no-op rather than being empty.
-        user_table: Final[_PrismaUserTable] = UserRepository(prisma_client).table
-        _returned_user: _PrismaUserRecord | None = await user_table.upsert(
+        _returned_user: _PrismaRecord | None = await _user_table(prisma_client).upsert(
             where={"user_id": new_member.user_id},
             data={
                 "create": {"teams": [team_id], **new_user_defaults},
@@ -363,11 +375,7 @@ async def add_new_member(
         new_user_defaults = get_new_internal_user_defaults(user_id=str(uuid.uuid4()), user_email=new_member.user_email)
         ## user email is not unique acc. to prisma schema -> future improvement
         ### for now: check if it exists in db, if not - insert it
-        existing_user_row: Final[list[_PrismaUserRecord] | None] = await prisma_client.get_data(
-            key_val={"user_email": new_member.user_email},
-            table_name="user",
-            query_type="find_all",
-        )
+        existing_user_row: Final = await _find_users_by_email(prisma_client, new_member.user_email)
         if existing_user_row is None or (isinstance(existing_user_row, list) and len(existing_user_row) == 0):
             new_user_defaults["teams"] = [team_id]
             _returned_user = await prisma_client.insert_data(data=new_user_defaults, table_name="user")
diff --git a/litellm/proxy/memory/memory_endpoints.py b/litellm/proxy/memory/memory_endpoints.py
index 3ae8dcf64b7..193f7e09f07 100644
--- a/litellm/proxy/memory/memory_endpoints.py
+++ b/litellm/proxy/memory/memory_endpoints.py
@@ -18,21 +18,20 @@ Scoping:
 """
 
 import json
-from collections.abc import Mapping, Sequence
-from datetime import datetime
-from typing import TYPE_CHECKING, Final, Protocol
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Final
 
 from fastapi import APIRouter, Depends, HTTPException, Query
 
 from litellm._logging import verbose_proxy_logger
 from litellm.proxy._types import (
     CommonProxyErrors,
-    LiteLLM_TeamTable,
     LitellmUserRoles,
     UserAPIKeyAuth,
     user_api_key_has_admin_view,
 )
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.repositories.prisma_protocols import TableActions
 from litellm.repositories.table_repositories import MemoryRepository
 from litellm.repositories.team_repository import TeamRepository
 from litellm.types.memory_management import (
@@ -44,54 +43,17 @@ from litellm.types.memory_management import (
 )
 
 if TYPE_CHECKING:
+    from prisma import models as prisma_models
+
     from litellm.proxy.utils import PrismaClient
 
 router: Final = APIRouter()
 
 
-class _MemoryRecord(Protocol):
-    memory_id: str
-    key: str
-    value: str
-    metadata: object
-    user_id: str | None
-    team_id: str | None
-    created_at: datetime | None
-    created_by: str | None
-    updated_at: datetime | None
-    updated_by: str | None
-
-
-class _MemoryTableActions(Protocol):
-    async def create(self, data: Mapping[str, object]) -> _MemoryRecord: ...
-
-    async def find_many(
-        self,
-        where: Mapping[str, object] | None = ...,
-        order: Mapping[str, str] | None = ...,
-        skip: int = ...,
-        take: int = ...,
-    ) -> Sequence[_MemoryRecord]: ...
-
-    async def count(self, where: Mapping[str, object] | None = ...) -> int: ...
-
-    async def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> _MemoryRecord: ...
-
-    async def delete(self, where: Mapping[str, object]) -> _MemoryRecord | None: ...
-
-
-def _memory_table(prisma_client: "PrismaClient") -> _MemoryTableActions:
+def _memory_table(prisma_client: "PrismaClient") -> TableActions["prisma_models.LiteLLM_MemoryTable"]:
     return MemoryRepository(prisma_client).table
 
 
-class _TeamTableActions(Protocol):
-    async def find_unique(self, where: Mapping[str, str]) -> LiteLLM_TeamTable | None: ...
-
-
-def _team_table(prisma_client: "PrismaClient") -> _TeamTableActions:
-    return TeamRepository(prisma_client).table
-
-
 def _serialize_metadata_for_prisma(metadata: object) -> str:
     """
     Encode a `metadata` payload for the `Json?` column.
@@ -129,7 +91,7 @@ def _visibility_filter(user_api_key_dict: UserAPIKeyAuth) -> Mapping[str, object
     return {"OR": ors}
 
 
-def _row_to_model(row: _MemoryRecord) -> LiteLLM_MemoryRow:
+def _row_to_model(row: "prisma_models.LiteLLM_MemoryTable") -> LiteLLM_MemoryRow:
     return LiteLLM_MemoryRow(
         memory_id=row.memory_id,
         key=row.key,
@@ -163,7 +125,7 @@ def _internal_error(log_message: str, exc: Exception, default_detail: str) -> HT
 
 
 async def _assert_write_access(
-    prisma_client: "PrismaClient", row: _MemoryRecord, user_api_key_dict: UserAPIKeyAuth
+    prisma_client: "PrismaClient", row: "prisma_models.LiteLLM_MemoryTable", user_api_key_dict: UserAPIKeyAuth
 ) -> None:
     """
     Enforce ownership for mutations (PUT/DELETE).
@@ -219,7 +181,7 @@ async def _is_team_admin_for(prisma_client: "PrismaClient", user_api_key_dict: U
     )
 
     try:
-        team_obj: Final = await _team_table(prisma_client).find_unique(where={"team_id": team_id})
+        team_obj: Final = await TeamRepository(prisma_client).find_by_id(team_id, id_field="team_id")
     except Exception as e:
         verbose_proxy_logger.exception("Error loading team for write-auth check (team_id=%s): %s", team_id, e)
         return False
@@ -407,7 +369,7 @@ async def list_memory(
 
 async def _find_memory_for_caller(
     prisma_client: "PrismaClient", key: str, user_api_key_dict: UserAPIKeyAuth
-) -> _MemoryRecord:
+) -> "prisma_models.LiteLLM_MemoryTable":
     """Look up a memory row by key, scoped to the caller's visibility."""
     key_filter: Final[Mapping[str, object]] = {"key": key}
     vis: Final = _visibility_filter(user_api_key_dict)
@@ -418,6 +380,18 @@ async def _find_memory_for_caller(
     return rows[0]
 
 
+async def _find_visible_memory_or_none(
+    prisma_client: "PrismaClient", key: str, user_api_key_dict: UserAPIKeyAuth
+) -> "prisma_models.LiteLLM_MemoryTable | None":
+    """The caller-visible row for `key`, or None when nothing is visible to them."""
+    try:
+        return await _find_memory_for_caller(prisma_client, key, user_api_key_dict)
+    except HTTPException as e:
+        if e.status_code == 404:
+            return None
+        raise
+
+
 @router.get(
     "/v1/memory/{key:path}",
     tags=["memory management"],
@@ -480,17 +454,8 @@ async def upsert_memory(
         )
     data["updated_by"] = user_api_key_dict.user_id
 
-    async def _find_existing() -> _MemoryRecord | None:
-        """Return the caller-visible row for `key`, or None."""
-        try:
-            return await _find_memory_for_caller(prisma_client, key, user_api_key_dict)
-        except HTTPException as e:
-            if e.status_code == 404:
-                return None
-            raise
-
     try:
-        existing: Final = await _find_existing()
+        existing: Final = await _find_visible_memory_or_none(prisma_client, key, user_api_key_dict)
         if existing is not None:
             # Visibility != write authority. Make sure the caller actually
             # owns this row (their user_id matches, or it's a pure team row in
@@ -530,7 +495,7 @@ async def upsert_memory(
                 # instead of surfacing a 500 on a unique-violation.
                 if not _is_unique_violation(e):
                     raise
-                existing_after_race: Final = await _find_existing()
+                existing_after_race: Final = await _find_visible_memory_or_none(prisma_client, key, user_api_key_dict)
                 if existing_after_race is None:
                     # Row exists globally but isn't visible to this caller
                     # (owned by someone else). Treat as conflict.
@@ -549,6 +514,8 @@ async def upsert_memory(
     except Exception as e:
         raise _internal_error("Error upserting memory: %s", e, "Internal error updating memory entry.")
 
+    if row is None:
+        raise HTTPException(status_code=404, detail=f"Memory with key '{key}' not found")
     return _row_to_model(row)
 
 
diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py
index 142aced4a38..2c8b926dc93 100644
--- a/litellm/proxy/openai_files_endpoints/common_utils.py
+++ b/litellm/proxy/openai_files_endpoints/common_utils.py
@@ -4,7 +4,16 @@ import re
 from collections.abc import Mapping
 from dataclasses import dataclass, field
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Final, Literal, Optional, Protocol, get_args, runtime_checkable
+from typing import (
+    TYPE_CHECKING,
+    Final,
+    Literal,
+    Optional,
+    Protocol,
+    cast,  # noqa: TID251  # prisma types Json columns as fields.Json but de-serializes them to plain python on read
+    get_args,
+    runtime_checkable,
+)
 
 from litellm.proxy._types import ProxyException
 from litellm.repositories.table_repositories import (
@@ -1183,7 +1192,7 @@ async def ensure_batch_response_managed_file_ids(
     prisma_client,
     verbose_proxy_logger,
     user_api_key_dict=None,
-    db_batch_object=None,
+    db_batch_object: "LiteLLM_ManagedObjectTable | None" = None,
     unified_batch_id: str | Literal[False] | None = None,
 ) -> None:
     """Normalize batch file IDs to managed unified IDs before DB persistence."""
@@ -1270,11 +1279,10 @@ async def get_batch_from_database(
             return None, None
 
         # Parse the batch object from database
-        batch_data: Final = (
-            json.loads(db_batch_object.file_object)
-            if isinstance(db_batch_object.file_object, str)
-            else db_batch_object.file_object
+        file_object: Final = cast(  # cast-ok: prisma types the Json column as str; reads return the decoded value
+            "Mapping[str, object] | str", db_batch_object.file_object
         )
+        batch_data: Final = json.loads(file_object) if isinstance(file_object, str) else file_object
         response: Final = LiteLLMBatch.model_validate(batch_data)
         response.id = batch_id
 
@@ -1360,7 +1368,7 @@ async def update_batch_in_database(
     managed_files_obj,
     prisma_client,
     verbose_proxy_logger,
-    db_batch_object=None,
+    db_batch_object: "LiteLLM_ManagedObjectTable | None" = None,
     operation: str = "update",
     user_api_key_dict=None,
     poller_owns_accounting: bool | None = None,
@@ -1427,7 +1435,7 @@ async def update_batch_in_database(
         # Normalize status for database storage
         db_status: Final = response.status if response.status != "completed" else "complete"
 
-        update_data: Final[dict] = {
+        update_data: Final[dict[str, object]] = {
             "status": db_status,
             "file_object": response.model_dump_json(),
             "updated_at": litellm.utils.get_utc_datetime(),
diff --git a/litellm/proxy/pass_through_endpoints/managed_id_rewriter.py b/litellm/proxy/pass_through_endpoints/managed_id_rewriter.py
index e08d277788f..4de6ef04d76 100644
--- a/litellm/proxy/pass_through_endpoints/managed_id_rewriter.py
+++ b/litellm/proxy/pass_through_endpoints/managed_id_rewriter.py
@@ -33,7 +33,13 @@ from __future__ import annotations
 import json
 import re
 from collections.abc import Callable, Mapping, Sequence
-from typing import TYPE_CHECKING, Final, TypeVar, overload
+from typing import (
+    TYPE_CHECKING,
+    Final,
+    TypeVar,
+    cast,  # noqa: TID251  # prisma stubs type Json columns as fields.Json but de-serialize them on read
+    overload,
+)
 from urllib.parse import quote, unquote
 
 from fastapi import HTTPException
@@ -286,11 +292,15 @@ def _canonical_path(route: str) -> str:
 
 
 def _file_table(prisma_client: PrismaClient) -> ManagedFileTable:
-    return ManagedFileRepository(prisma_client).table
+    return cast(  # cast-ok: stub-only mismatch, prisma returns real lists and de-serialized Json
+        ManagedFileTable, ManagedFileRepository(prisma_client).table
+    )
 
 
 def _object_table(prisma_client: PrismaClient) -> ManagedObjectTable:
-    return ManagedObjectRepository(prisma_client).table
+    return cast(  # cast-ok: stub-only mismatch, prisma returns real lists and de-serialized Json
+        ManagedObjectTable, ManagedObjectRepository(prisma_client).table
+    )
 
 
 async def _resolve_one(
diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
index 3d721dead4d..60b85cb42d0 100644
--- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
+++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
@@ -5,7 +5,7 @@ import json
 import posixpath
 import traceback
 from base64 import b64encode
-from collections.abc import AsyncGenerator, Callable, Iterable, Mapping
+from collections.abc import AsyncGenerator, Callable, Iterable, Mapping, Sequence
 from datetime import datetime
 from itertools import groupby
 from typing import Any, Final, TypedDict, cast
@@ -3183,13 +3183,18 @@ async def _filter_endpoints_by_team_allowed_routes(
         )
 
     # retrieve team metadata
-    team_metadata: Final = team.metadata
+    team_metadata: Final = cast(  # cast-ok: prisma types the Json column as str; reads hand back the decoded value
+        "Mapping[str, object] | None", team.metadata
+    )
     if team_metadata is not None and team_metadata.get("allowed_passthrough_routes") is not None:
         ## FILTER pass_through_endpoints by allowed_passthrough_routes
         pass_through_endpoints = [
             endpoint
             for endpoint in pass_through_endpoints
-            if endpoint.path in team_metadata.get("allowed_passthrough_routes")
+            if endpoint.path
+            in cast(  # cast-ok: guarded above; team metadata stores this key as a list of route paths
+                "Sequence[str]", team_metadata.get("allowed_passthrough_routes")
+            )
         ]
 
     return pass_through_endpoints
diff --git a/litellm/proxy/policy_engine/policy_registry.py b/litellm/proxy/policy_engine/policy_registry.py
index 6d7f651b9b4..f55eb4f7863 100644
--- a/litellm/proxy/policy_engine/policy_registry.py
+++ b/litellm/proxy/policy_engine/policy_registry.py
@@ -10,9 +10,20 @@ by policy_attachments (see AttachmentRegistry).
 import json
 from collections.abc import Mapping, Sequence
 from datetime import datetime, timezone
-from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, TypedDict, Union
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Final,
+    Literal,
+    Optional,
+    Protocol,
+    TypedDict,
+    Union,
+    cast,  # noqa: TID251  # prisma types the condition/pipeline Json columns as str, but reads return decoded values
+)
 
 from litellm._logging import verbose_proxy_logger
+from litellm.repositories.prisma_protocols import TableActions
 from litellm.repositories.table_repositories import PolicyRepository
 from litellm.types.proxy.policy_engine import (
     GuardrailPipeline,
@@ -65,15 +76,32 @@ class _PolicyRow(Protocol):
 
 
 class _PolicyVersionSourceRow(Protocol):
-    policy_id: str
-    policy_name: str
-    version_number: int
-    inherit: str | None
-    description: str | None
-    guardrails_add: Sequence[str] | None
-    guardrails_remove: Sequence[str] | None
-    condition: Mapping[str, object] | str | None
-    pipeline: Mapping[str, object] | str | None
+    @property
+    def policy_id(self) -> str: ...
+
+    @property
+    def policy_name(self) -> str: ...
+
+    @property
+    def version_number(self) -> int: ...
+
+    @property
+    def inherit(self) -> str | None: ...
+
+    @property
+    def description(self) -> str | None: ...
+
+    @property
+    def guardrails_add(self) -> Sequence[str] | None: ...
+
+    @property
+    def guardrails_remove(self) -> Sequence[str] | None: ...
+
+    @property
+    def condition(self) -> Mapping[str, object] | str | None: ...
+
+    @property
+    def pipeline(self) -> Mapping[str, object] | str | None: ...
 
 
 class _PolicyTableClient(Protocol):
@@ -96,23 +124,15 @@ class _PolicyTableClient(Protocol):
     async def delete_many(self, where: Mapping[str, object]) -> int: ...
 
 
-class _PolicyVersionSourceTableClient(Protocol):
-    async def find_unique(self, where: Mapping[str, object]) -> _PolicyVersionSourceRow | None: ...
-
-    async def find_first(
-        self,
-        where: Mapping[str, object],
-        order: Mapping[str, str] | None = None,
-    ) -> _PolicyVersionSourceRow | None: ...
-
-
 def _policy_table(prisma_client: "PrismaClient") -> _PolicyTableClient:
-    table: Final[_PolicyTableClient] = PolicyRepository(prisma_client).table
-    return table
+    table: Final = PolicyRepository(prisma_client).table
+    return cast(  # cast-ok: prisma types Json columns as str; the client hands back the decoded condition/pipeline
+        "_PolicyTableClient", table
+    )
 
 
-def _policy_version_source_table(prisma_client: "PrismaClient") -> _PolicyVersionSourceTableClient:
-    table: Final[_PolicyVersionSourceTableClient] = PolicyRepository(prisma_client).table
+def _policy_version_source_table(prisma_client: "PrismaClient") -> "TableActions[_PolicyVersionSourceRow]":
+    table: Final[TableActions[_PolicyVersionSourceRow]] = PolicyRepository(prisma_client).table
     return table
 
 
diff --git a/litellm/proxy/policy_engine/policy_resolve_endpoints.py b/litellm/proxy/policy_engine/policy_resolve_endpoints.py
index 346586c1e5a..611772dfbae 100644
--- a/litellm/proxy/policy_engine/policy_resolve_endpoints.py
+++ b/litellm/proxy/policy_engine/policy_resolve_endpoints.py
@@ -6,7 +6,8 @@ Policy resolve and attachment impact estimation endpoints.
 """
 
 import json
-from typing import Final
+from collections.abc import Sequence
+from typing import TYPE_CHECKING, Final
 
 from fastapi import APIRouter, Depends, HTTPException, Query
 
@@ -30,25 +31,28 @@ from litellm.types.proxy.policy_engine import (
     PolicyResolveResponse,
 )
 
+if TYPE_CHECKING:
+    from prisma import models as prisma_models
+
 router: Final = APIRouter()
 
 
-def _build_alias_where(field: str, patterns: list) -> dict:
+def _build_alias_where(field: str, patterns: Sequence[str]) -> dict[str, object]:
     """Build a Prisma ``where`` clause for alias patterns.
 
     Supports exact matches and suffix wildcards (``prefix*``).
     Returns something like:
         {"OR": [{"field": {"in": ["a","b"]}}, {"field": {"startsWith": "dev-"}}]}
     """
-    exact: Final[list] = []
-    prefix_conditions: Final[list] = []
+    exact: Final[list[str]] = []
+    prefix_conditions: Final[list[dict[str, object]]] = []
     for pat in patterns:
         if pat.endswith("*"):
             prefix_conditions.append({field: {"startsWith": pat[:-1]}})
         else:
             exact.append(pat)
 
-    conditions: Final[list] = []
+    conditions: Final[list[dict[str, object]]] = []
     if exact:
         conditions.append({field: {"in": exact}})
     conditions.extend(prefix_conditions)
@@ -79,7 +83,7 @@ def _get_tags_from_metadata(metadata: object, json_metadata: object = None) -> l
     return parsed.get("tags", []) or []
 
 
-async def _fetch_all_teams(prisma_client: object) -> list:
+async def _fetch_all_teams(prisma_client: object) -> "Sequence[prisma_models.LiteLLM_TeamTable]":
     """Fetch teams from DB once. Reuse the result across tag and alias lookups."""
     return await TeamRepository(prisma_client).table.find_many(
         where={},
@@ -88,13 +92,15 @@ async def _fetch_all_teams(prisma_client: object) -> list:
     )
 
 
-def _filter_keys_by_tags(keys: list, tag_patterns: list) -> tuple:
+def _filter_keys_by_tags(
+    keys: "Sequence[prisma_models.LiteLLM_VerificationToken]", tag_patterns: Sequence[str]
+) -> tuple[list[str], int]:
     """Filter key rows whose metadata.tags match any of the given patterns.
 
     Returns (named_aliases, unnamed_count).
     """
 
-    affected: Final[list] = []
+    affected: Final[list[str]] = []
     unnamed_count = 0
     for key in keys:
         key_alias = key.key_alias or ""
@@ -111,13 +117,15 @@ def _filter_keys_by_tags(keys: list, tag_patterns: list) -> tuple:
     return affected, unnamed_count
 
 
-def _filter_teams_by_tags(teams: list, tag_patterns: list) -> tuple:
+def _filter_teams_by_tags(
+    teams: "Sequence[prisma_models.LiteLLM_TeamTable]", tag_patterns: Sequence[str]
+) -> tuple[list[str], int]:
     """Filter pre-fetched team rows whose metadata.tags match any patterns.
 
     Returns (named_aliases, unnamed_count).
     """
 
-    affected: Final[list] = []
+    affected: Final[list[str]] = []
     unnamed_count = 0
     for team in teams:
         team_alias = team.team_alias or ""
@@ -136,18 +144,18 @@ def _filter_teams_by_tags(teams: list, tag_patterns: list) -> tuple:
 
 async def _find_affected_by_team_patterns(
     prisma_client: object,
-    all_teams: list,
-    team_patterns: list,
-    existing_teams: list,
-    existing_keys: list,
-) -> tuple:
+    all_teams: "Sequence[prisma_models.LiteLLM_TeamTable]",
+    team_patterns: Sequence[str],
+    existing_teams: Sequence[str],
+    existing_keys: Sequence[str],
+) -> tuple[list[str], list[str], int]:
     """Filter pre-fetched teams by alias patterns, then fetch their keys.
 
     Returns (new_teams, new_keys, unnamed_keys_count).
     """
 
-    new_teams: Final[list] = []
-    matched_team_ids: Final[list] = []
+    new_teams: Final[list[str]] = []
+    matched_team_ids: Final[list[str]] = []
 
     for team in all_teams:
         team_alias = team.team_alias or ""
@@ -158,7 +166,7 @@ async def _find_affected_by_team_patterns(
                 new_teams.append(team_alias)
                 matched_team_ids.append(str(team.team_id))
 
-    new_keys: Final[list] = []
+    new_keys: Final[list[str]] = []
     unnamed_keys_count = 0
     if matched_team_ids:
         keys: Final = await VerificationTokenRepository(prisma_client).table.find_many(
@@ -177,10 +185,12 @@ async def _find_affected_by_team_patterns(
     return new_teams, new_keys, unnamed_keys_count
 
 
-async def _find_affected_keys_by_alias(prisma_client: object, key_patterns: list, existing_keys: list) -> list:
+async def _find_affected_keys_by_alias(
+    prisma_client: object, key_patterns: Sequence[str], existing_keys: Sequence[str]
+) -> list[str]:
     """Find keys whose alias matches the given patterns."""
 
-    affected: Final[list] = []
+    affected: Final[list[str]] = []
 
     keys: Final = await VerificationTokenRepository(prisma_client).table.find_many(
         where=_build_alias_where("key_alias", key_patterns),
@@ -349,8 +359,8 @@ async def estimate_attachment_impact(
                 sample_teams=["(global scope — affects all teams)"],
             )
 
-        affected_keys: list = []
-        affected_teams: list = []
+        affected_keys: list[str] = []
+        affected_teams: list[str] = []
         unnamed_keys = 0
         unnamed_teams = 0
 
@@ -358,7 +368,7 @@ async def estimate_attachment_impact(
         team_patterns: Final = request.teams or []
 
         # Fetch teams once — reused by both tag-based and alias-based lookups
-        all_teams: list = []
+        all_teams: Sequence[prisma_models.LiteLLM_TeamTable] = []
         if tag_patterns or team_patterns:
             all_teams = await _fetch_all_teams(prisma_client)
 
diff --git a/litellm/proxy/prompts/prompt_endpoints.py b/litellm/proxy/prompts/prompt_endpoints.py
index 1d71ea658e4..a289ed7cbfb 100644
--- a/litellm/proxy/prompts/prompt_endpoints.py
+++ b/litellm/proxy/prompts/prompt_endpoints.py
@@ -93,7 +93,7 @@ class _PromptTableActions(Protocol):
 
     def create(self, *, data: Mapping[str, str | int | None]) -> Awaitable[_PromptRow]: ...
 
-    def update(self, *, where: Mapping[str, str | int], data: Mapping[str, str]) -> Awaitable[_PromptRow]: ...
+    def update(self, *, where: Mapping[str, str | int], data: Mapping[str, str]) -> Awaitable[_PromptRow | None]: ...
 
     def delete_many(self, *, where: Mapping[str, str]) -> Awaitable[int]: ...
 
@@ -1157,6 +1157,12 @@ async def patch_prompt(
             data=update_data,
         )
 
+        if updated_prompt_db_entry is None:
+            raise HTTPException(
+                status_code=404,
+                detail=f"Prompt with ID {base_prompt_id} not found in environment {env}",
+            )
+
         updated_prompt_spec: Final = create_versioned_prompt_spec(db_prompt=updated_prompt_db_entry)
 
         return _reload_prompt_in_registry(IN_MEMORY_PROMPT_REGISTRY, versioned_id, updated_prompt_spec)
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 7dced4e26b6..765c201953d 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -133,6 +133,7 @@ if TYPE_CHECKING:
     from aiohttp import ClientSession
     from fastapi.routing import APIRoute
     from opentelemetry.trace import Span as _Span
+    from prisma import models as prisma_models
 
     from litellm.integrations.opentelemetry import OpenTelemetry
 
@@ -634,6 +635,7 @@ from litellm.proxy.utils import (
 from litellm.proxy.video_endpoints.endpoints import router as video_router
 from litellm.repositories.base_repository import SupportsModelDump
 from litellm.repositories.credentials_repository import CredentialsRepository
+from litellm.repositories.prisma_protocols import TableActions
 from litellm.router import (
     AssistantsTypedDict,
     Deployment,
@@ -1642,12 +1644,21 @@ class _InvitationLinkRow(Protocol):
 class _UserTableRow(Protocol):
     user_id: str
     user_email: str | None
-    user_role: str
+    user_role: str | None
 
 
-class _ModelTableRow(Protocol):
-    model_id: str | None
-    created_by: str | None
+class _UserTeamsRow(Protocol):
+    @property
+    def teams(self) -> Sequence[str]: ...
+
+
+_ProxyModelRow: TypeAlias = "prisma_models.LiteLLM_ProxyModelTable"
+
+
+def _config_param_table(client: PrismaClient | None) -> TableActions[_ConfigParamRow]:
+    return cast(  # cast-ok: this is prisma's LiteLLM_Config actions object, which parses its Json column to a mapping
+        "TableActions[_ConfigParamRow]", ConfigRepository(client).table
+    )
 
 
 class _TTFTRow(TypedDict):
@@ -4370,7 +4381,7 @@ class ProxyConfig:
         if prisma_client is None or not (general_settings.get("store_model_in_db", False) is True or store_model_in_db):
             return
 
-        row: Final[_ConfigParamRow | None] = await ConfigRepository(prisma_client).table.find_first(
+        row: Final[_ConfigParamRow | None] = await _config_param_table(prisma_client).find_first(
             where={"param_name": "environment_variables"}
         )
         existing: Final[dict] = dict(row.param_value) if row is not None and row.param_value is not None else {}
@@ -6226,7 +6237,7 @@ class ProxyConfig:
         4. Update router settings
         """
         if llm_router is not None and prisma_client is not None:
-            db_router_settings: Final[_ConfigParamRow | None] = await ConfigRepository(prisma_client).table.find_first(
+            db_router_settings: Final[_ConfigParamRow | None] = await _config_param_table(prisma_client).find_first(
                 where={"param_name": "router_settings"}
             )
 
@@ -6654,7 +6665,7 @@ class ProxyConfig:
     def _should_load_db_object(self, object_type: str | SupportedDBObjectType) -> bool:
         return should_load_db_object(object_type=object_type)
 
-    async def _get_models_from_db(self, prisma_client: PrismaClient) -> list | None:
+    async def _get_models_from_db(self, prisma_client: PrismaClient) -> Sequence[_ProxyModelRow] | None:
         """
         Fetch all model deployments from the DB.
 
@@ -6664,7 +6675,7 @@ class ProxyConfig:
           as "all models deleted" and must not evict existing router deployments.
         """
         try:
-            new_models: Final[list[_ModelTableRow]] = await ModelRepository(prisma_client).table.find_many()
+            new_models: Final[Sequence[_ProxyModelRow]] = await ModelRepository(prisma_client).table.find_many()
             return new_models
         except Exception as e:
             verbose_proxy_logger.exception(
@@ -6950,10 +6961,13 @@ class ProxyConfig:
         """
 
         try:
-            sso_settings: Final[_SSOConfigRow | None] = await call_with_db_reconnect_retry(
-                prisma_client,
-                lambda: SSOConfigRepository(prisma_client).table.find_unique(where={"id": "sso_config"}),
-                reason="init_sso_settings_in_db_lookup_failure",
+            sso_settings: Final[_SSOConfigRow | None] = cast(  # cast-ok: prisma Json stub is `str`, runtime is a dict
+                "_SSOConfigRow | None",
+                await call_with_db_reconnect_retry(
+                    prisma_client,
+                    lambda: SSOConfigRepository(prisma_client).table.find_unique(where={"id": "sso_config"}),
+                    reason="init_sso_settings_in_db_lookup_failure",
+                ),
             )
             if sso_settings is not None:
                 sso_settings.sso_settings.pop("role_mappings", None)
@@ -6981,12 +6995,15 @@ class ProxyConfig:
         )
 
         try:
-            db_record: Final[_ConfigOverridesRow | None] = await call_with_db_reconnect_retry(
-                prisma_client,
-                lambda: ConfigOverridesRepository(prisma_client).table.find_unique(
-                    where={"config_type": "hashicorp_vault"}
+            db_record: Final[_ConfigOverridesRow | None] = cast(  # cast-ok: prisma Json stub is `str`, runtime dict
+                "_ConfigOverridesRow | None",
+                await call_with_db_reconnect_retry(
+                    prisma_client,
+                    lambda: ConfigOverridesRepository(prisma_client).table.find_unique(
+                        where={"config_type": "hashicorp_vault"}
+                    ),
+                    reason="init_hashicorp_vault_config_override_lookup_failure",
                 ),
-                reason="init_hashicorp_vault_config_override_lookup_failure",
             )
 
             if db_record is None or db_record.config_value is None:
@@ -8834,8 +8851,9 @@ class ProxyStartupEvent:
 
             if prisma_client is None:
                 return
-            db_record: Final[_UISettingsRow | None] = await UISettingsRepository(prisma_client).table.find_unique(
-                where={"id": "ui_settings"}
+            db_record: Final[_UISettingsRow | None] = cast(  # cast-ok: prisma Json stub is `str`, runtime is a dict
+                "_UISettingsRow | None",
+                await UISettingsRepository(prisma_client).table.find_unique(where={"id": "ui_settings"}),
             )
             if db_record and db_record.ui_settings:
                 raw: Final = db_record.ui_settings
@@ -8998,7 +9016,7 @@ class ProxyStartupEvent:
         # but YAML config has False.
         if store_model_in_db is not True and prisma_client is not None:
             try:
-                _db_gs_record: Final[_ConfigParamRow | None] = await ConfigRepository(prisma_client).table.find_first(
+                _db_gs_record: Final[_ConfigParamRow | None] = await _config_param_table(prisma_client).find_first(
                     where={"param_name": "general_settings"}
                 )
                 if _db_gs_record is not None and isinstance(_db_gs_record.param_value, dict):
@@ -12143,14 +12161,14 @@ async def _check_if_model_is_user_added(
         id = model.get("model_info", {}).get("id", None)
         if id is None:
             continue
-        db_model: _ModelTableRow | None = await ModelRepository(prisma_client).table.find_unique(where={"model_id": id})
+        db_model: _ProxyModelRow | None = await ModelRepository(prisma_client).table.find_unique(where={"model_id": id})
         if db_model is not None:
             if db_model.created_by == user_api_key_dict.user_id:
                 filtered_models.append(model)
     return filtered_models
 
 
-def _check_if_model_is_team_model(models: list[DeploymentTypedDict], user_row: LiteLLM_UserTable) -> list[dict]:
+def _check_if_model_is_team_model(models: list[DeploymentTypedDict], user_row: _UserTeamsRow) -> list[dict]:
     """
     Check if model is a team model
 
@@ -12202,6 +12220,9 @@ async def non_admin_all_models(
         except Exception:
             raise HTTPException(status_code=400, detail={"error": "User not found"})
 
+        if user_row is None:
+            raise HTTPException(status_code=400, detail={"error": "User not found"})
+
         # Get all models that are team models, when model team_id == user_row.teams
         all_models += _check_if_model_is_team_model(
             models=llm_router.get_model_list() or [],
@@ -12630,7 +12651,7 @@ async def _fetch_db_models_for_search(
 
     db_models_total_count: Final = await ModelRepository(prisma_client).table.count(where=db_where_condition)
 
-    db_models_raw: list = []
+    db_models_raw: Sequence[_ProxyModelRow] = []
     if take_limit > 0:
         db_models_raw = await ModelRepository(prisma_client).table.find_many(
             where=db_where_condition,
@@ -13020,7 +13041,7 @@ async def _gather_team_accessible_model_ids(
     try:
         if team_object.models and SpecialModelNames.all_proxy_models.value not in team_object.models:
             _resolved_names: Final = _team_models_resolve_to_names(team_object.models, access_groups)
-            db_models: Final[Sequence[_ModelTableRow]] = await ModelRepository(prisma_client).table.find_many(
+            db_models: Final[Sequence[_ProxyModelRow]] = await ModelRepository(prisma_client).table.find_many(
                 where={"model_name": {"in": _resolved_names}}
             )
             for db_model in db_models:
@@ -14494,14 +14515,18 @@ async def alerting_settings(
         )
 
     ## get general settings from db
-    db_general_settings: Final = await ConfigRepository(prisma_client).table.find_first(
+    db_general_settings: Final = await _config_param_table(prisma_client).find_first(
         where={"param_name": "general_settings"}
     )
 
     if db_general_settings is not None and db_general_settings.param_value is not None:
         db_general_settings_dict: Final = dict(db_general_settings.param_value)
-        alerting_args_dict: dict = db_general_settings_dict.get("alerting_args", {})
-        alerting_values: list | None = db_general_settings_dict.get("alerting")
+        alerting_args_dict: dict = cast(  # cast-ok: ConfigGeneralSettings validates alerting_args as a dict on write
+            dict[str, JsonValue], db_general_settings_dict.get("alerting_args", {})
+        )
+        alerting_values: list | None = cast(  # cast-ok: ConfigGeneralSettings validates alerting as a list on write
+            list[JsonValue] | None, db_general_settings_dict.get("alerting")
+        )
     else:
         alerting_args_dict = {}
         alerting_values = None
@@ -15052,7 +15077,7 @@ async def onboarding(invite_link: str, request: Request):
         user_id=user_obj.user_id,
         key=onboarding_token,
         user_email=user_obj.user_email,
-        user_role=user_obj.user_role,
+        user_role=user_obj.user_role,  # pyright: ignore[reportArgumentType]  # nullable DB column, no unset contract
         login_method="username_password",
         premium_user=premium_user,
         auth_header_name=general_settings.get("litellm_key_header_name", "Authorization"),
@@ -15161,7 +15186,7 @@ async def _generate_onboarding_ui_session_token(user_obj: _UserTableRow) -> str:
         user_id=user_obj.user_id,
         key=key,
         user_email=user_obj.user_email,
-        user_role=user_obj.user_role,
+        user_role=user_obj.user_role,  # pyright: ignore[reportArgumentType]  # nullable DB column, no unset contract
         login_method="username_password",
         premium_user=premium_user,
         auth_header_name=general_settings.get("litellm_key_header_name", "Authorization"),
@@ -15722,7 +15747,7 @@ async def update_config(
             raise Exception("No DB Connected")
 
         async def _read_section(param_name: str) -> dict:
-            row: Final[_ConfigParamRow | None] = await ConfigRepository(prisma_client).table.find_first(
+            row: Final[_ConfigParamRow | None] = await _config_param_table(prisma_client).find_first(
                 where={"param_name": param_name}
             )
             if row is None or row.param_value is None:
@@ -15979,7 +16004,7 @@ async def update_config_general_settings(
         )
 
     ## get general settings from db
-    db_general_settings: Final = await ConfigRepository(prisma_client).table.find_first(
+    db_general_settings: Final = await _config_param_table(prisma_client).find_first(
         where={"param_name": "general_settings"}
     )
     ### update value
@@ -15997,7 +16022,7 @@ async def update_config_general_settings(
     if data.field_name == "plugins":
         field_value = _preserve_redacted_plugin_keys(field_value, general_settings.get("plugins"))
 
-    general_settings[data.field_name] = field_value
+    general_settings[data.field_name] = cast(JsonValue, field_value)  # cast-ok: ConfigGeneralSettings validated it
 
     response: Final = await ConfigRepository(prisma_client).table.upsert(
         where={"param_name": "general_settings"},
@@ -16017,7 +16042,7 @@ async def update_config_general_settings(
     )
 
     if data.field_name == "plugins":
-        register_plugins_from_config(general_settings)
+        register_plugins_from_config(cast(dict[str, object], general_settings))  # cast-ok: the callee only reads it
     _apply_ssrf_general_settings(general_settings)
 
     return response
@@ -16193,7 +16218,7 @@ async def get_config_general_settings(
         )
 
     ## get general settings from db
-    db_general_settings: Final[_ConfigParamRow | None] = await ConfigRepository(prisma_client).table.find_first(
+    db_general_settings: Final[_ConfigParamRow | None] = await _config_param_table(prisma_client).find_first(
         where={"param_name": "general_settings"}
     )
     ### pop the value
@@ -16382,7 +16407,7 @@ async def get_config_list(
     is_full_admin: Final = user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN
 
     ## get general settings from db
-    db_general_settings: Final[_ConfigParamRow | None] = await ConfigRepository(prisma_client).table.find_first(
+    db_general_settings: Final[_ConfigParamRow | None] = await _config_param_table(prisma_client).find_first(
         where={"param_name": "general_settings"}
     )
 
@@ -16478,7 +16503,7 @@ async def get_config_list(
                 )
                 return_val.append(_response_obj)
 
-    db_litellm_settings_row: Final[_ConfigParamRow | None] = await ConfigRepository(prisma_client).table.find_first(
+    db_litellm_settings_row: Final[_ConfigParamRow | None] = await _config_param_table(prisma_client).find_first(
         where={"param_name": "litellm_settings"}
     )
     db_litellm_settings: Final[dict] = (
@@ -16555,7 +16580,7 @@ async def delete_config_general_settings(
         )
 
     ## get general settings from db
-    db_general_settings: Final[_ConfigParamRow | None] = await ConfigRepository(prisma_client).table.find_first(
+    db_general_settings: Final[_ConfigParamRow | None] = await _config_param_table(prisma_client).find_first(
         where={"param_name": "general_settings"}
     )
     ### pop the value
@@ -17122,7 +17147,7 @@ async def reload_anthropic_beta_headers(
         last_anthropic_beta_headers_reload = current_time.isoformat()
 
         # Set force reload flag in database for other pods, preserving existing interval_hours
-        existing_beta_config: Final[_ConfigParamRow | None] = await ConfigRepository(prisma_client).table.find_unique(
+        existing_beta_config: Final[_ConfigParamRow | None] = await _config_param_table(prisma_client).find_unique(
             where={"param_name": "anthropic_beta_headers_reload_config"}
         )
         existing_beta_interval = None
@@ -17300,7 +17325,7 @@ async def get_anthropic_beta_headers_reload_status(
             }
 
         # Get reload configuration from database
-        config_record: Final = await ConfigRepository(prisma_client).table.find_unique(
+        config_record: Final = await _config_param_table(prisma_client).find_unique(
             where={"param_name": "anthropic_beta_headers_reload_config"}
         )
 
@@ -17314,7 +17339,9 @@ async def get_anthropic_beta_headers_reload_status(
             }
 
         config: Final = config_record.param_value
-        interval_hours: Final = config.get("interval_hours")
+        interval_hours: Final = cast(  # cast-ok: every writer of this key stores `hours: int` or an explicit None
+            int | None, config.get("interval_hours")
+        )
 
         if interval_hours is None:
             verbose_proxy_logger.info("No interval configured, returning not scheduled")
diff --git a/litellm/proxy/spend_tracking/cloudzero_endpoints.py b/litellm/proxy/spend_tracking/cloudzero_endpoints.py
index 93c81f7fc67..6c0d11a2174 100644
--- a/litellm/proxy/spend_tracking/cloudzero_endpoints.py
+++ b/litellm/proxy/spend_tracking/cloudzero_endpoints.py
@@ -1,5 +1,11 @@
 import json
-from typing import Final
+from collections.abc import Mapping
+from typing import (
+    TYPE_CHECKING,
+    Final,
+    Protocol,
+    cast,  # noqa: TID251  # the config repository's table protocol omits find_first
+)
 
 from fastapi import APIRouter, Depends, HTTPException
 
@@ -13,6 +19,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import (
 )
 from litellm.proxy.management_endpoints.common_utils import _user_has_admin_view
 from litellm.repositories.config_repository import ConfigRepository
+from litellm.repositories.prisma_protocols import TableActions
 from litellm.types.proxy.cloudzero_endpoints import (
     CloudZeroExportRequest,
     CloudZeroExportResponse,
@@ -22,6 +29,9 @@ from litellm.types.proxy.cloudzero_endpoints import (
     CloudZeroSettingsView,
 )
 
+if TYPE_CHECKING:
+    from litellm.proxy.proxy_server import PrismaClient
+
 router: Final = APIRouter()
 
 
@@ -29,6 +39,18 @@ router: Final = APIRouter()
 _sensitive_masker: Final = SensitiveDataMasker()
 
 
+class _CloudZeroConfigRow(Protocol):
+    """The ``LiteLLM_Config`` row holding ``cloudzero_settings``, as this module reads it."""
+
+    @property
+    def param_value(self) -> str | Mapping[str, str] | None: ...
+
+
+def _config_table(prisma_client: "PrismaClient") -> TableActions[_CloudZeroConfigRow]:
+    repository_table: Final = ConfigRepository(prisma_client).table
+    return cast(TableActions[_CloudZeroConfigRow], repository_table)  # cast-ok: repo protocol omits find_first
+
+
 async def _set_cloudzero_settings(api_key: str, connection_id: str, timezone: str):
     """
     Store CloudZero settings in the database with encrypted API key.
@@ -82,9 +104,7 @@ async def _get_cloudzero_settings():
             detail={"error": CommonProxyErrors.db_not_connected_error.value},
         )
 
-    cloudzero_config: Final = await ConfigRepository(prisma_client).table.find_first(
-        where={"param_name": "cloudzero_settings"}
-    )
+    cloudzero_config: Final = await _config_table(prisma_client).find_first(where={"param_name": "cloudzero_settings"})
     if cloudzero_config is None or cloudzero_config.param_value is None:
         return {}
 
@@ -268,7 +288,7 @@ async def is_cloudzero_setup_in_db() -> bool:
             return False
 
         # Check for CloudZero settings in database
-        cloudzero_config: Final = await ConfigRepository(prisma_client).table.find_first(
+        cloudzero_config: Final = await _config_table(prisma_client).find_first(
             where={"param_name": "cloudzero_settings"}
         )
 
@@ -530,7 +550,7 @@ async def delete_cloudzero_settings(
             )
 
         # Check if CloudZero settings exist
-        cloudzero_config: Final = await ConfigRepository(prisma_client).table.find_first(
+        cloudzero_config: Final = await _config_table(prisma_client).find_first(
             where={"param_name": "cloudzero_settings"}
         )
 
diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py
index ed2ecd8325a..06395a3c3cc 100644
--- a/litellm/proxy/spend_tracking/spend_management_endpoints.py
+++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py
@@ -4,10 +4,22 @@ import json
 import os
 from collections.abc import Mapping, Sequence
 from datetime import datetime, timedelta, timezone
-from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, NamedTuple, Protocol, TypedDict, TypeVar
+from typing import (
+    TYPE_CHECKING,
+    Annotated,
+    Any,
+    Final,
+    Literal,
+    NamedTuple,
+    Protocol,
+    TypedDict,
+    TypeVar,
+    cast,  # noqa: TID251  # prisma group_by returns untyped aggregate mappings
+)
 
 import fastapi
 from fastapi import APIRouter, Depends, HTTPException, Request, status
+from typing_extensions import ReadOnly
 
 import litellm
 from litellm._logging import verbose_proxy_logger
@@ -23,6 +35,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import (
     get_spend_by_team_and_customer,
 )
 from litellm.proxy.utils import handle_exception_on_proxy
+from litellm.repositories.prisma_protocols import TableActions
 from litellm.repositories.table_repositories import SpendLogsRepository
 from litellm.repositories.team_repository import TeamRepository
 from litellm.repositories.verification_token_repository import (
@@ -30,6 +43,8 @@ from litellm.repositories.verification_token_repository import (
 )
 
 if TYPE_CHECKING:
+    from prisma import models as prisma_models
+
     from litellm.proxy.proxy_server import PrismaClient
     from litellm.proxy.spend_tracking.cold_storage_handler import ColdStorageHandler
 else:
@@ -139,6 +154,18 @@ class _SessionSpendRow(TypedDict):
     mcp_tool_call_spend: float
 
 
+class _SpendSumAggregate(TypedDict, total=False):
+    spend: ReadOnly[float]
+
+
+class _SpendGroupByRow(TypedDict):
+    api_key: ReadOnly[str]
+    user: ReadOnly[str | None]
+    model: ReadOnly[str]
+    startTime: ReadOnly[object]
+    _sum: ReadOnly[_SpendSumAggregate]
+
+
 async def _query_raw(prisma_client: PrismaClient, sql_query: str, *args: object) -> Sequence[_RowT]:
     """Run a raw read query and return its rows as the row type the caller declares."""
     return await prisma_client.db.query_raw(sql_query, *args)
@@ -149,24 +176,6 @@ async def _query_raw_or_none(prisma_client: PrismaClient, sql_query: str, *args:
     return await _query_raw(prisma_client, sql_query, *args)
 
 
-class _SpendLogsTable(Protocol):
-    """The subset of the Prisma spend-logs table API this module uses."""
-
-    async def find_many(
-        self, *, where: Mapping[str, object], order: Mapping[str, str]
-    ) -> Sequence[_SupportsModelDump]: ...
-
-    async def find_unique(
-        self, *, where: Mapping[str, object], include: None = None
-    ) -> _SpendLogOwnershipRow | None: ...
-
-    async def count(self, *, where: Mapping[str, object]) -> int: ...
-
-    async def group_by(
-        self, *, by: Sequence[str], where: Mapping[str, object], count: Mapping[str, bool]
-    ) -> Sequence[_SessionCountRow]: ...
-
-
 class _TeamTable(Protocol):
     """The subset of the Prisma team table API this module uses."""
 
@@ -183,7 +192,7 @@ class _VerificationTokenTable(Protocol):
     async def update_many(self, *, data: Mapping[str, float], where: Mapping[str, object]) -> int: ...
 
 
-def _spend_logs_table(prisma_client: PrismaClient) -> _SpendLogsTable:
+def _spend_logs_table(prisma_client: PrismaClient) -> TableActions["prisma_models.LiteLLM_SpendLogs"]:
     return SpendLogsRepository(prisma_client).table
 
 
@@ -221,11 +230,12 @@ async def _count_logs_per_session(
     prisma_client: PrismaClient, session_ids: Sequence[str | None]
 ) -> Sequence[_SessionCountRow]:
     """Count spend log rows per session for the given session ids."""
-    return await _spend_logs_table(prisma_client).group_by(
+    rows: Final = await _spend_logs_table(prisma_client).group_by(
         by=["session_id"],
         where={"session_id": {"in": session_ids}},
         count={"session_id": True},
     )
+    return cast(Sequence[_SessionCountRow], rows)  # cast-ok: group_by(count=) shape is fixed by the by/count args
 
 
 async def _find_team_row(prisma_client: PrismaClient, team_id: str) -> _SupportsModelDump | None:
@@ -2974,8 +2984,9 @@ async def view_spend_logs(
             )
 
             if isinstance(response, list) and len(response) > 0 and isinstance(response[0], dict):
+                spend_rows: Final = cast(Sequence[_SpendGroupByRow], response)  # cast-ok: by/sum fix the shape
                 result: Final[dict] = {}
-                for record in response:
+                for record in spend_rows:
                     dt_object = datetime.strptime(str(record["startTime"]), "%Y-%m-%dT%H:%M:%S.%fZ")
                     date = dt_object.date()
                     if date not in result:
diff --git a/litellm/proxy/spend_tracking/vantage_endpoints.py b/litellm/proxy/spend_tracking/vantage_endpoints.py
index 00d0554d783..c71105ad283 100644
--- a/litellm/proxy/spend_tracking/vantage_endpoints.py
+++ b/litellm/proxy/spend_tracking/vantage_endpoints.py
@@ -1,5 +1,11 @@
 import json
-from typing import Final
+from collections.abc import Mapping
+from typing import (
+    TYPE_CHECKING,
+    Final,
+    Protocol,
+    cast,  # noqa: TID251  # the config repository's table protocol omits find_first
+)
 
 from fastapi import APIRouter, Depends, HTTPException
 
@@ -14,6 +20,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import (
 )
 from litellm.proxy.management_endpoints.common_utils import _user_has_admin_view
 from litellm.repositories.config_repository import ConfigRepository
+from litellm.repositories.prisma_protocols import TableActions
 from litellm.types.proxy.vantage_endpoints import (
     VantageDryRunRequest,
     VantageExportRequest,
@@ -24,6 +31,9 @@ from litellm.types.proxy.vantage_endpoints import (
     VantageSettingsView,
 )
 
+if TYPE_CHECKING:
+    from litellm.proxy.proxy_server import PrismaClient
+
 router: Final = APIRouter()
 
 _sensitive_masker: Final = SensitiveDataMasker()
@@ -31,6 +41,18 @@ _sensitive_masker: Final = SensitiveDataMasker()
 VANTAGE_SETTINGS_PARAM_NAME: Final = "vantage_settings"
 
 
+class _VantageConfigRow(Protocol):
+    """The ``LiteLLM_Config`` row holding ``vantage_settings``, as this module reads it."""
+
+    @property
+    def param_value(self) -> str | Mapping[str, str] | None: ...
+
+
+def _config_table(prisma_client: "PrismaClient") -> TableActions[_VantageConfigRow]:
+    repository_table: Final = ConfigRepository(prisma_client).table
+    return cast(TableActions[_VantageConfigRow], repository_table)  # cast-ok: repo protocol omits find_first
+
+
 def _get_registered_vantage_logger():
     """Return the VantageLogger already registered in litellm.callbacks, if any."""
     from litellm.integrations.vantage.vantage_logger import VantageLogger
@@ -82,7 +104,7 @@ async def _get_vantage_settings():
             detail={"error": CommonProxyErrors.db_not_connected_error.value},
         )
 
-    vantage_config: Final = await ConfigRepository(prisma_client).table.find_first(
+    vantage_config: Final = await _config_table(prisma_client).find_first(
         where={"param_name": VANTAGE_SETTINGS_PARAM_NAME}
     )
     if vantage_config is None or vantage_config.param_value is None:
@@ -251,7 +273,7 @@ async def is_vantage_setup_in_db() -> bool:
         if prisma_client is None:
             return False
 
-        vantage_config: Final = await ConfigRepository(prisma_client).table.find_first(
+        vantage_config: Final = await _config_table(prisma_client).find_first(
             where={"param_name": VANTAGE_SETTINGS_PARAM_NAME}
         )
 
@@ -525,7 +547,7 @@ async def delete_vantage_settings(
                 detail={"error": CommonProxyErrors.db_not_connected_error.value},
             )
 
-        vantage_config: Final = await ConfigRepository(prisma_client).table.find_first(
+        vantage_config: Final = await _config_table(prisma_client).find_first(
             where={"param_name": VANTAGE_SETTINGS_PARAM_NAME}
         )
 
diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
index 66a8c0622fa..a1eb7ed06eb 100644
--- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
+++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py
@@ -4,7 +4,12 @@ import json
 import os
 from collections import Counter
 from collections.abc import Mapping
-from typing import Any, Final, Protocol, TypeVar
+from typing import (
+    Any,
+    Final,
+    Protocol,
+    cast,  # noqa: TID251  # prisma types Json columns as fields.Json but de-serializes them to plain python on read
+)
 from urllib.parse import urlparse
 
 from fastapi import APIRouter, Body, Depends, File, HTTPException, UploadFile
@@ -25,6 +30,7 @@ from litellm.proxy.spend_tracking.ptu_feature_flag import is_ptu_cost_attributio
 from litellm.proxy.utils import invalidate_config_param
 from litellm.repositories.config_repository import ConfigRepository
 from litellm.repositories.organization_repository import OrganizationRepository
+from litellm.repositories.prisma_protocols import TableActions
 from litellm.repositories.table_repositories import (
     SSOConfigRepository,
     UISettingsRepository,
@@ -37,29 +43,16 @@ from litellm.types.proxy.management_endpoints.ui_sso import (
 
 router: Final = APIRouter()
 
-_DbRecordT: Final = TypeVar("_DbRecordT", covariant=True)
-
-
-class _PrismaTableActions(Protocol[_DbRecordT]):
-    async def find_unique(self, where: Mapping[str, object]) -> _DbRecordT | None: ...
-
-    async def update(self, where: Mapping[str, object], data: Mapping[str, object]) -> _DbRecordT: ...
-
-    async def upsert(self, where: Mapping[str, object], data: Mapping[str, object]) -> _DbRecordT: ...
-
 
 class _SsoSettingsMappingRow(Protocol):
     @property
     def sso_settings(self) -> Mapping[str, object] | None: ...
 
 
-class _HasSsoSettingsMappingTable(Protocol):
-    @property
-    def table(self) -> _PrismaTableActions[_SsoSettingsMappingRow]: ...
-
-
-def _sso_settings_mapping_db(repo: _HasSsoSettingsMappingTable) -> _PrismaTableActions[_SsoSettingsMappingRow]:
-    return repo.table
+def _sso_settings_mapping_db(repo: SSOConfigRepository) -> TableActions[_SsoSettingsMappingRow]:
+    return cast(  # cast-ok: prisma types Json columns as str; the client hands back the deserialized value
+        "TableActions[_SsoSettingsMappingRow]", repo.table
+    )
 
 
 class _StoredSsoSettingsRow(Protocol):
@@ -67,12 +60,7 @@ class _StoredSsoSettingsRow(Protocol):
     def sso_settings(self) -> object: ...
 
 
-class _HasStoredSsoSettingsTable(Protocol):
-    @property
-    def table(self) -> _PrismaTableActions[_StoredSsoSettingsRow]: ...
-
-
-def _stored_sso_settings_db(repo: _HasStoredSsoSettingsTable) -> _PrismaTableActions[_StoredSsoSettingsRow]:
+def _stored_sso_settings_db(repo: SSOConfigRepository) -> TableActions[_StoredSsoSettingsRow]:
     return repo.table
 
 
@@ -81,13 +69,10 @@ class _UiSettingsRow(Protocol):
     def ui_settings(self) -> str | Mapping[str, JsonValue] | None: ...
 
 
-class _HasUiSettingsTable(Protocol):
-    @property
-    def table(self) -> _PrismaTableActions[_UiSettingsRow]: ...
-
-
-def _ui_settings_db(repo: _HasUiSettingsTable) -> _PrismaTableActions[_UiSettingsRow]:
-    return repo.table
+def _ui_settings_db(repo: UISettingsRepository) -> TableActions[_UiSettingsRow]:
+    return cast(  # cast-ok: prisma types Json columns as str; the client hands back the deserialized value
+        "TableActions[_UiSettingsRow]", repo.table
+    )
 
 
 class _ConfigParamRow(Protocol):
@@ -95,13 +80,10 @@ class _ConfigParamRow(Protocol):
     def param_value(self) -> str | Mapping[str, object] | None: ...
 
 
-class _HasConfigParamTable(Protocol):
-    @property
-    def table(self) -> _PrismaTableActions[_ConfigParamRow]: ...
-
-
-def _config_param_db(repo: _HasConfigParamTable) -> _PrismaTableActions[_ConfigParamRow]:
-    return repo.table
+def _config_param_db(repo: ConfigRepository) -> TableActions[_ConfigParamRow]:
+    return cast(  # cast-ok: prisma's LiteLLM_Config actions object, whose Json column parses to a mapping
+        "TableActions[_ConfigParamRow]", repo.table
+    )
 
 
 # Maps each UIThemeConfig field to the env var the UI branding path reads it
diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index 86d954c0913..9f9dd4e6af9 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -177,6 +177,7 @@ from litellm.types.utils import LLMResponseTypes, LoggedLiteLLMParams
 if TYPE_CHECKING:
     from mcp.types import CallToolResult
     from opentelemetry.trace import Span as _Span
+    from prisma import models as prisma_models
     from prisma.actions import LiteLLM_DeprecatedVerificationTokenActions
     from prisma.client import TransactionManager
     from prisma.models import LiteLLM_DeprecatedVerificationToken
@@ -186,6 +187,7 @@ if TYPE_CHECKING:
     from litellm.models.team import LiteLLM_TeamTableCachedObj
     from litellm.proxy.db.autorouter_session_rollup import AutoRouterTurnTransaction
     from litellm.proxy.db.spend_log_tool_index import ToolUsageTransaction
+    from litellm.repositories.prisma_protocols import TableActions
     from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline
 
     Span = _Span | object
@@ -3266,7 +3268,10 @@ async def prefetch_config_params(prisma_client: "PrismaClient | None", param_nam
     if not param_names:
         return
     try:
-        rows: Final = await ConfigRepository(prisma_client).table.find_many(where={"param_name": {"in": param_names}})
+        config_table: Final = cast(  # cast-ok: ConfigRepository.table is prisma's litellm_config actions object
+            "TableActions[prisma_models.LiteLLM_Config]", ConfigRepository(prisma_client).table
+        )
+        rows: Final = await config_table.find_many(where={"param_name": {"in": param_names}})
     except Exception as e:
         verbose_proxy_logger.debug(
             "prefetch_config_params failed, falling through to per-param queries: %s",
@@ -3555,8 +3560,8 @@ class PrismaClient:
 
         return hashed_token
 
-    def jsonify_object(self, data: dict) -> dict:
-        db_data: Final = copy.deepcopy(data)
+    def jsonify_object(self, data: Mapping[str, object]) -> dict[str, object]:
+        db_data: Final[dict[str, object]] = copy.deepcopy(dict(data))
 
         for k, v in db_data.items():
             if isinstance(v, dict):
@@ -3690,7 +3695,10 @@ class PrismaClient:
             elif table_name == "keys":
                 return await VerificationTokenRepository(self).table.find_first(where={key: value})
             elif table_name == "config":
-                return await ConfigRepository(self).table.find_first(where={key: value})
+                config_table: Final = cast(  # cast-ok: ConfigRepository.table is prisma's litellm_config actions object
+                    "TableActions[prisma_models.LiteLLM_Config]", ConfigRepository(self).table
+                )
+                return await config_table.find_first(where={key: value})
             elif table_name == "spend":
                 return await self.db.l.find_first(where={key: value})
             return None
@@ -3793,9 +3801,9 @@ class PrismaClient:
         self,
         token: str | list | None = None,
         user_id: str | None = None,
-        user_id_list: list | None = None,
+        user_id_list: Sequence[str] | None = None,
         team_id: str | None = None,
-        team_id_list: list | None = None,
+        team_id_list: Sequence[str] | None = None,
         key_val: dict | None = None,
         table_name: Literal[
             "user", "key", "config", "spend", "enduser", "budget", "team", "user_notification", "combined_view"
@@ -3878,14 +3886,14 @@ class PrismaClient:
                             if isinstance(r.expires, datetime):
                                 r.expires = r.expires.isoformat()
                 elif query_type == "find_all":
-                    where_filter: Final[dict] = {}
+                    where_filter: Final[dict[str, dict[str, Sequence[str]]]] = {}
                     if token is not None:
                         where_filter["token"] = {}
                         if isinstance(token, str):
                             token = _hash_token_if_needed(token=token)
                             where_filter["token"]["in"] = [token]
                         elif isinstance(token, list):
-                            hashed_tokens: Final = []
+                            hashed_tokens: Final[list[str]] = []
                             for t in token:
                                 assert isinstance(t, str)
                                 if t.startswith("sk-"):
@@ -4182,7 +4190,7 @@ class PrismaClient:
             )
             raise e
 
-    def jsonify_team_object(self, db_data: dict):
+    def jsonify_team_object(self, db_data: Mapping[str, object]) -> dict[str, object]:
         db_data = self.jsonify_object(data=db_data)
         if db_data.get("members_with_roles", None) is not None and isinstance(db_data["members_with_roles"], list):
             db_data["members_with_roles"] = json.dumps(db_data["members_with_roles"])
@@ -4200,7 +4208,7 @@ class PrismaClient:
     )
     async def insert_data(
         self,
-        data: dict,
+        data: Mapping[str, object],
         table_name: Literal["user", "key", "config", "spend", "team", "user_notification"],
     ):
         """
@@ -4210,10 +4218,12 @@ class PrismaClient:
         try:
             verbose_proxy_logger.debug(
                 "PrismaClient: insert_data: %s",
-                {**data, "token": self.hash_token(token=data["token"])} if data.get("token") is not None else data,
+                {**data, "token": self.hash_token(token=cast("str", data["token"]))}  # cast-ok: a key token is a str
+                if data.get("token") is not None
+                else data,
             )
             if table_name == "key":
-                token: Final = data["token"]
+                token: Final = cast("str", data["token"])  # cast-ok: the key table's token column is a str
                 hashed_token: Final = self.hash_token(token=token)
                 db_data = self.jsonify_object(data=data)
                 db_data["token"] = hashed_token
@@ -4348,14 +4358,14 @@ class PrismaClient:
     async def update_data(
         self,
         token: str | None = None,
-        data: dict = {},
+        data: Mapping[str, object] = {},
         data_list: list | None = None,
         user_id: str | None = None,
         team_id: str | None = None,
         query_type: Literal["update", "update_many"] = "update",
         table_name: Literal["user", "key", "config", "spend", "team", "enduser", "budget"] | None = None,
-        update_key_values: dict | None = None,
-        update_key_values_custom_query: dict | None = None,
+        update_key_values: dict[str, object] | None = None,
+        update_key_values_custom_query: dict[str, object] | None = None,
     ):
         """
         Update existing data
@@ -4381,14 +4391,14 @@ class PrismaClient:
                     try:
                         _data = response.model_dump()
                     except Exception:
-                        _data = response.dict()
+                        _data = response.dict()  # pyright: ignore[reportDeprecated]  # pydantic-v1 row fallback
                 return {"token": token, "data": _data}
             elif user_id is not None or (table_name is not None and table_name == "user") and query_type == "update":
                 """
                 If data['spend'] + data['user'], update the user table with spend info as well
                 """
                 if user_id is None:
-                    user_id = db_data["user_id"]
+                    user_id = cast("str", db_data["user_id"])  # cast-ok: the user table's user_id column is a str
                 if update_key_values is None:
                     if update_key_values_custom_query is not None:
                         update_key_values = update_key_values_custom_query
@@ -4410,7 +4420,7 @@ class PrismaClient:
                 If data['spend'] + data['user'], update the user table with spend info as well
                 """
                 if team_id is None:
-                    team_id = db_data["team_id"]
+                    team_id = cast("str | None", db_data["team_id"])  # cast-ok: team_id column is a nullable str
                 if update_key_values is None:
                     update_key_values = db_data
                 if "team_id" not in db_data and team_id is not None:
@@ -4584,8 +4594,8 @@ class PrismaClient:
     )
     async def delete_data(
         self,
-        tokens: list | None = None,
-        team_id_list: list | None = None,
+        tokens: Sequence[str | None] | None = None,
+        team_id_list: Sequence[str] | None = None,
         table_name: Literal["user", "key", "config", "spend", "team"] | None = None,
         user_id: str | None = None,
     ):
@@ -4597,14 +4607,14 @@ class PrismaClient:
         start_time: Final = time.time()
         try:
             if tokens is not None and isinstance(tokens, list):
-                hashed_tokens: Final = []
+                hashed_tokens: Final[list[str | None]] = []
                 for token in tokens:
                     if isinstance(token, str) and token.startswith("sk-"):
                         hashed_token = self.hash_token(token=token)
                     else:
                         hashed_token = token
                     hashed_tokens.append(hashed_token)
-                filter_query: dict = {}
+                filter_query: dict[str, object] = {}
                 if user_id is not None:
                     filter_query = {"AND": [{"token": {"in": hashed_tokens}}, {"user_id": user_id}]}
                 else:
@@ -5749,12 +5759,12 @@ class PrismaClient:
         limit: int = 100,
         offset: int = 0,
         status_filter: str | None = None,
-    ):
+    ) -> "Sequence[prisma_models.LiteLLM_HealthCheckTable]":
         """
         Get health check history with optional filtering
         """
         try:
-            where_clause: Final = {}
+            where_clause: Final[dict[str, str]] = {}
             if model_name:
                 where_clause["model_name"] = model_name
             if status_filter:
@@ -5771,7 +5781,7 @@ class PrismaClient:
             verbose_proxy_logger.error("Error getting health check history: %s", e)
             return []
 
-    async def get_all_latest_health_checks(self):
+    async def get_all_latest_health_checks(self) -> "Sequence[prisma_models.LiteLLM_HealthCheckTable]":
         """
         Get the latest health check for each model.
 
@@ -5949,15 +5959,17 @@ async def migrate_passwords_to_scrypt_async(prisma_client) -> str:
         return len(s) == 64 and all(c in "0123456789abcdef" for c in s)
 
     plaintext_users: Final = [
-        u for u in all_with_pw if u.password and not u.password.startswith("scrypt:") and not _is_sha256_hex(u.password)
+        (u.user_id, u.password)
+        for u in all_with_pw
+        if u.password and not u.password.startswith("scrypt:") and not _is_sha256_hex(u.password)
     ]
     if not plaintext_users:
         return "No plaintext passwords found"
 
-    for user in plaintext_users:
+    for user_id, plaintext_password in plaintext_users:
         await UserRepository(prisma_client).table.update(
-            where={"user_id": user.user_id},
-            data={"password": hash_password(user.password)},
+            where={"user_id": user_id},
+            data={"password": hash_password(plaintext_password)},
         )
     return f"Migrated {len(plaintext_users)} plaintext passwords to scrypt"
 
diff --git a/litellm/proxy/vector_store_endpoints/endpoints.py b/litellm/proxy/vector_store_endpoints/endpoints.py
index b497247f576..a59d7a277cc 100644
--- a/litellm/proxy/vector_store_endpoints/endpoints.py
+++ b/litellm/proxy/vector_store_endpoints/endpoints.py
@@ -1,4 +1,9 @@
-from typing import Annotated, Any, Final
+from typing import (
+    Annotated,
+    Any,  # noqa: TID251  # jsonify_object in proxy/utils.py is annotated with a bare dict
+    Final,
+    cast,  # noqa: TID251  # jsonify_object in proxy/utils.py is annotated with a bare dict
+)
 
 from fastapi import APIRouter, Depends, HTTPException, Request, Response
 
@@ -591,7 +596,11 @@ async def index_create(
     index_data: Final = index_create_request.model_dump(exclude_none=True)
     index_data["created_by"] = user_api_key_dict.user_id
     index_data["updated_by"] = user_api_key_dict.user_id
-    new_index = await ManagedVectorStoreIndexRepository(prisma_client).table.create(data=jsonify_object(index_data))
+    new_index = await ManagedVectorStoreIndexRepository(prisma_client).table.create(
+        data=cast(  # cast-ok: jsonify_object deep-copies a model_dump, so keys are str and values plain objects
+            "dict[str, object]", jsonify_object(index_data)
+        )
+    )
 
     return new_index.model_dump()
 
diff --git a/litellm/proxy/vector_store_endpoints/management_endpoints.py b/litellm/proxy/vector_store_endpoints/management_endpoints.py
index 2b037bef795..183a03cc13c 100644
--- a/litellm/proxy/vector_store_endpoints/management_endpoints.py
+++ b/litellm/proxy/vector_store_endpoints/management_endpoints.py
@@ -10,8 +10,7 @@ All /vector_store management endpoints
 
 import copy
 import json
-from collections.abc import Mapping
-from typing import TYPE_CHECKING, Any, Final, Protocol
+from typing import TYPE_CHECKING, Any, Final
 
 from fastapi import APIRouter, Depends, HTTPException
 
@@ -37,6 +36,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helpe
 from litellm.proxy.common_utils.rbac_utils import check_feature_access_for_user
 from litellm.proxy.vector_store_endpoints.utils import can_user_access_vector_store
 from litellm.repositories.model_repository import ModelRepository
+from litellm.repositories.prisma_protocols import TableActions
 from litellm.repositories.table_repositories import ManagedVectorStoresRepository
 from litellm.secret_managers.main import get_secret
 from litellm.types.vector_stores import (
@@ -51,17 +51,7 @@ from litellm.vector_stores.vector_store_registry import VectorStoreRegistry
 router: Final = APIRouter()
 
 
-class _VectorStoreTableActions(Protocol):
-    async def find_unique(self, where: Mapping[str, str]) -> "_VectorStoreRow | None": ...
-
-    async def create(self, data: Mapping[str, object]) -> "_VectorStoreRow": ...
-
-    async def update(self, where: Mapping[str, str], data: Mapping[str, object]) -> "_VectorStoreRow": ...
-
-    async def delete(self, where: Mapping[str, str]) -> "_VectorStoreRow | None": ...
-
-
-def _vector_store_table(prisma_client: "PrismaClient") -> _VectorStoreTableActions:
+def _vector_store_table(prisma_client: "PrismaClient") -> "TableActions[_VectorStoreRow]":
     return ManagedVectorStoresRepository(prisma_client).table
 
 
@@ -277,7 +267,7 @@ async def _resolve_embedding_config_from_db(
             if db_model and db_model.litellm_params:
                 # Extract litellm_params (could be dict or JSON string)
                 model_params = db_model.litellm_params
-                if isinstance(model_params, str):
+                if isinstance(model_params, str):  # pyright: ignore[reportUnnecessaryIsInstance]  # prisma Json is str
                     model_params = json.loads(model_params)
 
                 # Decrypt values from database (similar to how proxy_server.py does it)
@@ -888,6 +878,12 @@ async def update_vector_store(
             data=update_data,
         )
 
+        if updated is None:
+            raise HTTPException(
+                status_code=404,
+                detail=f"Vector store with ID {vector_store_id} not found",
+            )
+
         updated_vs: Final = _row_to_vector_store(updated)
 
         # Immediately update in-memory registry to keep it in sync
diff --git a/litellm/repositories/base_repository.py b/litellm/repositories/base_repository.py
index 7008099fe8c..568e3b50ed2 100644
--- a/litellm/repositories/base_repository.py
+++ b/litellm/repositories/base_repository.py
@@ -8,6 +8,8 @@ from typing import Any, Final, Generic, Protocol, TypeVar, runtime_checkable
 
 from pydantic import BaseModel
 
+from litellm.repositories.prisma_protocols import TableActions
+
 T = TypeVar("T", bound=BaseModel)
 
 
@@ -49,7 +51,7 @@ class BaseRepository(ABC, Generic[T]):
 
     @property
     @abstractmethod
-    def table(self) -> Any:  # any-ok: Prisma table actions are reached through the untyped client wrapper
+    def table(self) -> TableActions[DbRecord]:
         """Return the Prisma table for this repository."""
         ...
 
@@ -76,33 +78,28 @@ class BaseRepository(ABC, Generic[T]):
 
     async def find_many(
         self,
-        where: dict[str, Any] | None = None,
+        where: Mapping[str, object] | None = None,
         skip: int | None = None,
         take: int | None = None,
-        order: dict[str, str] | None = None,
+        order: Mapping[str, str] | None = None,
     ) -> list[T]:
         """Find multiple records matching the criteria."""
-        kwargs: Final[dict[str, Any]] = {}
-        if where:
-            kwargs["where"] = where
-        if skip is not None:
-            kwargs["skip"] = skip
-        if take is not None:
-            kwargs["take"] = take
-        if order:
-            kwargs["order"] = order
-
-        records: Final = await self.table.find_many(**kwargs)
+        records: Final = await self.table.find_many(
+            take=take,
+            skip=skip,
+            where=where or None,
+            order=order or None,
+        )
         return self._to_model_list(records)
 
-    async def create(self, data: dict[str, Any]) -> T:
+    async def create(self, data: Mapping[str, object]) -> T:
         """Create a new record."""
         record: Final = await self.table.create(data=data)
         model: Final = self._to_model(record)
         assert model is not None
         return model
 
-    async def update(self, id_value: str, data: dict[str, Any], id_field: str = "id") -> T | None:
+    async def update(self, id_value: str, data: Mapping[str, object], id_field: str = "id") -> T | None:
         """Update an existing record."""
         record: Final = await self.table.update(where={id_field: id_value}, data=data)
         return self._to_model(record)
@@ -112,7 +109,7 @@ class BaseRepository(ABC, Generic[T]):
         record: Final = await self.table.delete(where={id_field: id_value})
         return self._to_model(record)
 
-    async def count(self, where: dict[str, Any] | None = None) -> int:
+    async def count(self, where: Mapping[str, object] | None = None) -> int:
         """Count records matching the criteria."""
         return await self.table.count(where=where)
 
diff --git a/litellm/repositories/budget_repository.py b/litellm/repositories/budget_repository.py
index f6c47b2d639..62632ffb5f6 100644
--- a/litellm/repositories/budget_repository.py
+++ b/litellm/repositories/budget_repository.py
@@ -2,17 +2,21 @@
 Budget repository for database operations on LiteLLM_BudgetTable.
 """
 
-from typing import Any, Final
+from typing import TYPE_CHECKING, Any, Final
 
 from litellm.models.budget import LiteLLM_BudgetTable
 from litellm.repositories.base_repository import BaseRepository
+from litellm.repositories.prisma_protocols import TableActions
+
+if TYPE_CHECKING:
+    from prisma import models as prisma_models
 
 
 class BudgetRepository(BaseRepository[LiteLLM_BudgetTable]):
     """Repository for budget database operations."""
 
     @property
-    def table(self) -> Any:
+    def table(self) -> TableActions["prisma_models.LiteLLM_BudgetTable"]:
         return self.prisma_client.db.litellm_budgettable
 
     @property
diff --git a/litellm/repositories/config_repository.py b/litellm/repositories/config_repository.py
index 71ae39e89c6..76b9a3a5809 100644
--- a/litellm/repositories/config_repository.py
+++ b/litellm/repositories/config_repository.py
@@ -77,7 +77,7 @@ class ConfigRepository:
         return self.prisma_client.db.litellm_config
 
     @property
-    def table(self) -> Any:
+    def table(self) -> _ConfigTable:
         return self._config_table
 
     async def get_param(self, param_name: str) -> ConfigParam | None:
diff --git a/litellm/repositories/credentials_repository.py b/litellm/repositories/credentials_repository.py
index 9fdb6e4aca7..ddb9767b2b9 100644
--- a/litellm/repositories/credentials_repository.py
+++ b/litellm/repositories/credentials_repository.py
@@ -6,54 +6,77 @@ credential values is the caller's responsibility (see ``CredentialHelperUtils``)
 so reads return the stored values verbatim.
 """
 
-from typing import Any, Final
+from collections.abc import Mapping, Sequence
+from typing import TYPE_CHECKING, Any, Final, Protocol, TypeAlias
 
 from litellm.models.credentials import CredentialItem
 from litellm.proxy.common_utils.config_sync_pubsub import wrap_table_actions_for_config_sync
+from litellm.repositories.base_repository import DbRecord, record_to_dict
+from litellm.repositories.prisma_protocols import TableActions
+
+if TYPE_CHECKING:
+    from prisma import models as prisma_models
+
+    _CredentialsTable: TypeAlias = TableActions[prisma_models.LiteLLM_CredentialsTable]
+
+
+class _PrismaCredentialsDb(Protocol):
+    @property
+    def litellm_credentialstable(self) -> "_CredentialsTable": ...
+
+
+class _PrismaClientView(Protocol):
+    @property
+    def db(self) -> _PrismaCredentialsDb: ...
 
 
 class CredentialsRepository:
     """Repository for credentials database operations, keyed by credential name."""
 
-    def __init__(self, prisma_client: Any):
+    def __init__(self, prisma_client: Any):  # any-ok: PrismaClient is an untyped runtime wrapper
         self._prisma_client = prisma_client
 
     @property
-    def prisma_client(self) -> Any:
+    def prisma_client(self) -> _PrismaClientView:
         if self._prisma_client is None:
             raise RuntimeError("No DB Connected. See - https://docs.litellm.ai/docs/proxy/virtual_keys")
-        return self._prisma_client
+        client: Final[_PrismaClientView] = self._prisma_client
+        return client
 
     @property
-    def table(self) -> Any:
+    def table(self) -> "_CredentialsTable":
         return wrap_table_actions_for_config_sync(
             actions=self.prisma_client.db.litellm_credentialstable,
             table_name="litellm_credentialstable",
         )
 
     @staticmethod
-    def _to_model(record: Any) -> CredentialItem | None:
+    def _to_model(record: DbRecord | None) -> CredentialItem | None:
         if record is None:
             return None
-        data: Final = record.dict() if hasattr(record, "dict") else dict(record)
-        return CredentialItem(
-            credential_name=data["credential_name"],
-            credential_values=data.get("credential_values") or {},
-            credential_info=data.get("credential_info") or {},
+        data: Final = record_to_dict(record)
+        return CredentialItem.model_validate(
+            {
+                "credential_name": data["credential_name"],
+                "credential_values": data.get("credential_values") or {},
+                "credential_info": data.get("credential_info") or {},
+            }
         )
 
-    async def find_all(self) -> Any:
+    async def find_all(self) -> Sequence["prisma_models.LiteLLM_CredentialsTable"]:
         return await self.table.find_many()
 
-    async def create(self, data: dict[str, Any]) -> Any:
+    async def create(self, data: Mapping[str, object]) -> "prisma_models.LiteLLM_CredentialsTable":
         return await self.table.create(data=data)
 
     async def find_by_name(self, credential_name: str) -> CredentialItem | None:
         record: Final = await self.table.find_unique(where={"credential_name": credential_name})
         return self._to_model(record)
 
-    async def update_by_name(self, credential_name: str, data: dict[str, Any]) -> Any:
+    async def update_by_name(
+        self, credential_name: str, data: Mapping[str, object]
+    ) -> "prisma_models.LiteLLM_CredentialsTable | None":
         return await self.table.update(where={"credential_name": credential_name}, data=data)
 
-    async def delete_by_name(self, credential_name: str) -> Any:
+    async def delete_by_name(self, credential_name: str) -> "prisma_models.LiteLLM_CredentialsTable | None":
         return await self.table.delete(where={"credential_name": credential_name})
diff --git a/litellm/repositories/model_repository.py b/litellm/repositories/model_repository.py
index 27e23a39cc9..cc2b1f19a3c 100644
--- a/litellm/repositories/model_repository.py
+++ b/litellm/repositories/model_repository.py
@@ -3,8 +3,8 @@ Model repository for database operations on LiteLLM_ProxyModelTable.
 """
 
 import json
-from collections.abc import Awaitable, Mapping, Sequence
-from typing import Any, Final, Protocol
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Any, Final, Protocol
 
 from litellm.models.model import LiteLLM_ProxyModelTable
 from litellm.proxy.common_utils.config_sync_pubsub import wrap_table_actions_for_config_sync
@@ -12,25 +12,21 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import (
     decrypt_value_helper,
     encrypt_value_helper,
 )
-from litellm.repositories.base_repository import BaseRepository, DbRecord
+from litellm.repositories.base_repository import BaseRepository
+from litellm.repositories.prisma_protocols import TableActions
+
+if TYPE_CHECKING:
+    from prisma import models as prisma_models
 
 
 class _PrismaModelDb(Protocol):
-    litellm_proxymodeltable: object
+    @property
+    def litellm_proxymodeltable(self) -> TableActions["prisma_models.LiteLLM_ProxyModelTable"]: ...
 
 
 class _PrismaClientView(Protocol):
-    db: _PrismaModelDb
-
-
-class _ProxyModelActions(Protocol):
-    """Prisma table actions used by :class:`ModelRepository`."""
-
-    def find_many(self, *, where: Mapping[str, object] | None = None) -> Awaitable[Sequence[DbRecord]]: ...
-
-    def create(self, *, data: Mapping[str, object]) -> Awaitable[DbRecord]: ...
-
-    def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> Awaitable[DbRecord | None]: ...
+    @property
+    def db(self) -> _PrismaModelDb: ...
 
 
 class ModelRepository(BaseRepository[LiteLLM_ProxyModelTable]):
@@ -41,17 +37,13 @@ class ModelRepository(BaseRepository[LiteLLM_ProxyModelTable]):
         self._encryption_key = encryption_key
 
     @property
-    def table(self) -> Any:
+    def table(self) -> TableActions["prisma_models.LiteLLM_ProxyModelTable"]:
         client: Final[_PrismaClientView] = self.prisma_client
         return wrap_table_actions_for_config_sync(
             actions=client.db.litellm_proxymodeltable,
             table_name="litellm_proxymodeltable",
         )
 
-    @property
-    def _model_table(self) -> _ProxyModelActions:
-        return self.table
-
     @property
     def model_class(self) -> type[LiteLLM_ProxyModelTable]:
         return LiteLLM_ProxyModelTable
@@ -100,17 +92,17 @@ class ModelRepository(BaseRepository[LiteLLM_ProxyModelTable]):
 
     async def find_by_name(self, model_name: str) -> list[LiteLLM_ProxyModelTable]:
         """Find models by name."""
-        records: Final = await self._model_table.find_many(where={"model_name": model_name})
+        records: Final = await self.table.find_many(where={"model_name": model_name})
         return self._to_model_list(records)
 
     async def find_all(self) -> list[LiteLLM_ProxyModelTable]:
         """Find all models."""
-        records: Final = await self._model_table.find_many()
+        records: Final = await self.table.find_many()
         return self._to_model_list(records)
 
     async def find_unblocked(self) -> list[LiteLLM_ProxyModelTable]:
         """Find all models that are not blocked."""
-        records: Final = await self._model_table.find_many(where={"blocked": False})
+        records: Final = await self.table.find_many(where={"blocked": False})
         return self._to_model_list(records)
 
     async def find_by_team_id(self, team_id: str) -> list[LiteLLM_ProxyModelTable]:
@@ -147,7 +139,7 @@ class ModelRepository(BaseRepository[LiteLLM_ProxyModelTable]):
         if model_info is not None:
             data["model_info"] = json.dumps(model_info)
 
-        record: Final = await self._model_table.create(data=data)
+        record: Final = await self.table.create(data=data)
         model: Final = self._to_model(record)
         assert model is not None
         return model
@@ -173,7 +165,7 @@ class ModelRepository(BaseRepository[LiteLLM_ProxyModelTable]):
         if blocked is not None:
             data["blocked"] = blocked
 
-        record: Final = await self._model_table.update(where={"model_id": model_id}, data=data)
+        record: Final = await self.table.update(where={"model_id": model_id}, data=data)
         return self._to_model(record)
 
     async def delete_model(self, model_id: str) -> LiteLLM_ProxyModelTable | None:
diff --git a/litellm/repositories/object_permission_repository.py b/litellm/repositories/object_permission_repository.py
index 54a311c4a77..6b1f9c68e47 100644
--- a/litellm/repositories/object_permission_repository.py
+++ b/litellm/repositories/object_permission_repository.py
@@ -2,17 +2,21 @@
 ObjectPermission repository for database operations on LiteLLM_ObjectPermissionTable.
 """
 
-from typing import Any, Final
+from typing import TYPE_CHECKING, Any, Final
 
 from litellm.models.object_permission import LiteLLM_ObjectPermissionTable
 from litellm.repositories.base_repository import BaseRepository
+from litellm.repositories.prisma_protocols import TableActions
+
+if TYPE_CHECKING:
+    from prisma import models as prisma_models
 
 
 class ObjectPermissionRepository(BaseRepository[LiteLLM_ObjectPermissionTable]):
     """Repository for object permission database operations."""
 
     @property
-    def table(self) -> Any:
+    def table(self) -> TableActions["prisma_models.LiteLLM_ObjectPermissionTable"]:
         return self.prisma_client.db.litellm_objectpermissiontable
 
     @property
diff --git a/litellm/repositories/organization_repository.py b/litellm/repositories/organization_repository.py
index 8a1350903b7..5a9bd3724e0 100644
--- a/litellm/repositories/organization_repository.py
+++ b/litellm/repositories/organization_repository.py
@@ -2,17 +2,21 @@
 Organization repository for database operations on LiteLLM_OrganizationTable.
 """
 
-from typing import Any, Final
+from typing import TYPE_CHECKING, Any, Final
 
 from litellm.models.organization import LiteLLM_OrganizationTable
 from litellm.repositories.base_repository import BaseRepository
+from litellm.repositories.prisma_protocols import TableActions
+
+if TYPE_CHECKING:
+    from prisma import models as prisma_models
 
 
 class OrganizationRepository(BaseRepository[LiteLLM_OrganizationTable]):
     """Repository for organization database operations."""
 
     @property
-    def table(self) -> Any:
+    def table(self) -> TableActions["prisma_models.LiteLLM_OrganizationTable"]:
         return self.prisma_client.db.litellm_organizationtable
 
     @property
diff --git a/litellm/repositories/prisma_protocols.py b/litellm/repositories/prisma_protocols.py
index 055c68163f9..2aa1b8e0e3f 100644
--- a/litellm/repositories/prisma_protocols.py
+++ b/litellm/repositories/prisma_protocols.py
@@ -12,6 +12,93 @@ from typing import Protocol, TypeVar
 RowT_co = TypeVar("RowT_co", covariant=True)
 
 
+class TableActions(Protocol[RowT_co]):
+    """The prisma-client-py per-model action surface, keyed to the row it returns.
+
+    Query inputs stay `Mapping[str, object]` rather than the generated
+    `types.*` TypedDicts so callers can keep passing plain dicts, while every
+    result carries the row type the repository is bound to.
+    """
+
+    async def find_unique(
+        self, where: Mapping[str, object], include: Mapping[str, object] | None = None
+    ) -> RowT_co | None: ...
+
+    async def find_first(
+        self,
+        skip: int | None = None,
+        where: Mapping[str, object] | None = None,
+        cursor: Mapping[str, object] | None = None,
+        include: Mapping[str, object] | None = None,
+        order: Mapping[str, object] | Sequence[Mapping[str, object]] | None = None,
+        distinct: Sequence[str] | None = None,
+    ) -> RowT_co | None: ...
+
+    async def find_many(
+        self,
+        take: int | None = None,
+        skip: int | None = None,
+        where: Mapping[str, object] | None = None,
+        cursor: Mapping[str, object] | None = None,
+        include: Mapping[str, object] | None = None,
+        order: Mapping[str, object] | Sequence[Mapping[str, object]] | None = None,
+        distinct: Sequence[str] | None = None,
+    ) -> Sequence[RowT_co]: ...
+
+    async def create(self, data: Mapping[str, object], include: Mapping[str, object] | None = None) -> RowT_co: ...
+
+    async def create_many(
+        self, data: Sequence[Mapping[str, object]], *, skip_duplicates: bool | None = None
+    ) -> int: ...
+
+    async def upsert(
+        self,
+        where: Mapping[str, object],
+        data: Mapping[str, object],
+        include: Mapping[str, object] | None = None,
+    ) -> RowT_co: ...
+
+    async def update(
+        self,
+        data: Mapping[str, object],
+        where: Mapping[str, object],
+        include: Mapping[str, object] | None = None,
+    ) -> RowT_co | None: ...
+
+    async def update_many(self, data: Mapping[str, object], where: Mapping[str, object]) -> int: ...
+
+    async def delete(
+        self, where: Mapping[str, object], include: Mapping[str, object] | None = None
+    ) -> RowT_co | None: ...
+
+    async def delete_many(self, where: Mapping[str, object] | None = None) -> int: ...
+
+    async def count(
+        self,
+        select: None = None,
+        take: int | None = None,
+        skip: int | None = None,
+        where: Mapping[str, object] | None = None,
+        cursor: Mapping[str, object] | None = None,
+    ) -> int: ...
+
+    async def group_by(
+        self,
+        by: Sequence[str],
+        *,
+        where: Mapping[str, object] | None = None,
+        take: int | None = None,
+        skip: int | None = None,
+        order: Mapping[str, object] | Sequence[Mapping[str, object]] | None = None,
+        having: Mapping[str, object] | None = None,
+        count: bool | Mapping[str, object] | None = None,
+        sum: bool | Mapping[str, object] | None = None,
+        avg: bool | Mapping[str, object] | None = None,
+        min: bool | Mapping[str, object] | None = None,
+        max: bool | Mapping[str, object] | None = None,
+    ) -> Sequence[Mapping[str, object]]: ...
+
+
 class PrismaRecord(Protocol):
     def dict(self) -> Mapping[str, object]: ...
 
diff --git a/litellm/repositories/project_repository.py b/litellm/repositories/project_repository.py
index c8b2c62f9bf..48e55efd258 100644
--- a/litellm/repositories/project_repository.py
+++ b/litellm/repositories/project_repository.py
@@ -2,17 +2,21 @@
 Project repository for database operations on LiteLLM_ProjectTable.
 """
 
-from typing import Any, Final
+from typing import TYPE_CHECKING, Any, Final
 
 from litellm.models.project import LiteLLM_ProjectTable
 from litellm.repositories.base_repository import BaseRepository
+from litellm.repositories.prisma_protocols import TableActions
+
+if TYPE_CHECKING:
+    from prisma import models as prisma_models
 
 
 class ProjectRepository(BaseRepository[LiteLLM_ProjectTable]):
     """Repository for project database operations."""
 
     @property
-    def table(self) -> Any:
+    def table(self) -> TableActions["prisma_models.LiteLLM_ProjectTable"]:
         return self.prisma_client.db.litellm_projecttable
 
     @property
diff --git a/litellm/repositories/table_repositories.py b/litellm/repositories/table_repositories.py
index 131f4d377ef..e02f652caf6 100644
--- a/litellm/repositories/table_repositories.py
+++ b/litellm/repositories/table_repositories.py
@@ -7,12 +7,16 @@ These are thin wrappers for tables that do not (yet) need domain-specific query
 methods; richer repositories live in their own modules.
 """
 
-from typing import Any
+from typing import TYPE_CHECKING, Any, Final, Generic
 
 from litellm.proxy.common_utils.config_sync_pubsub import wrap_table_actions_for_config_sync
+from litellm.repositories.prisma_protocols import RowT_co, TableActions
+
+if TYPE_CHECKING:
+    from prisma import models as prisma_models  # noqa: F401  # used by quoted base-class subscripts
 
 
-class PrismaTableRepository:
+class PrismaTableRepository(Generic[RowT_co]):
     """Base for repositories that expose a single Prisma table."""
 
     table_name: str
@@ -27,208 +31,206 @@ class PrismaTableRepository:
         return self._prisma_client
 
     @property
-    def table(self) -> Any:  # any-ok: Prisma table actions are reached through the untyped client wrapper
-        return wrap_table_actions_for_config_sync(
-            actions=getattr(self.prisma_client.db, self.table_name),
-            table_name=self.table_name,
-        )
+    def table(self) -> TableActions[RowT_co]:
+        actions: Final[TableActions[RowT_co]] = getattr(self.prisma_client.db, self.table_name)
+        return wrap_table_actions_for_config_sync(actions=actions, table_name=self.table_name)
 
 
-class PolicyRepository(PrismaTableRepository):
+class PolicyRepository(PrismaTableRepository["prisma_models.LiteLLM_PolicyTable"]):
     table_name = "litellm_policytable"
 
 
-class AgentsRepository(PrismaTableRepository):
+class AgentsRepository(PrismaTableRepository["prisma_models.LiteLLM_AgentsTable"]):
     table_name = "litellm_agentstable"
 
 
-class ObjectPermissionRepository(PrismaTableRepository):
+class ObjectPermissionRepository(PrismaTableRepository["prisma_models.LiteLLM_ObjectPermissionTable"]):
     table_name = "litellm_objectpermissiontable"
 
 
-class GuardrailsRepository(PrismaTableRepository):
+class GuardrailsRepository(PrismaTableRepository["prisma_models.LiteLLM_GuardrailsTable"]):
     table_name = "litellm_guardrailstable"
 
 
-class MCPServerRepository(PrismaTableRepository):
+class MCPServerRepository(PrismaTableRepository["prisma_models.LiteLLM_MCPServerTable"]):
     table_name = "litellm_mcpservertable"
 
 
-class ManagedObjectRepository(PrismaTableRepository):
+class ManagedObjectRepository(PrismaTableRepository["prisma_models.LiteLLM_ManagedObjectTable"]):
     table_name = "litellm_managedobjecttable"
 
 
-class OrganizationMembershipRepository(PrismaTableRepository):
+class OrganizationMembershipRepository(PrismaTableRepository["prisma_models.LiteLLM_OrganizationMembership"]):
     table_name = "litellm_organizationmembership"
 
 
-class SpendLogsRepository(PrismaTableRepository):
+class SpendLogsRepository(PrismaTableRepository["prisma_models.LiteLLM_SpendLogs"]):
     table_name = "litellm_spendlogs"
 
 
-class ClaudeCodePluginRepository(PrismaTableRepository):
+class ClaudeCodePluginRepository(PrismaTableRepository["prisma_models.LiteLLM_ClaudeCodePluginTable"]):
     table_name = "litellm_claudecodeplugintable"
 
 
-class TeamMembershipRepository(PrismaTableRepository):
+class TeamMembershipRepository(PrismaTableRepository["prisma_models.LiteLLM_TeamMembership"]):
     table_name = "litellm_teammembership"
 
 
-class EndUserRepository(PrismaTableRepository):
+class EndUserRepository(PrismaTableRepository["prisma_models.LiteLLM_EndUserTable"]):
     table_name = "litellm_endusertable"
 
 
-class ManagedVectorStoresRepository(PrismaTableRepository):
+class ManagedVectorStoresRepository(PrismaTableRepository["prisma_models.LiteLLM_ManagedVectorStoresTable"]):
     table_name = "litellm_managedvectorstorestable"
 
 
-class MCPUserCredentialsRepository(PrismaTableRepository):
+class MCPUserCredentialsRepository(PrismaTableRepository["prisma_models.LiteLLM_MCPUserCredentials"]):
     table_name = "litellm_mcpusercredentials"
 
 
-class MCPServerOAuthClientRepository(PrismaTableRepository):
+class MCPServerOAuthClientRepository(PrismaTableRepository["prisma_models.LiteLLM_MCPServerOAuthClient"]):
     table_name = "litellm_mcpserveroauthclient"
 
 
-class PromptRepository(PrismaTableRepository):
+class PromptRepository(PrismaTableRepository["prisma_models.LiteLLM_PromptTable"]):
     table_name = "litellm_prompttable"
 
 
-class TagRepository(PrismaTableRepository):
+class TagRepository(PrismaTableRepository["prisma_models.LiteLLM_TagTable"]):
     table_name = "litellm_tagtable"
 
 
-class InvitationLinkRepository(PrismaTableRepository):
+class InvitationLinkRepository(PrismaTableRepository["prisma_models.LiteLLM_InvitationLink"]):
     table_name = "litellm_invitationlink"
 
 
-class JWTKeyMappingRepository(PrismaTableRepository):
+class JWTKeyMappingRepository(PrismaTableRepository["prisma_models.LiteLLM_JWTKeyMapping"]):
     table_name = "litellm_jwtkeymapping"
 
 
-class ManagedFileRepository(PrismaTableRepository):
+class ManagedFileRepository(PrismaTableRepository["prisma_models.LiteLLM_ManagedFileTable"]):
     table_name = "litellm_managedfiletable"
 
 
-class MemoryRepository(PrismaTableRepository):
+class MemoryRepository(PrismaTableRepository["prisma_models.LiteLLM_MemoryTable"]):
     table_name = "litellm_memorytable"
 
 
-class SearchToolsRepository(PrismaTableRepository):
+class SearchToolsRepository(PrismaTableRepository["prisma_models.LiteLLM_SearchToolsTable"]):
     table_name = "litellm_searchtoolstable"
 
 
-class ConfigOverridesRepository(PrismaTableRepository):
+class ConfigOverridesRepository(PrismaTableRepository["prisma_models.LiteLLM_ConfigOverrides"]):
     table_name = "litellm_configoverrides"
 
 
-class MCPToolsetRepository(PrismaTableRepository):
+class MCPToolsetRepository(PrismaTableRepository["prisma_models.LiteLLM_MCPToolsetTable"]):
     table_name = "litellm_mcptoolsettable"
 
 
-class ToolRepository(PrismaTableRepository):
+class ToolRepository(PrismaTableRepository["prisma_models.LiteLLM_ToolTable"]):
     table_name = "litellm_tooltable"
 
 
-class DeletedVerificationTokenRepository(PrismaTableRepository):
+class DeletedVerificationTokenRepository(PrismaTableRepository["prisma_models.LiteLLM_DeletedVerificationToken"]):
     table_name = "litellm_deletedverificationtoken"
 
 
-class WorkflowRunRepository(PrismaTableRepository):
+class WorkflowRunRepository(PrismaTableRepository["prisma_models.LiteLLM_WorkflowRun"]):
     table_name = "litellm_workflowrun"
 
 
-class ModelTableRepository(PrismaTableRepository):
+class ModelTableRepository(PrismaTableRepository["prisma_models.LiteLLM_ModelTable"]):
     table_name = "litellm_modeltable"
 
 
-class AccessGroupRepository(PrismaTableRepository):
+class AccessGroupRepository(PrismaTableRepository["prisma_models.LiteLLM_AccessGroupTable"]):
     table_name = "litellm_accessgrouptable"
 
 
-class SSOConfigRepository(PrismaTableRepository):
+class SSOConfigRepository(PrismaTableRepository["prisma_models.LiteLLM_SSOConfig"]):
     table_name = "litellm_ssoconfig"
 
 
-class UISettingsRepository(PrismaTableRepository):
+class UISettingsRepository(PrismaTableRepository["prisma_models.LiteLLM_UISettings"]):
     table_name = "litellm_uisettings"
 
 
-class DailyGuardrailMetricsRepository(PrismaTableRepository):
+class DailyGuardrailMetricsRepository(PrismaTableRepository["prisma_models.LiteLLM_DailyGuardrailMetrics"]):
     table_name = "litellm_dailyguardrailmetrics"
 
 
-class DailyGuardrailUsageUnitsRepository(PrismaTableRepository):
+class DailyGuardrailUsageUnitsRepository(PrismaTableRepository["prisma_models.LiteLLM_DailyGuardrailUsageUnits"]):
     table_name = "litellm_dailyguardrailusageunits"
 
 
-class PolicyAttachmentRepository(PrismaTableRepository):
+class PolicyAttachmentRepository(PrismaTableRepository["prisma_models.LiteLLM_PolicyAttachmentTable"]):
     table_name = "litellm_policyattachmenttable"
 
 
-class DeletedTeamRepository(PrismaTableRepository):
+class DeletedTeamRepository(PrismaTableRepository["prisma_models.LiteLLM_DeletedTeamTable"]):
     table_name = "litellm_deletedteamtable"
 
 
-class SkillsRepository(PrismaTableRepository):
+class SkillsRepository(PrismaTableRepository["prisma_models.LiteLLM_SkillsTable"]):
     table_name = "litellm_skillstable"
 
 
-class CacheConfigRepository(PrismaTableRepository):
+class CacheConfigRepository(PrismaTableRepository["prisma_models.LiteLLM_CacheConfig"]):
     table_name = "litellm_cacheconfig"
 
 
-class ManagedVectorStoreIndexRepository(PrismaTableRepository):
+class ManagedVectorStoreIndexRepository(PrismaTableRepository["prisma_models.LiteLLM_ManagedVectorStoreIndexTable"]):
     table_name = "litellm_managedvectorstoreindextable"
 
 
-class WorkflowMessageRepository(PrismaTableRepository):
+class WorkflowMessageRepository(PrismaTableRepository["prisma_models.LiteLLM_WorkflowMessage"]):
     table_name = "litellm_workflowmessage"
 
 
-class DailyTagSpendRepository(PrismaTableRepository):
+class DailyTagSpendRepository(PrismaTableRepository["prisma_models.LiteLLM_DailyTagSpend"]):
     table_name = "litellm_dailytagspend"
 
 
-class SpendLogToolIndexRepository(PrismaTableRepository):
+class SpendLogToolIndexRepository(PrismaTableRepository["prisma_models.LiteLLM_SpendLogToolIndex"]):
     table_name = "litellm_spendlogtoolindex"
 
 
-class DailyToolSpendRepository(PrismaTableRepository):
+class DailyToolSpendRepository(PrismaTableRepository["prisma_models.LiteLLM_DailyToolSpend"]):
     table_name = "litellm_dailytoolspend"
 
 
-class SpendLogGuardrailIndexRepository(PrismaTableRepository):
+class SpendLogGuardrailIndexRepository(PrismaTableRepository["prisma_models.LiteLLM_SpendLogGuardrailIndex"]):
     table_name = "litellm_spendlogguardrailindex"
 
 
-class UserNotificationsRepository(PrismaTableRepository):
+class UserNotificationsRepository(PrismaTableRepository["prisma_models.LiteLLM_UserNotifications"]):
     table_name = "litellm_usernotifications"
 
 
-class HealthCheckRepository(PrismaTableRepository):
+class HealthCheckRepository(PrismaTableRepository["prisma_models.LiteLLM_HealthCheckTable"]):
     table_name = "litellm_healthchecktable"
 
 
-class DeprecatedVerificationTokenRepository(PrismaTableRepository):
+class DeprecatedVerificationTokenRepository(PrismaTableRepository["prisma_models.LiteLLM_DeprecatedVerificationToken"]):
     table_name = "litellm_deprecatedverificationtoken"
 
 
-class WorkflowEventRepository(PrismaTableRepository):
+class WorkflowEventRepository(PrismaTableRepository["prisma_models.LiteLLM_WorkflowEvent"]):
     table_name = "litellm_workflowevent"
 
 
-class DailyPolicyMetricsRepository(PrismaTableRepository):
+class DailyPolicyMetricsRepository(PrismaTableRepository["prisma_models.LiteLLM_DailyPolicyMetrics"]):
     table_name = "litellm_dailypolicymetrics"
 
 
-class AdaptiveRouterStateRepository(PrismaTableRepository):
+class AdaptiveRouterStateRepository(PrismaTableRepository["prisma_models.LiteLLM_AdaptiveRouterState"]):
     table_name = "litellm_adaptiverouterstate"
 
 
-class AuditLogRepository(PrismaTableRepository):
+class AuditLogRepository(PrismaTableRepository["prisma_models.LiteLLM_AuditLog"]):
     table_name = "litellm_auditlog"
 
 
-class AdaptiveRouterSessionRepository(PrismaTableRepository):
+class AdaptiveRouterSessionRepository(PrismaTableRepository["prisma_models.LiteLLM_AdaptiveRouterSession"]):
     table_name = "litellm_adaptiveroutersession"
diff --git a/litellm/repositories/team_repository.py b/litellm/repositories/team_repository.py
index 7efd32288e4..221f5b22c41 100644
--- a/litellm/repositories/team_repository.py
+++ b/litellm/repositories/team_repository.py
@@ -5,7 +5,7 @@ Team repository for database operations on LiteLLM_TeamTable.
 import json
 from collections.abc import Mapping
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Final
+from typing import TYPE_CHECKING, Final
 
 from pydantic import TypeAdapter
 
@@ -15,9 +15,11 @@ from litellm.repositories.base_repository import (
     DbRecord,
     record_to_dict,
 )
+from litellm.repositories.prisma_protocols import TableActions
 
 if TYPE_CHECKING:
     from prisma import Prisma
+    from prisma import models as prisma_models
 
 _MEMBERS_WITH_ROLES_ADAPTER: Final = TypeAdapter(list[Member])
 _JSON_ENCODED_TEAM_FIELDS: Final = (
@@ -34,11 +36,11 @@ class TeamRepository(BaseRepository[LiteLLM_TeamTable]):
     """Repository for team database operations."""
 
     @property
-    def table(self) -> Any:  # any-ok: PrismaClient.db is an untyped runtime wrapper
+    def table(self) -> TableActions["prisma_models.LiteLLM_TeamTable"]:
         return self.prisma_client.db.litellm_teamtable
 
     @property
-    def deleted_table(self) -> Any:  # any-ok: PrismaClient.db is an untyped runtime wrapper
+    def deleted_table(self) -> TableActions["prisma_models.LiteLLM_DeletedTeamTable"]:
         return self.prisma_client.db.litellm_deletedteamtable
 
     @property
diff --git a/litellm/repositories/user_banner_repository.py b/litellm/repositories/user_banner_repository.py
index 3b69e433853..c1ed977e048 100644
--- a/litellm/repositories/user_banner_repository.py
+++ b/litellm/repositories/user_banner_repository.py
@@ -1,11 +1,14 @@
-from typing import Final
+from typing import TYPE_CHECKING, Final
 
 from litellm.repositories.table_repositories import PrismaTableRepository
 
+if TYPE_CHECKING:
+    from prisma import models as prisma_models  # noqa: F401  # resolved only from the quoted base-class subscript below
+
 USER_BANNER_ROW_ID: Final = "user_banner"
 
 
-class UserBannerRepository(PrismaTableRepository):
+class UserBannerRepository(PrismaTableRepository["prisma_models.LiteLLM_UISettings"]):
     table_name = "litellm_uisettings"
 
     async def get_raw_settings(self) -> object:
diff --git a/litellm/repositories/user_repository.py b/litellm/repositories/user_repository.py
index d0d366e1772..9df1bceac9c 100644
--- a/litellm/repositories/user_repository.py
+++ b/litellm/repositories/user_repository.py
@@ -4,10 +4,14 @@ User repository for database operations on LiteLLM_UserTable.
 
 import json
 from collections.abc import Mapping
-from typing import Any, Final
+from typing import TYPE_CHECKING, Final
 
 from litellm.models.user import LiteLLM_UserTable
 from litellm.repositories.base_repository import BaseRepository, DbRecord, record_to_dict
+from litellm.repositories.prisma_protocols import TableActions
+
+if TYPE_CHECKING:
+    from prisma import models as prisma_models
 
 _JSON_ENCODED_COLUMNS: Final = frozenset({"metadata", "model_spend", "model_max_budget"})
 
@@ -16,7 +20,7 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]):
     """Repository for user database operations."""
 
     @property
-    def table(self) -> Any:  # any-ok: Prisma table actions are reached through the untyped client wrapper
+    def table(self) -> TableActions["prisma_models.LiteLLM_UserTable"]:
         return self.prisma_client.db.litellm_usertable
 
     @property
diff --git a/litellm/repositories/verification_token_repository.py b/litellm/repositories/verification_token_repository.py
index 3790ad25914..c0e59f9b975 100644
--- a/litellm/repositories/verification_token_repository.py
+++ b/litellm/repositories/verification_token_repository.py
@@ -3,9 +3,9 @@ VerificationToken repository for database operations on LiteLLM_VerificationToke
 """
 
 import json
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Final
+from typing import TYPE_CHECKING, Final
 
 from litellm.models.verification_token import (
     LiteLLM_VerificationToken,
@@ -15,8 +15,12 @@ from litellm.repositories.base_repository import (
     DbRecord,
     record_to_dict,
 )
+from litellm.repositories.prisma_protocols import TableActions
 
 if TYPE_CHECKING:
+    from prisma.models import (
+        LiteLLM_DeletedVerificationToken as PrismaDeletedVerificationToken,
+    )
     from prisma.models import (
         LiteLLM_VerificationToken as PrismaVerificationToken,
     )
@@ -45,11 +49,11 @@ class VerificationTokenRepository(BaseRepository[LiteLLM_VerificationToken]):
         return prisma_client
 
     @property
-    def table(self) -> Any:
+    def table(self) -> TableActions["PrismaVerificationToken"]:
         return self.prisma_client.db.litellm_verificationtoken
 
     @property
-    def deleted_table(self) -> Any:
+    def deleted_table(self) -> TableActions["PrismaDeletedVerificationToken"]:
         return self.prisma_client.db.litellm_deletedverificationtoken
 
     @property
@@ -79,29 +83,29 @@ class VerificationTokenRepository(BaseRepository[LiteLLM_VerificationToken]):
 
     async def find_by_alias(self, key_alias: str) -> LiteLLM_VerificationToken | None:
         """Find a token by key alias."""
-        records: Final[list[PrismaVerificationToken]] = await self.table.find_many(where={"key_alias": key_alias})
+        records: Final[Sequence[PrismaVerificationToken]] = await self.table.find_many(where={"key_alias": key_alias})
         if records:
             return self._to_model(records[0])
         return None
 
     async def find_by_user_id(self, user_id: str) -> list[LiteLLM_VerificationToken]:
         """Find all tokens belonging to a user."""
-        records: Final[list[PrismaVerificationToken]] = await self.table.find_many(where={"user_id": user_id})
+        records: Final[Sequence[PrismaVerificationToken]] = await self.table.find_many(where={"user_id": user_id})
         return self._to_model_list(records)
 
     async def find_by_team_id(self, team_id: str) -> list[LiteLLM_VerificationToken]:
         """Find all tokens belonging to a team."""
-        records: Final[list[PrismaVerificationToken]] = await self.table.find_many(where={"team_id": team_id})
+        records: Final[Sequence[PrismaVerificationToken]] = await self.table.find_many(where={"team_id": team_id})
         return self._to_model_list(records)
 
     async def find_by_project_id(self, project_id: str) -> list[LiteLLM_VerificationToken]:
         """Find all tokens belonging to a project."""
-        records: Final[list[PrismaVerificationToken]] = await self.table.find_many(where={"project_id": project_id})
+        records: Final[Sequence[PrismaVerificationToken]] = await self.table.find_many(where={"project_id": project_id})
         return self._to_model_list(records)
 
     async def find_active_tokens(self) -> list[LiteLLM_VerificationToken]:
         """Find all active (non-expired, non-blocked) tokens."""
-        records: Final[list[PrismaVerificationToken]] = await self.table.find_many(
+        records: Final[Sequence[PrismaVerificationToken]] = await self.table.find_many(
             where={
                 "blocked": {"not": True},
                 "OR": [{"expires": None}, {"expires": {"gt": datetime.utcnow()}}],
diff --git a/litellm/responses/file_search/emulated_handler.py b/litellm/responses/file_search/emulated_handler.py
index e9e7ae908a5..0418f0c5e14 100644
--- a/litellm/responses/file_search/emulated_handler.py
+++ b/litellm/responses/file_search/emulated_handler.py
@@ -15,7 +15,9 @@ import json
 import time
 import uuid
 from collections.abc import Iterable, Sequence
-from typing import TYPE_CHECKING, Any, Final, TypeAlias, cast
+from typing import TYPE_CHECKING, Any, Final, TypeAlias, cast  # noqa: TID251  # see kwargs-ok / cast-ok markers
+
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 from litellm._internal_context import is_internal_call
 from litellm._logging import verbose_logger
@@ -31,6 +33,12 @@ ToolParam: TypeAlias = object
 FILE_SEARCH_FUNCTION_NAME: Final = "litellm_file_search"
 
 
+class FileSearchToolCallArgs(TypedDict):
+    queries: ReadOnly[NotRequired[object]]
+    query: ReadOnly[NotRequired[object]]
+    vector_store_id: ReadOnly[NotRequired[object]]
+
+
 # ---------------------------------------------------------------------------
 # Detection
 # ---------------------------------------------------------------------------
@@ -175,13 +183,20 @@ async def _run_vector_searches(
 # ---------------------------------------------------------------------------
 
 
-def _get_field(result: object, key: str, default: object = None) -> Any:
+def _get_field(result: object, key: str, default: object = None) -> object:
     """Read a field from either a dict/TypedDict or an attribute-based object."""
     if isinstance(result, dict):
         return result.get(key, default)
     return getattr(result, key, default)
 
 
+def _joined_content_text(result: object) -> str:
+    """Concatenate the text of every content chunk on a search result."""
+    content_items: Final = cast(Iterable[object], _get_field(result, "content") or [])  # cast-ok: iterated as today
+    text_chunks: Final = [c.get("text", "") if isinstance(c, dict) else getattr(c, "text", "") for c in content_items]
+    return " ".join(t for t in text_chunks if t)
+
+
 def _format_search_results_as_tool_output(
     results: list[VectorStoreSearchResult],
 ) -> str:
@@ -194,9 +209,7 @@ def _format_search_results_as_tool_output(
         score = _get_field(result, "score")
         file_id = _get_field(result, "file_id")
         filename = _get_field(result, "filename")
-        content_items = _get_field(result, "content") or []
-        text_chunks = [c.get("text", "") if isinstance(c, dict) else getattr(c, "text", "") for c in content_items]
-        text = " ".join(t for t in text_chunks if t)
+        text = _joined_content_text(result)
 
         header = f"[Result {i}"
         if filename:
@@ -226,9 +239,7 @@ def _build_search_results_for_include(
     formatted: Final[list[dict[str, object]]] = []
     for result in results:
         file_id = _get_field(result, "file_id") or ""
-        content_items = _get_field(result, "content") or []
-        text_chunks = [c.get("text", "") if isinstance(c, dict) else getattr(c, "text", "") for c in content_items]
-        text = " ".join(t for t in text_chunks if t)
+        text = _joined_content_text(result)
         formatted.append(
             {
                 "file_id": file_id,
@@ -353,14 +364,14 @@ def _synthesize_responses_api_response(
         created_at=getattr(original_response, "created_at", int(time.time())),
         status="completed",
         model=getattr(original_response, "model", ""),
-        output=cast(list[ResponseOutputItem | dict[str, Any]], synthesized_output),
+        output=cast(list[ResponseOutputItem | dict[str, object]], synthesized_output),  # cast-ok: list is invariant
         usage=getattr(original_response, "usage", None),
         error=None,
     )
     if hasattr(original_response, "_hidden_params"):
         hidden: Final = dict(getattr(original_response, "_hidden_params") or {})
         if first_response is not None and hasattr(first_response, "_hidden_params"):
-            first_hidden: Final = getattr(first_response, "_hidden_params") or {}
+            first_hidden: Final[object] = getattr(first_response, "_hidden_params") or {}
             first_cost: Final = (
                 first_hidden.get("response_cost")
                 if isinstance(first_hidden, dict)
@@ -385,9 +396,10 @@ async def _call_aresponses(input, model, tools, **kwargs):  # pragma: no cover 
 
 
 def _prepare_emulated_file_search_call(
-    kwargs: dict[str, Any],
+    kwargs: dict[str, object],
 ) -> tuple[bool, dict[str, object]]:
-    include_items: Final[list[str]] = list(kwargs.get("include") or [])
+    raw_include: Final = kwargs.get("include") or []
+    include_items: Final[list[object]] = list(cast(Iterable[object], raw_include))  # cast-ok: iterated as today
     include_search_results: Final = "file_search_call.results" in include_items
 
     original_stream: Final = kwargs.get("stream")
@@ -413,16 +425,16 @@ def _extract_tool_call_fields(tool_call: object, fallback_call_id: str) -> tuple
     return call_id, raw_args
 
 
-def _resolve_queries_from_args(args: dict[str, Any], input: object) -> list[str]:
+def _resolve_queries_from_args(args: FileSearchToolCallArgs, input: object) -> list[str]:
     """Pull the queries list out of parsed tool-call arguments, with backward-compat fallbacks."""
     queries_from_call: Final = args.get("queries")
     if not queries_from_call:
         # Fallback: check for single "query" field (backward compat)
         single_query: Final = args.get("query")
-        return [single_query] if single_query else [str(input)]
+        return [cast(str, single_query)] if single_query else [str(input)]  # cast-ok: model-supplied, as today
     if not isinstance(queries_from_call, list):
         return [str(queries_from_call)]
-    return queries_from_call
+    return cast(list[str], queries_from_call)  # cast-ok: model-supplied elements, forwarded unchecked as today
 
 
 async def _execute_file_search_tool_calls(
@@ -440,14 +452,14 @@ async def _execute_file_search_tool_calls(
         call_id, raw_args = _extract_tool_call_fields(tool_call, fallback_call_id=file_search_call_id)
 
         try:
-            args = json.loads(raw_args) if isinstance(raw_args, str) else raw_args
+            args: FileSearchToolCallArgs = json.loads(raw_args) if isinstance(raw_args, str) else raw_args
         except json.JSONDecodeError:
             args = {}
 
         queries_from_call = _resolve_queries_from_args(args, input)
 
         vs_id_arg = args.get("vector_store_id")
-        vs_ids_for_call = [vs_id_arg] if vs_id_arg else all_vs_ids
+        vs_ids_for_call = [cast(str, vs_id_arg)] if vs_id_arg else all_vs_ids  # cast-ok: model-supplied, as today
 
         queries, results = await _run_vector_searches(
             queries=queries_from_call,
@@ -481,7 +493,7 @@ def _build_follow_up_input(
     original_input_items: Final[list[object]] = (
         list(input) if isinstance(input, (list, tuple)) else [{"role": "user", "content": str(input)}]
     )
-    first_response_output_items: Final[list[Any]] = []
+    first_response_output_items: Final[list[object]] = []
     for _item in first_response.output:
         if isinstance(_item, dict):
             first_response_output_items.append(_item)
@@ -498,7 +510,7 @@ async def aresponses_with_emulated_file_search(
     model: str,
     tools: Iterable[ToolParam] | None = None,
     # Pass-through params — forwarded as-is to the underlying aresponses call
-    **kwargs: Any,
+    **kwargs: Any,  # kwargs-ok: `object` would surface the caller's partially-unknown dict at its call site
 ) -> ResponsesAPIResponse:
     """
     Emulated file_search for providers that don't support it natively.
@@ -507,7 +519,7 @@ async def aresponses_with_emulated_file_search(
     runs vector search, and synthesizes an OpenAI-format response.
     """
     # Determine whether caller wants search_results populated in the output.
-    _include_search_results, kwargs = _prepare_emulated_file_search_call(kwargs=kwargs)
+    _include_search_results, call_kwargs = _prepare_emulated_file_search_call(kwargs=kwargs)
 
     # 1. Replace file_search tools with function tool
     transformed_tools, all_vs_ids = _replace_file_search_tools(tools)
@@ -524,7 +536,7 @@ async def aresponses_with_emulated_file_search(
                 input=input,
                 model=model,
                 tools=transformed_tools or None,
-                **kwargs,
+                **call_kwargs,
             ),
         )
     finally:
@@ -588,7 +600,7 @@ async def aresponses_with_emulated_file_search(
                 input=follow_up_input,
                 model=model,
                 tools=None,  # no tools needed for the answer step
-                **kwargs,
+                **call_kwargs,
             ),
         )
     finally:
diff --git a/litellm/responses/litellm_completion_transformation/custom_tools.py b/litellm/responses/litellm_completion_transformation/custom_tools.py
index fa4ed73a1d6..cccae06c74b 100644
--- a/litellm/responses/litellm_completion_transformation/custom_tools.py
+++ b/litellm/responses/litellm_completion_transformation/custom_tools.py
@@ -16,8 +16,8 @@ logic.
 """
 
 import json
-from collections.abc import Mapping
-from typing import Any, Final
+from collections.abc import Mapping, Sequence
+from typing import Final
 
 from pydantic import BaseModel, TypeAdapter, ValidationError
 
@@ -29,7 +29,7 @@ from litellm.types.llms.openai import (
 _MAX_ARGUMENTS_LEN: Final = 1_000_000
 
 
-def extract_custom_tool_names(tools: list[Any] | None) -> set[str]:
+def extract_custom_tool_names(tools: Sequence[object] | None) -> set[str]:
     """Extract names of tools originally defined as ``type: "custom"``."""
     if not tools:
         return set()
@@ -73,7 +73,7 @@ def build_tool_call_item_kwargs(
     arguments_or_input: str,
     status: str,
     custom_tool_names: set[str],
-) -> dict[str, Any]:
+) -> dict[str, str]:
     """Build kwargs for an output item dict that is either a ``function_call``
     or a ``custom_tool_call`` depending on whether *name* is in
     *custom_tool_names*.
@@ -86,7 +86,7 @@ def build_tool_call_item_kwargs(
     """
     custom: Final = is_custom_tool_call(name, custom_tool_names)
     item_type: Final = "custom_tool_call" if custom else "function_call"
-    kwargs: Final[dict[str, Any]] = {
+    kwargs: Final[dict[str, str]] = {
         "type": item_type,
         "id": call_id,
         "call_id": call_id,
diff --git a/litellm/responses/litellm_completion_transformation/handler.py b/litellm/responses/litellm_completion_transformation/handler.py
index 555e3258773..a0e8cd278e6 100644
--- a/litellm/responses/litellm_completion_transformation/handler.py
+++ b/litellm/responses/litellm_completion_transformation/handler.py
@@ -2,8 +2,8 @@
 Handler for transforming responses api requests to litellm.completion requests
 """
 
-from collections.abc import Coroutine
-from typing import Any, Final
+from collections.abc import Coroutine, Mapping
+from typing import Final
 
 import litellm
 from litellm.responses.litellm_completion_transformation.streaming_iterator import (
@@ -30,12 +30,12 @@ class LiteLLMCompletionTransformationHandler:
         custom_llm_provider: str | None = None,
         _is_async: bool = False,
         stream: bool | None = None,
-        extra_headers: dict[str, Any] | None = None,
+        extra_headers: Mapping[str, object] | None = None,
         **kwargs,
     ) -> (
         ResponsesAPIResponse
         | BaseResponsesAPIStreamingIterator
-        | Coroutine[Any, Any, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator]
+        | Coroutine[object, object, ResponsesAPIResponse | BaseResponsesAPIStreamingIterator]
     ):
         litellm_completion_request: Final[dict] = (
             LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request(
diff --git a/litellm/responses/litellm_completion_transformation/session_handler.py b/litellm/responses/litellm_completion_transformation/session_handler.py
index 1566bb1bdd7..6c66c3fafc6 100644
--- a/litellm/responses/litellm_completion_transformation/session_handler.py
+++ b/litellm/responses/litellm_completion_transformation/session_handler.py
@@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Any, Final, cast
 
 import litellm
 from litellm._logging import verbose_proxy_logger
-from litellm.proxy._types import SpendLogsPayload
+from litellm.proxy._types import SpendLogsMetadata, SpendLogsPayload
 from litellm.proxy.spend_tracking.cold_storage_handler import ColdStorageHandler
 from litellm.responses.utils import ResponsesAPIRequestUtils
 from litellm.types.llms.openai import (
@@ -143,7 +143,7 @@ class ResponsesSessionHandler:
             model_response: Final = ModelResponse(**_response_output)
             for choice in model_response.choices:
                 if hasattr(choice, "message"):
-                    chat_completion_message_history.append(getattr(choice, "message"))
+                    chat_completion_message_history.append(choice.message)
         return chat_completion_message_history
 
     @staticmethod
@@ -195,7 +195,7 @@ class ResponsesSessionHandler:
         try:
             metadata_str: Final = spend_log.get("metadata", "{}")
             if isinstance(metadata_str, str):
-                metadata_dict: Final = json.loads(metadata_str)
+                metadata_dict: Final[SpendLogsMetadata] = json.loads(metadata_str)
                 return metadata_dict.get("cold_storage_object_key")
             elif isinstance(metadata_str, dict):
                 return metadata_str.get("cold_storage_object_key")
diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py
index 92bbca9ee5b..8b1eeb30306 100644
--- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py
+++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py
@@ -1,5 +1,6 @@
 import time
 import uuid
+from collections.abc import Sequence
 from typing import Any, Final, cast
 
 import litellm
@@ -48,14 +49,18 @@ from litellm.types.utils import (
 )
 
 
+def _index_of_output_item_type(items: Sequence[object], item_type: str) -> int | None:
+    return next(
+        (index for index, item in enumerate(items) if getattr(item, "type", None) == item_type),
+        None,
+    )
+
+
 def _output_items_with_id(items: tuple[Any, ...], item_type: str, item_id: str | None) -> tuple[Any, ...]:
     if item_id is None:
         return items
 
-    target_index: Final = next(
-        (index for index, item in enumerate(items) if getattr(item, "type", None) == item_type),
-        None,
-    )
+    target_index: Final = _index_of_output_item_type(items, item_type)
     if target_index is None:
         return items
 
@@ -86,7 +91,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
         self.litellm_metadata: dict | None = litellm_metadata or {}
         # Store lightweight dict snapshots for stream_chunk_builder to reduce
         # repeated Pydantic attribute access in end-of-stream assembly.
-        self.collected_chat_completion_chunks: list[dict[str, Any]] = []
+        self.collected_chat_completion_chunks: list[dict[str, object]] = []
         self.finished: bool = False
         self.litellm_logging_obj = litellm_custom_stream_wrapper.logging_obj
         self.sent_response_created_event: bool = False
@@ -98,7 +103,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
         self.sent_output_item_done_event: bool = False
         self.sent_annotation_events: bool = False
         self.litellm_model_response: ModelResponse | TextCompletionResponse | None = None
-        self.completed_response: Any = None
+        self.completed_response = None
         self.final_text: str = ""
         self._cached_item_id: str | None = None
         self._cached_response_id: str | None = None
@@ -123,7 +128,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
         self._reasoning_done_emitted = False
         self._reasoning_item_id: str | None = None
         self._accumulated_reasoning_content_parts: list[str] = []
-        self._accumulated_provider_specific_fields: dict[str, Any] = {}
+        self._accumulated_provider_specific_fields: dict[str, object] = {}
         self._custom_tool_names: set[str] = extract_custom_tool_names(self.responses_api_request.get("tools"))
         self._namespace_tool_names = LiteLLMCompletionResponsesConfig.namespace_tool_name_map(
             self.responses_api_request.get("tools")
@@ -543,7 +548,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
     @staticmethod
     def _snapshot_chunk_for_stream_chunk_builder(
         chunk: ModelResponseStream,
-    ) -> dict[str, Any]:
+    ) -> dict[str, object]:
         """
         Convert a streaming chunk into a plain dict for end-of-stream assembly.
         Keep _hidden_params so downstream usage/header behavior is preserved.
@@ -1161,7 +1166,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
         if litellm_model_response:
             # Add cost to usage object if include_cost_in_streaming_usage is True
             if litellm.include_cost_in_streaming_usage and self.litellm_logging_obj is not None:
-                usage: Final = getattr(litellm_model_response, "usage", None)
+                usage: Final[object] = getattr(litellm_model_response, "usage", None)
                 if usage is not None:
                     setattr(
                         usage,
diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py
index b8d7b726a28..aba24692d5d 100644
--- a/litellm/responses/litellm_completion_transformation/transformation.py
+++ b/litellm/responses/litellm_completion_transformation/transformation.py
@@ -5,7 +5,7 @@ Handles transforming from Responses API -> LiteLLM completion  (Chat Completion
 import json
 import re
 import uuid
-from collections.abc import Iterator, Mapping, Sequence
+from collections.abc import Iterable, Iterator, Mapping, Sequence
 from types import MappingProxyType
 from typing import (
     TYPE_CHECKING,
@@ -28,7 +28,7 @@ from openai.types.responses import ResponseFunctionToolCall
 from openai.types.responses.response_create_params import ResponseInputParam
 from openai.types.responses.tool_param import FunctionToolParam
 from pydantic import TypeAdapter
-from typing_extensions import TypedDict
+from typing_extensions import ReadOnly, TypedDict
 
 from litellm._logging import verbose_logger
 from litellm.caching import InMemoryCache
@@ -46,6 +46,7 @@ from litellm.types.llms.openai import (
     ChatCompletionRedactedThinkingBlock,
     ChatCompletionResponseMessage,
     ChatCompletionSystemMessage,
+    ChatCompletionTextObject,
     ChatCompletionThinkingBlock,
     ChatCompletionToolCallChunk,
     ChatCompletionToolCallFunctionChunk,
@@ -129,6 +130,30 @@ class _HasId(Protocol):
     id: object
 
 
+class _ResponsesToolCallItem(Protocol):
+    name: object
+    arguments: object
+
+    def get(self, key: str, /) -> object: ...
+
+
+class _ToolFunctionDefinition(TypedDict, total=False):
+    name: ReadOnly[str]
+    description: ReadOnly[str]
+    parameters: ReadOnly[dict[str, object]]
+    strict: ReadOnly[bool | None]
+
+
+def _attribute_fields(value: object) -> dict[str, object]:
+    if not hasattr(value, "__dict__"):
+        return {}  # mutable-ok: provider_specific_fields payload
+    return dict(cast("Iterable[tuple[str, object]]", value))  # cast-ok: dict() raises on non-pair values, as before
+
+
+def _input_item_role(input_item: Mapping[str, object]) -> str:
+    return cast(str, input_item.get("role") or "user")  # cast-ok: client-supplied role forwarded verbatim, unvalidated
+
+
 class ChatCompletionSession(TypedDict, total=False):
     messages: list[
         AllMessageValues
@@ -677,7 +702,7 @@ class LiteLLMCompletionResponsesConfig:
                 existing_text: Final = _reasoning_text(msg)
                 combined: Final = "\n".join(pending_texts + ((existing_text,) if existing_text else ()))
                 if isinstance(msg, dict):
-                    cast(dict[str, Any], msg)["reasoning_content"] = combined  # cast-ok: mutable reasoning carrier
+                    cast(dict[str, object], msg)["reasoning_content"] = combined  # cast-ok: mutable reasoning carrier
                 else:
                     setattr(msg, "reasoning_content", combined)  # noqa: B010  # attribute name is fixed, not dynamic
             if pending_blocks:
@@ -685,7 +710,7 @@ class LiteLLMCompletionResponsesConfig:
                     pending_blocks + (_thinking_blocks(msg) or ())
                 )
                 if isinstance(msg, dict):
-                    cast(dict[str, Any], msg)["thinking_blocks"] = replayed  # cast-ok: mutable reasoning carrier
+                    cast(dict[str, object], msg)["thinking_blocks"] = replayed  # cast-ok: mutable reasoning carrier
                 else:
                     setattr(msg, "thinking_blocks", replayed)  # noqa: B010  # attribute name is fixed, not dynamic
 
@@ -1034,7 +1059,7 @@ class LiteLLMCompletionResponsesConfig:
     def _add_tool_call_to_assistant(assistant_message: object, tool_call_chunk: ChatCompletionToolCallChunk) -> None:
         """Add a tool_call to an assistant message."""
         if isinstance(assistant_message, dict):
-            prev_assistant_dict: Final = cast(dict[str, Any], assistant_message)
+            prev_assistant_dict: Final = cast(dict[str, object], assistant_message)
             if "tool_calls" not in prev_assistant_dict:
                 prev_assistant_dict["tool_calls"] = []
             tool_calls_list: Final = prev_assistant_dict["tool_calls"]
@@ -1119,7 +1144,7 @@ class LiteLLMCompletionResponsesConfig:
                     # Type-safe way to set tool_call_id on tool message
                     if isinstance(message, dict):
                         # Cast to dict to allow setting tool_call_id
-                        message_dict = cast(dict[str, Any], message)
+                        message_dict = cast(dict[str, object], message)
                         message_dict["tool_call_id"] = tool_call_id
                     elif hasattr(message, "tool_call_id"):
                         setattr(message, "tool_call_id", tool_call_id)
@@ -1171,7 +1196,7 @@ class LiteLLMCompletionResponsesConfig:
 
     @staticmethod
     def _transform_responses_api_input_item_to_chat_completion_message(
-        input_item: Any,
+        input_item: Mapping[str, object],
         replay_reasoning: bool = False,
     ) -> list[AllMessageValues | GenericChatCompletionMessage | ChatCompletionResponseMessage]:
         """
@@ -1199,7 +1224,9 @@ class LiteLLMCompletionResponsesConfig:
         elif LiteLLMCompletionResponsesConfig._is_input_item_function_call(input_item):
             # handle function call input items
             return LiteLLMCompletionResponsesConfig._transform_responses_api_function_call_to_chat_completion_message(
-                function_call=input_item
+                function_call=cast(  # cast-ok: callee coerces every field it reads with `or ""` / str()
+                    Mapping[str, str], input_item
+                )
             )
         elif input_item.get("type") == "reasoning":
             # A ResponseReasoningItemParam carries the prior-turn chain-of-thought.
@@ -1224,7 +1251,7 @@ class LiteLLMCompletionResponsesConfig:
                     return []  # mutable-ok: empty drop result
                 return [  # mutable-ok: single message result
                     GenericChatCompletionMessage(
-                        role=input_item.get("role") or "user",
+                        role=_input_item_role(input_item),
                         content=LiteLLMCompletionResponsesConfig._transform_responses_api_content_to_chat_completion_content(
                             inspectable
                         ),
@@ -1252,7 +1279,7 @@ class LiteLLMCompletionResponsesConfig:
                 return []
             return [
                 GenericChatCompletionMessage(
-                    role=input_item.get("role") or "user",
+                    role=_input_item_role(input_item),
                     content=LiteLLMCompletionResponsesConfig._transform_responses_api_content_to_chat_completion_content(
                         content
                     ),
@@ -1339,7 +1366,7 @@ class LiteLLMCompletionResponsesConfig:
         if not isinstance(encrypted_content, str) or not encrypted_content.strip():
             return None
         try:
-            decoded: Final[object] = json.loads(encrypted_content)
+            decoded: Final[object] = cast(object, json.loads(encrypted_content))  # cast-ok: json.loads returns Any
         except ValueError:
             return None
         if not isinstance(decoded, list):
@@ -1406,7 +1433,7 @@ class LiteLLMCompletionResponsesConfig:
 
         def _normalize_function_call_output_to_tool_content(
             output: object,
-        ) -> Any:
+        ) -> str | list[ChatCompletionTextObject | ChatCompletionImageObject]:
             """
             Normalize Responses API function_call_output.output into a shape that downstream
             chat adapters (esp. Gemini) can reliably consume.
@@ -1428,7 +1455,7 @@ class LiteLLMCompletionResponsesConfig:
 
             # Some adapters represent tool output as a list of "input_*" parts
             if isinstance(output, list):
-                normalized_blocks: Final[list[dict[str, object]]] = []
+                normalized_blocks: Final[list[ChatCompletionTextObject | ChatCompletionImageObject]] = []
                 text_acc: Final[list[str]] = []
                 for part in output:
                     if not isinstance(part, dict):
@@ -1899,7 +1926,7 @@ class LiteLLMCompletionResponsesConfig:
                 result.append(tool)
                 continue
             if tool.get("type") == "function":
-                fn = cast(dict[str, Any], tool.get("function") or {})
+                fn = cast(_ToolFunctionDefinition, tool.get("function") or {})
                 parameters = dict(fn.get("parameters", {}) or {})
                 if not parameters or "type" not in parameters:
                     parameters["type"] = "object"
@@ -2095,7 +2122,7 @@ class LiteLLMCompletionResponsesConfig:
 
     @staticmethod
     def convert_response_function_tool_call_to_chat_completion_tool_call(
-        tool_call_item: Any,
+        tool_call_item: object,
         index: int = 0,
     ) -> dict[str, object]:
         """
@@ -2108,24 +2135,25 @@ class LiteLLMCompletionResponsesConfig:
         Returns:
             Dictionary in ChatCompletionToolCallChunk format
         """
+        item: Final = cast(  # cast-ok: duck-typed tool call item, .get access guarded by hasattr below
+            _ResponsesToolCallItem, tool_call_item
+        )
         # Extract provider_specific_fields if present
-        provider_specific_fields = getattr(tool_call_item, "provider_specific_fields", None)
+        provider_specific_fields: object = getattr(tool_call_item, "provider_specific_fields", None)
         if provider_specific_fields and not isinstance(provider_specific_fields, dict):
-            provider_specific_fields = (
-                dict(provider_specific_fields) if hasattr(provider_specific_fields, "__dict__") else {}
-            )
-        elif hasattr(tool_call_item, "get") and callable(tool_call_item.get):
-            provider_fields: Final = tool_call_item.get("provider_specific_fields")
+            provider_specific_fields = _attribute_fields(provider_specific_fields)
+        elif hasattr(tool_call_item, "get") and callable(item.get):
+            provider_fields: Final = item.get("provider_specific_fields")
             if provider_fields:
                 provider_specific_fields = (
-                    provider_fields
+                    cast("dict[str, object]", provider_fields)  # cast-ok: passed through as-is, keys unvalidated
                     if isinstance(provider_fields, dict)
-                    else (dict(provider_fields) if hasattr(provider_fields, "__dict__") else {})
+                    else _attribute_fields(provider_fields)
                 )
 
         function_dict: Final[dict[str, object]] = {
-            "name": tool_call_item.name,
-            "arguments": tool_call_item.arguments,
+            "name": item.name,
+            "arguments": item.arguments,
         }
 
         if provider_specific_fields:
@@ -2306,7 +2334,7 @@ class LiteLLMCompletionResponsesConfig:
         """
         output_items: Final[list] = []
         for choice in chat_completion_response.choices or []:
-            message = getattr(choice, "message", None)
+            message: object = getattr(choice, "message", None)
             if not message:
                 continue
             psf = getattr(message, "provider_specific_fields", None)
@@ -2338,7 +2366,7 @@ class LiteLLMCompletionResponsesConfig:
         for choice in choices:
             if hasattr(choice, "message") and choice.message:
                 message = choice.message
-                reasoning_content = getattr(message, "reasoning_content", None) or ""
+                reasoning_content: str = getattr(message, "reasoning_content", None) or ""
                 encrypted_content = LiteLLMCompletionResponsesConfig._encode_thinking_blocks(message)
                 if reasoning_content or encrypted_content:
                     # Only check the first choice for reasoning content
diff --git a/litellm/responses/main.py b/litellm/responses/main.py
index 34058e8eca7..d6ebc44ac52 100644
--- a/litellm/responses/main.py
+++ b/litellm/responses/main.py
@@ -697,7 +697,7 @@ def _apply_managed_file_id_mapping(
         tools = cast(
             Iterable[ToolParam] | None,
             update_responses_tools_with_model_file_ids(
-                tools=cast(list[dict[str, Any]] | None, tools),
+                tools=cast(list[dict[str, object]] | None, tools),
                 model_id=model_info_id,
                 model_file_id_mapping=model_file_id_mapping,
             ),
@@ -734,7 +734,7 @@ def _responses_try_dispatch_mcp_gateway(
     extra_body: dict[str, object] | None,
     timeout: float | httpx.Timeout | None,
     custom_llm_provider: str | None,
-    kwargs: dict[str, Any],
+    kwargs: dict[str, object],
     _is_async: bool,
 ) -> Any | None:
     """Return a response when MCP gateway handles the call; otherwise None."""
diff --git a/litellm/responses/mcp/chat_completions_handler.py b/litellm/responses/mcp/chat_completions_handler.py
index 2a0406f9a4d..a75b3768636 100644
--- a/litellm/responses/mcp/chat_completions_handler.py
+++ b/litellm/responses/mcp/chat_completions_handler.py
@@ -1,7 +1,9 @@
 """Helpers for handling MCP-aware `/chat/completions` requests."""
 
 import logging
-from typing import TYPE_CHECKING, Any, Final, cast
+from typing import TYPE_CHECKING, Final, cast
+
+from typing_extensions import TypedDict, Unpack
 
 from litellm.responses.mcp.litellm_proxy_mcp_handler import (
     LiteLLM_Proxy_MCP_Handler,
@@ -14,6 +16,10 @@ if TYPE_CHECKING:
     from litellm.proxy._types import UserAPIKeyAuth
 
 
+class _MCPCompletionKwargs(TypedDict, total=False, extra_items=object):
+    """Extra keywords forwarded verbatim to ``litellm.acompletion``, which owns their contract."""
+
+
 def _add_mcp_metadata_to_response(
     response: ModelResponse | CustomStreamWrapper,
     openai_tools: list | None,
@@ -79,7 +85,7 @@ async def acompletion_with_mcp(
     model: str,
     messages: list,
     tools: list | None = None,
-    **kwargs: Any,
+    **kwargs: Unpack[_MCPCompletionKwargs],  # kwargs-ok: forwarded verbatim to litellm.acompletion, which owns them
 ) -> ModelResponse | CustomStreamWrapper:
     """
     Async completion with MCP integration.
@@ -126,7 +132,7 @@ async def acompletion_with_mcp(
     ) = await LiteLLM_Proxy_MCP_Handler._process_mcp_tools_without_openai_transform(
         user_api_key_auth=user_api_key_auth,
         mcp_tools_with_litellm_proxy=mcp_tools_with_litellm_proxy,
-        litellm_trace_id=kwargs.get("litellm_trace_id"),
+        litellm_trace_id=context.litellm_trace_id,
         mcp_auth_header=mcp_auth_header,
         mcp_server_auth_headers=mcp_server_auth_headers,
         request_tags=request_tags,
@@ -168,7 +174,7 @@ async def acompletion_with_mcp(
         return response
 
     # For auto-execute: handle streaming vs non-streaming differently
-    stream: Final[bool] = kwargs.get("stream", False)
+    stream: Final[object] = kwargs.get("stream", False)
     mock_tool_calls: Final = base_call_args.pop("mock_tool_calls", None)
 
     if stream:
@@ -490,8 +496,8 @@ async def acompletion_with_mcp(
             mcp_server_auth_headers=mcp_server_auth_headers,
             oauth2_headers=oauth2_headers,
             raw_headers=raw_headers,
-            litellm_call_id=kwargs.get("litellm_call_id"),
-            litellm_trace_id=kwargs.get("litellm_trace_id"),
+            litellm_call_id=context.litellm_call_id,
+            litellm_trace_id=context.litellm_trace_id,
             openai_tools=openai_tools,
             base_call_args=base_call_args,
             request_tags=request_tags,
@@ -604,8 +610,8 @@ async def acompletion_with_mcp(
         mcp_server_auth_headers=mcp_server_auth_headers,
         oauth2_headers=oauth2_headers,
         raw_headers=raw_headers,
-        litellm_call_id=kwargs.get("litellm_call_id"),
-        litellm_trace_id=kwargs.get("litellm_trace_id"),
+        litellm_call_id=context.litellm_call_id,
+        litellm_trace_id=context.litellm_trace_id,
         request_tags=request_tags,
     )
 
diff --git a/litellm/responses/mcp/mcp_streaming_iterator.py b/litellm/responses/mcp/mcp_streaming_iterator.py
index c7471518398..8f5dc926c68 100644
--- a/litellm/responses/mcp/mcp_streaming_iterator.py
+++ b/litellm/responses/mcp/mcp_streaming_iterator.py
@@ -22,6 +22,8 @@ from litellm.types.llms.openai import (
 )
 
 if TYPE_CHECKING:
+    from collections.abc import AsyncIterator, Iterator
+
     from mcp.types import Tool as MCPTool
 
     from litellm.proxy._types import UserAPIKeyAuth
@@ -511,7 +513,9 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator):
         if self.base_iterator:
             if hasattr(self.base_iterator, "__anext__"):
                 try:
-                    chunk: Final[ResponsesAPIStreamingResponse] = await cast(Any, self.base_iterator).__anext__()
+                    chunk: Final[ResponsesAPIStreamingResponse] = await cast(  # cast-ok: hasattr __anext__ checked
+                        "AsyncIterator[ResponsesAPIStreamingResponse]", self.base_iterator
+                    ).__anext__()
 
                     # Capture the response ID from the first event to ensure consistency
                     if self._cached_response_id is None and hasattr(chunk, "response"):
@@ -569,7 +573,9 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator):
         if not self.base_iterator or not hasattr(self.base_iterator, "__anext__"):
             raise StopAsyncIteration
 
-        chunk: Final[ResponsesAPIStreamingResponse] = await cast(Any, self.base_iterator).__anext__()
+        chunk: Final[ResponsesAPIStreamingResponse] = await cast(  # cast-ok: hasattr __anext__ checked above
+            "AsyncIterator[ResponsesAPIStreamingResponse]", self.base_iterator
+        ).__anext__()
 
         if self._cached_response_id is None and hasattr(chunk, "response"):
             new_response: Final[ResponsesAPIResponse | None] = getattr(chunk, "response", None)
@@ -834,7 +840,9 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator):
         if not self.is_async:
             try:
                 if self.base_iterator and hasattr(self.base_iterator, "__next__"):
-                    return next(cast(Any, self.base_iterator))
+                    return next(
+                        cast("Iterator[ResponsesAPIStreamingResponse]", self.base_iterator)  # cast-ok: hasattr-checked
+                    )
                 else:
                     raise StopIteration
             except StopIteration:
diff --git a/litellm/responses/mcp/request_context.py b/litellm/responses/mcp/request_context.py
index 0689c041a95..22869dcd502 100644
--- a/litellm/responses/mcp/request_context.py
+++ b/litellm/responses/mcp/request_context.py
@@ -10,14 +10,25 @@ still executes the tool, just with no credentials.
 
 from collections.abc import Iterable, Mapping, Sequence
 from dataclasses import dataclass
-from typing import Any, Final
+from typing import TYPE_CHECKING, Any, Final
+
+from typing_extensions import NotRequired, ReadOnly, TypedDict
+
+if TYPE_CHECKING:
+    from litellm.proxy._types import UserAPIKeyAuth
+
+
+class _AuthCarryingMetadata(TypedDict):
+    """The one key this module reads out of a request's ``metadata`` / ``litellm_metadata``."""
+
+    user_api_key_auth: ReadOnly[NotRequired["UserAPIKeyAuth | None"]]
 
 
 @dataclass(frozen=True, slots=True)
 class MCPRequestContext:
     """Everything a gateway handler must forward to MCP tool listing and execution."""
 
-    user_api_key_auth: Any  # any-ok: UserAPIKeyAuth is proxy-only; importing it here would create a cycle
+    user_api_key_auth: "UserAPIKeyAuth | None"
     mcp_auth_header: str | None = None
     mcp_server_auth_headers: Mapping[str, Mapping[str, str]] | None = None
     oauth2_headers: Mapping[str, str] | None = None
@@ -30,7 +41,7 @@ class MCPRequestContext:
     def resolve(
         cls,
         kwargs: Mapping[str, Any],
-        tools: Iterable[Any] | None,
+        tools: Iterable[object] | None,
     ) -> "MCPRequestContext":
         """
         Build the context from a gateway handler's kwargs.
@@ -44,9 +55,9 @@ class MCPRequestContext:
         )
         from litellm.responses.utils import ResponsesAPIRequestUtils
 
-        litellm_metadata: Final = kwargs.get("litellm_metadata") or {}
-        metadata: Final = kwargs.get("metadata") or {}
-        user_api_key_auth: Final = (
+        litellm_metadata: Final[_AuthCarryingMetadata] = kwargs.get("litellm_metadata") or {}
+        metadata: Final[_AuthCarryingMetadata] = kwargs.get("metadata") or {}
+        user_api_key_auth: Final[UserAPIKeyAuth | None] = (
             kwargs.get("user_api_key_auth")
             or litellm_metadata.get("user_api_key_auth")
             or metadata.get("user_api_key_auth")
diff --git a/litellm/responses/sse_output_recovery.py b/litellm/responses/sse_output_recovery.py
index 208dec10c62..adc6a30319c 100644
--- a/litellm/responses/sse_output_recovery.py
+++ b/litellm/responses/sse_output_recovery.py
@@ -8,14 +8,17 @@ caller automatically applies to all of them.
 """
 
 import json
-from typing import Any, Final
+from collections.abc import Mapping
+from typing import Final, SupportsInt, TypeAlias, cast  # noqa: TID251  # int() re-checks the cast below at runtime
 
 from litellm.constants import STREAM_SSE_DONE_STRING
 
 _MAX_CONTENT_INDEX: Final = 1024
 
+_ConvertibleToInt: TypeAlias = SupportsInt | str
 
-def parse_sse_json_chunk(chunk: str) -> dict[str, Any] | None:
+
+def parse_sse_json_chunk(chunk: str) -> dict[str, object] | None:
     """Parse a single raw SSE line into a JSON object dict.
 
     Returns ``None`` for empty lines, ``event:`` lines, ``[DONE]`` markers,
@@ -30,7 +33,7 @@ def parse_sse_json_chunk(chunk: str) -> dict[str, Any] | None:
     if not stripped_chunk or stripped_chunk == STREAM_SSE_DONE_STRING or stripped_chunk.startswith("event:"):
         return None
     try:
-        parsed_chunk: Final = json.loads(stripped_chunk)
+        parsed_chunk: Final[object] = json.loads(stripped_chunk)
     except json.JSONDecodeError:
         return None
     if not isinstance(parsed_chunk, dict):
@@ -38,9 +41,19 @@ def parse_sse_json_chunk(chunk: str) -> dict[str, Any] | None:
     return parsed_chunk
 
 
+def _chunk_index(parsed_chunk: Mapping[str, object], key: str, fallback: int) -> int:
+    raw_index: Final = parsed_chunk.get(key)
+    if raw_index is None:
+        return fallback
+    try:
+        return int(cast(_ConvertibleToInt, raw_index))  # cast-ok: int() raises TypeError otherwise, caught below
+    except (TypeError, ValueError):
+        return fallback
+
+
 def record_output_item_chunk(
-    parsed_chunk: dict[str, Any],
-    output_items: dict[int, dict[str, Any]],
+    parsed_chunk: Mapping[str, object],
+    output_items: dict[int, dict[str, object]],
 ) -> None:
     """Record an OUTPUT_ITEM_DONE chunk into ``output_items`` keyed by
     ``output_index`` (falling back to the next free slot when missing).
@@ -48,20 +61,14 @@ def record_output_item_chunk(
     item: Final = parsed_chunk.get("item")
     if not isinstance(item, dict):
         return
-    try:
-        output_index_raw: Final = parsed_chunk.get("output_index")
-        if output_index_raw is None:
-            raise ValueError("missing output_index")
-        output_index = int(output_index_raw)
-    except (TypeError, ValueError):
-        output_index = len(output_items)
+    output_index: Final = _chunk_index(parsed_chunk, "output_index", len(output_items))
     output_items[output_index] = item
 
 
 def record_output_text_chunk(
-    parsed_chunk: dict[str, Any],
-    output_items: dict[int, dict[str, Any]],
-    text_only_items: dict[int, dict[str, Any]],
+    parsed_chunk: Mapping[str, object],
+    output_items: Mapping[int, dict[str, object]],
+    text_only_items: dict[int, dict[str, object]],
 ) -> None:
     """Record an OUTPUT_TEXT_DONE chunk as a synthetic message item in
     ``text_only_items``. Real OUTPUT_ITEM_DONE events already captured in
@@ -71,13 +78,7 @@ def record_output_text_chunk(
     if not isinstance(text, str):
         return
 
-    try:
-        output_index_raw: Final = parsed_chunk.get("output_index")
-        if output_index_raw is None:
-            raise ValueError("missing output_index")
-        output_index = int(output_index_raw)
-    except (TypeError, ValueError):
-        output_index = len(text_only_items)
+    output_index: Final = _chunk_index(parsed_chunk, "output_index", len(text_only_items))
 
     if output_index in output_items:
         return
@@ -97,13 +98,7 @@ def record_output_text_chunk(
     if not isinstance(content, list):
         return
 
-    try:
-        content_index_raw: Final = parsed_chunk.get("content_index")
-        if content_index_raw is None:
-            raise ValueError("missing content_index")
-        content_index = int(content_index_raw)
-    except (TypeError, ValueError):
-        content_index = len(content)
+    content_index: Final = _chunk_index(parsed_chunk, "content_index", len(content))
 
     if content_index < 0 or content_index > _MAX_CONTENT_INDEX:
         return
diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py
index a6924c1d87a..5c0d6fc536e 100644
--- a/litellm/responses/streaming_iterator.py
+++ b/litellm/responses/streaming_iterator.py
@@ -49,6 +49,21 @@ if TYPE_CHECKING:
         ResponsesClientWebSocket,
     )
 
+    class _StreamCachingHandler(Protocol):
+        """The ``_llm_caching_handler`` attached to a logging object, as this module uses it."""
+
+        original_function: Callable[..., object]
+
+        def _should_store_result_in_cache(
+            self, original_function: Callable[..., object], kwargs: Mapping[str, object]
+        ) -> bool: ...
+
+    class PiiUnmaskingGuardrailCallback(PresidioGuardrailCallback, Protocol):
+        """Guardrail callback that can also reverse its own masking, selected by
+        ``llm_http_handler`` on exactly this attribute."""
+
+        def _unmask_pii_text(self, text: str, pii_tokens: Mapping[str, str]) -> str: ...
+
 
 class ProjectQuotaCallback(Protocol):
     async def enforce_project_io_token_quota_for_frame(
@@ -84,6 +99,11 @@ def _load_json_object(payload: str | bytes) -> dict[str, object]:
     return json.loads(payload)
 
 
+def _load_json_value(payload: str | bytes) -> object:
+    """Parse a JSON payload whose top-level shape the caller narrows itself."""
+    return json.loads(payload)
+
+
 def _model_id_from_metadata(litellm_metadata: dict[str, object] | None) -> str | None:
     model_info: Final = litellm_metadata.get("model_info") if litellm_metadata else None
     model_id: Final = model_info.get("id") if _is_json_object(model_info) else None
@@ -243,10 +263,10 @@ class BaseResponsesAPIStreamingIterator:
 
         try:
             # Parse the JSON chunk
-            parsed_chunk: Final = json.loads(chunk)
+            parsed_chunk: Final = _load_json_value(chunk)
 
             # Format as ResponsesAPIStreamingResponse
-            if isinstance(parsed_chunk, dict):
+            if _is_json_object(parsed_chunk):
                 if self.responses_api_provider_config is None:
                     raise ValueError("responses_api_provider_config is required to process live streaming chunks")
                 openai_responses_api_chunk: Final = self.responses_api_provider_config.transform_streaming_response(
@@ -529,7 +549,7 @@ class BaseResponsesAPIStreamingIterator:
         if response_obj is None:
             return
 
-        caching_handler: Final = getattr(self.logging_obj, "_llm_caching_handler", None)
+        caching_handler: Final[_StreamCachingHandler | None] = getattr(self.logging_obj, "_llm_caching_handler", None)
         if caching_handler is None:
             return
 
@@ -547,7 +567,7 @@ class BaseResponsesAPIStreamingIterator:
         if preset_cache_key is not None:
             request_kwargs["cache_key"] = preset_cache_key
 
-        if not caching_handler._should_store_result_in_cache(
+        if not caching_handler._should_store_result_in_cache(  # pyright: ignore[reportPrivateUsage]  # no public API
             original_function=caching_handler.original_function,
             kwargs=request_kwargs,
         ):
@@ -1401,7 +1421,7 @@ async def _enforce_frame_project_quota(
     if not quota_callbacks:
         return
     try:
-        msg_obj = json.loads(raw_message)
+        msg_obj: Final = _load_json_value(raw_message)
     except (json.JSONDecodeError, TypeError):
         return
     if not _is_json_object(msg_obj) or msg_obj.get("type") != "response.create":
@@ -1451,7 +1471,7 @@ class ResponsesWebSocketStreaming:
         user_api_key_dict: UserAPIKeyAuth | None = None,
         request_data: dict[str, object] | None = None,
         first_message: str | None = None,
-        guardrail_callbacks: list[Any] | None = None,
+        guardrail_callbacks: list[PiiUnmaskingGuardrailCallback] | None = None,
         output_guardrail_callbacks: list[PresidioGuardrailCallback] | None = None,
         quota_callbacks: Sequence[ProjectQuotaCallback] | None = None,
         authorized_model: str | None = None,
@@ -1464,7 +1484,7 @@ class ResponsesWebSocketStreaming:
         self.messages: list[dict[str, object]] = []
         self.input_messages: list[dict[str, object]] = []
         self.first_message = first_message
-        self.guardrail_callbacks: list[Any] = guardrail_callbacks or []
+        self.guardrail_callbacks: list[PiiUnmaskingGuardrailCallback] = guardrail_callbacks or []
         self.output_guardrail_callbacks: list[PresidioGuardrailCallback] = output_guardrail_callbacks or []
         self.quota_callbacks: tuple[ProjectQuotaCallback, ...] = tuple(quota_callbacks) if quota_callbacks else ()
         # Model name authorized at connection time; enforced on every
@@ -1780,7 +1800,9 @@ class ResponsesWebSocketStreaming:
                         continue
                     text = content_block.get("text")
                     if isinstance(text, str):
-                        unmasked = cb._unmask_pii_text(text, pii_tokens)
+                        unmasked = cb._unmask_pii_text(  # pyright: ignore[reportPrivateUsage]  # no public unmasker
+                            text, pii_tokens
+                        )
                         if unmasked != text:
                             content_block["text"] = unmasked
                             modified = True
@@ -1789,7 +1811,9 @@ class ResponsesWebSocketStreaming:
         if event_type in self._DELTA_EVENT_TYPES:
             delta: Final = evt_obj.get("delta")
             if isinstance(delta, str):
-                unmasked = cb._unmask_pii_text(delta, pii_tokens)
+                unmasked = cb._unmask_pii_text(  # pyright: ignore[reportPrivateUsage]  # no public unmasker
+                    delta, pii_tokens
+                )
                 if unmasked != delta:
                     evt_obj["delta"] = unmasked
                     return json.dumps(evt_obj)
diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py
index 716a815547d..0ff6bc8a7d2 100644
--- a/litellm/responses/utils.py
+++ b/litellm/responses/utils.py
@@ -1,15 +1,17 @@
 import base64
 import re
-from collections.abc import Iterable, Mapping
+from collections.abc import Iterable, Mapping, Sequence
 from typing import Any, Final, Optional, Union, cast, get_type_hints, overload
 
 from pydantic import BaseModel
+from typing_extensions import TypeIs  # noqa: TID251  # narrows untyped wire payloads without a runtime conversion
 
 import litellm
 from litellm._logging import verbose_logger
 from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
 from litellm.types.llms.openai import (
     AllMessageValues,
+    OutputTokensDetails,
     ResponseAPIUsage,
     ResponseInputParam,
     ResponsesAPIOptionalRequestParams,
@@ -26,6 +28,16 @@ from litellm.types.utils import (
 )
 
 
+def _is_object_sequence(value: object) -> TypeIs[Sequence[object]]:  # guard-ok: a list is a Sequence of anything
+    return isinstance(value, list)
+
+
+def _is_object_dict(
+    value: object,
+) -> TypeIs[dict[str, object]]:  # guard-ok: wire dicts have str keys  # mutable-ok: callers rewrite ids in place
+    return isinstance(value, dict)
+
+
 def normalize_responses_api_stream_options(
     stream_options: object,
 ) -> ResponsesAPIStreamOptions | None:
@@ -703,12 +715,12 @@ class ResponsesAPIRequestUtils:
 
     @staticmethod
     def _encode_container_ids_in_annotations(
-        annotations: Any,
+        annotations: object,
         custom_llm_provider: str | None,
         model_id: str | None,
     ) -> None:
         """Encode ``container_id`` on each annotation (e.g. ``container_file_citation``)."""
-        if not annotations or not isinstance(annotations, list):
+        if not annotations or not _is_object_sequence(annotations):
             return
         for ann in annotations:
             ResponsesAPIRequestUtils._encode_container_id_on_output_item(
@@ -719,16 +731,16 @@ class ResponsesAPIRequestUtils:
 
     @staticmethod
     def _encode_container_ids_in_message_content(
-        content: Any,
+        content: object,
         custom_llm_provider: str | None,
         model_id: str | None,
     ) -> None:
         """Walk message ``content`` parts and encode citation ``container_id`` values."""
         if not content:
             return
-        if isinstance(content, list):
+        if _is_object_sequence(content):
             for part in content:
-                if isinstance(part, dict):
+                if _is_object_dict(part):
                     ResponsesAPIRequestUtils._encode_container_ids_in_annotations(
                         part.get("annotations"),
                         custom_llm_provider,
@@ -743,7 +755,7 @@ class ResponsesAPIRequestUtils:
 
     @staticmethod
     def _encode_container_id_on_output_item(
-        item: Any,
+        item: object,
         custom_llm_provider: str | None,
         model_id: str | None,
     ) -> None:
@@ -770,14 +782,14 @@ class ResponsesAPIRequestUtils:
                 container_id=container_id,
             )
 
-        if isinstance(item, dict):
+        if _is_object_dict(item):
             cid: Final = item.get("container_id")
             if isinstance(cid, str):
                 enc = _maybe_encode(cid)
                 if enc is not None:
-                    item["container_id"] = enc
+                    item["container_id"] = enc  # rebind-ok: this helper's contract is to rewrite the item in place
             nested: Final = item.get("code_interpreter_call")
-            if isinstance(nested, dict):
+            if _is_object_dict(nested):
                 nc: Final = nested.get("container_id")
                 if isinstance(nc, str):
                     enc = _maybe_encode(nc)
@@ -803,7 +815,7 @@ class ResponsesAPIRequestUtils:
                         exc_info=True,
                     )
 
-        nested_obj: Final = getattr(item, "code_interpreter_call", None)
+        nested_obj: Final[object] = getattr(item, "code_interpreter_call", None)
         if nested_obj is not None:
             ResponsesAPIRequestUtils._encode_container_id_on_output_item(
                 nested_obj,
@@ -820,24 +832,24 @@ class ResponsesAPIRequestUtils:
 
     @staticmethod
     def _collect_container_ids_from_annotations(
-        annotations: Any,
+        annotations: object,
         collected: set[str],
     ) -> None:
-        if not annotations or not isinstance(annotations, list):
+        if not annotations or not _is_object_sequence(annotations):
             return
         for ann in annotations:
             ResponsesAPIRequestUtils._collect_container_ids_from_output_item(ann, collected)
 
     @staticmethod
     def _collect_container_ids_from_message_content(
-        content: Any,
+        content: object,
         collected: set[str],
     ) -> None:
         if not content:
             return
-        if isinstance(content, list):
+        if _is_object_sequence(content):
             for part in content:
-                if isinstance(part, dict):
+                if _is_object_dict(part):
                     ResponsesAPIRequestUtils._collect_container_ids_from_annotations(
                         part.get("annotations"),
                         collected,
@@ -850,19 +862,19 @@ class ResponsesAPIRequestUtils:
 
     @staticmethod
     def _collect_container_ids_from_output_item(
-        item: Any,
+        item: object,
         collected: set[str],
     ) -> None:
         """Collect managed or raw ``container_id`` values from one output item."""
         if item is None:
             return
 
-        if isinstance(item, dict):
+        if _is_object_dict(item):
             cid: Final = item.get("container_id")
             if isinstance(cid, str) and cid:
                 collected.add(cid)
             nested: Final = item.get("code_interpreter_call")
-            if isinstance(nested, dict):
+            if _is_object_dict(nested):
                 nc: Final = nested.get("container_id")
                 if isinstance(nc, str) and nc:
                     collected.add(nc)
@@ -877,7 +889,7 @@ class ResponsesAPIRequestUtils:
         if isinstance(cid_attr, str) and cid_attr:
             collected.add(cid_attr)
 
-        nested_obj: Final = getattr(item, "code_interpreter_call", None)
+        nested_obj: Final[object] = getattr(item, "code_interpreter_call", None)
         if nested_obj is not None:
             ResponsesAPIRequestUtils._collect_container_ids_from_output_item(nested_obj, collected)
 
@@ -1108,7 +1120,9 @@ class ResponseAPILoggingUtils:
                     cache_write_tokens=getattr(response_api_usage.input_tokens_details, "cache_write_tokens", None),
                 )
         completion_tokens_details: CompletionTokensDetailsWrapper | None = None
-        output_tokens_details: Final = getattr(response_api_usage, "output_tokens_details", None)
+        output_tokens_details: Final[OutputTokensDetails | None] = getattr(
+            response_api_usage, "output_tokens_details", None
+        )
         if output_tokens_details:
             completion_tokens_details = CompletionTokensDetailsWrapper(
                 reasoning_tokens=getattr(output_tokens_details, "reasoning_tokens", None),
diff --git a/litellm/vector_stores/vector_store_registry.py b/litellm/vector_stores/vector_store_registry.py
index bd9a7bff101..22d27bc3266 100644
--- a/litellm/vector_stores/vector_store_registry.py
+++ b/litellm/vector_stores/vector_store_registry.py
@@ -1,7 +1,14 @@
 # litellm/proxy/vector_stores/vector_store_registry.py
 import json
+from collections.abc import Mapping
 from datetime import datetime, timezone
-from typing import TYPE_CHECKING, Any, Final, get_args
+from typing import (
+    TYPE_CHECKING,
+    Any,  # noqa: TID251  # untyped non_default_params dict is the only source of the unknown key type
+    Final,
+    cast,  # noqa: TID251  # untyped non_default_params dict is the only source of the unknown key type
+    get_args,
+)
 
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.core_helpers import remove_items_at_indices
@@ -336,7 +343,9 @@ class VectorStoreRegistry:
                 try:
                     # Check if it still exists in database
                     db_vector_store = await ManagedVectorStoresRepository(prisma_client).table.find_unique(
-                        where={"vector_store_id": vector_store_id}
+                        where=cast(  # cast-ok: every value is already an object, only the popped id is stub-untyped
+                            "Mapping[str, object]", {"vector_store_id": vector_store_id}
+                        )
                     )
                     if db_vector_store is None:
                         # Vector store was deleted from database, remove from cache
diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json
index 03318718fb5..d5cddbb2a20 100644
--- a/ruff-strict-budget.json
+++ b/ruff-strict-budget.json
@@ -1,6 +1,6 @@
 {
   "ANN001": {
-    "limit": 3018
+    "limit": 3016
   },
   "ANN002": {
     "limit": 71
@@ -9,7 +9,7 @@
     "limit": 827
   },
   "ANN201": {
-    "limit": 2016
+    "limit": 2013
   },
   "ANN202": {
     "limit": 852
@@ -24,7 +24,7 @@
     "limit": 133
   },
   "ANN401": {
-    "limit": 1188
+    "limit": 1157
   },
   "ASYNC230": {
     "limit": 11
@@ -33,13 +33,13 @@
     "limit": 2
   },
   "B006": {
-    "limit": 177
+    "limit": 176
   },
   "B008": {
     "limit": 503
   },
   "B009": {
-    "limit": 59
+    "limit": 58
   },
   "B010": {
     "limit": 190
@@ -231,7 +231,7 @@
     "limit": 5
   },
   "TID251": {
-    "limit": 1212
+    "limit": 1201
   },
   "TRY002": {
     "limit": 524
diff --git a/tests/proxy_unit_tests/test_jwt_key_mapping.py b/tests/proxy_unit_tests/test_jwt_key_mapping.py
index 65d075b7f99..4b50f83e9eb 100644
--- a/tests/proxy_unit_tests/test_jwt_key_mapping.py
+++ b/tests/proxy_unit_tests/test_jwt_key_mapping.py
@@ -416,6 +416,34 @@ async def test_update_returns_404_when_not_found():
         assert exc_info.value.status_code == 404
 
 
+@pytest.mark.asyncio
+async def test_update_returns_404_when_row_deleted_before_write():
+    """A mapping deleted between the read and the write must 404, not 500.
+
+    Prisma's update returns None when the row is gone, and the endpoint used to
+    dereference it for the cache key.
+    """
+    from litellm.proxy._types import UpdateJWTKeyMappingRequest
+
+    mock_prisma = _mock_prisma()
+    mock_prisma.db.litellm_jwtkeymapping.find_unique.return_value = _mock_mapping()
+    mock_prisma.db.litellm_jwtkeymapping.update.return_value = None
+    mock_cache = AsyncMock()
+
+    data = UpdateJWTKeyMappingRequest(id="mapping-1", description="test")
+
+    with (
+        patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),  # test-quality-ok: proxy_server module global is the endpoint's only injection point
+        patch("litellm.proxy.proxy_server.user_api_key_cache", mock_cache),  # test-quality-ok: proxy_server module global is the endpoint's only injection point
+    ):
+        with pytest.raises(HTTPException) as exc_info:
+            await update_jwt_key_mapping(
+                data=data, user_api_key_dict=_make_admin_auth()
+            )
+        assert exc_info.value.status_code == 404
+        assert exc_info.value.detail == "Mapping not found"
+
+
 @pytest.mark.asyncio
 async def test_info_returns_404_when_not_found():
     """Getting info for non-existent mapping should return 404."""
diff --git a/tests/test_litellm/proxy/agent_endpoints/test_agent_registry.py b/tests/test_litellm/proxy/agent_endpoints/test_agent_registry.py
index a87f5384c6f..7ce62fdf648 100644
--- a/tests/test_litellm/proxy/agent_endpoints/test_agent_registry.py
+++ b/tests/test_litellm/proxy/agent_endpoints/test_agent_registry.py
@@ -428,3 +428,60 @@ async def test_migrate_legacy_grant_ids_no_ops_without_config_agents():
 
     assert await registry.migrate_legacy_grant_ids(table=table) == GrantMigrationResult(rewritten=0, missed=0)
     table.find_many.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_update_agent_in_db_raises_when_row_deleted_mid_update():
+    """Prisma's update returns None when the row vanished between read and write. Without a
+    guard the code dereferences None and reports an opaque AttributeError instead of the id."""
+    registry: Final = AgentRegistry()
+    mock_prisma: Final = MagicMock()
+    mock_prisma.db.litellm_agentstable.update = AsyncMock(return_value=None)
+
+    with pytest.raises(Exception, match="Error updating agent in DB") as exc_info:
+        await registry.update_agent_in_db(
+            agent_id="agent-123",
+            agent={
+                "agent_name": "Updated Agent",
+                "agent_card_params": _sample_agent_card_params(),
+                "litellm_params": {},
+            },
+            prisma_client=mock_prisma,
+            updated_by="test-user",
+        )
+
+    assert str(exc_info.value) == "Error updating agent in DB: Agent not found, passed agent_id=agent-123"
+
+
+@pytest.mark.asyncio
+async def test_patch_agent_in_db_raises_when_row_deleted_mid_update():
+    """Same race on PATCH: the existing row is read, then deleted before the update lands."""
+    registry: Final = AgentRegistry()
+    mock_prisma: Final = MagicMock()
+    mock_prisma.db.litellm_agentstable.find_unique = AsyncMock(
+        return_value={"agent_id": "agent-123", "agent_name": "Old Agent", "object_permission_id": None}
+    )
+    mock_prisma.db.litellm_agentstable.update = AsyncMock(return_value=None)
+
+    with pytest.raises(Exception, match="Error patching agent in DB") as exc_info:
+        await registry.patch_agent_in_db(
+            agent_id="agent-123",
+            agent={"agent_name": "Patched Agent"},
+            prisma_client=mock_prisma,
+            updated_by="test-user",
+        )
+
+    assert str(exc_info.value) == "Error patching agent in DB: Agent not found, passed agent_id=agent-123"
+
+
+@pytest.mark.asyncio
+async def test_delete_agent_from_db_raises_when_row_already_gone():
+    """Prisma's delete returns None for a missing row, which dict() cannot consume."""
+    registry: Final = AgentRegistry()
+    mock_prisma: Final = MagicMock()
+    mock_prisma.db.litellm_agentstable.delete = AsyncMock(return_value=None)
+
+    with pytest.raises(Exception, match="Error deleting agent from DB") as exc_info:
+        await registry.delete_agent_from_db(agent_id="agent-123", prisma_client=mock_prisma)
+
+    assert str(exc_info.value) == "Error deleting agent from DB: Agent not found, passed agent_id=agent-123"
diff --git a/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_marketplace.py b/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_marketplace.py
index d1ccd86044c..99b09f48fe5 100644
--- a/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_marketplace.py
+++ b/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_marketplace.py
@@ -231,6 +231,30 @@ async def test_update_plugin_db_error_maps_to_structured_500():
     assert "connection lost" in exc_info.value.detail["error"]
 
 
+@pytest.mark.asyncio
+async def test_update_plugin_deleted_mid_update_returns_404():
+    """A concurrent delete between the find_unique pre-check and the update makes prisma's
+    update return None; that must surface the same 404 as a plain miss, not an AttributeError."""
+    name = "my-monorepo-plugin"
+    await register_plugin(
+        request=RegisterPluginRequest(name=name, source=_GIT_SUBDIR_SOURCE, version="1.0.0"),
+        user_api_key_dict=_USER,
+    )
+
+    table = litellm.proxy.proxy_server.prisma_client.db.litellm_claudecodeplugintable
+    table.update = AsyncMock(return_value=None)
+
+    with pytest.raises(HTTPException) as exc_info:
+        await update_plugin(
+            plugin_name=name,
+            request=UpdatePluginRequest(source={"source": "github", "repo": "org/replacement"}),
+            user_api_key_dict=_USER,
+        )
+
+    assert exc_info.value.status_code == 404
+    assert exc_info.value.detail == {"error": f"Plugin '{name}' not found"}
+
+
 @pytest.mark.asyncio
 async def test_get_marketplace_skips_plugin_with_null_manifest():
     await register_plugin(
diff --git a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py
index 25c177a308d..c575265ed91 100644
--- a/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py
+++ b/tests/test_litellm/proxy/common_utils/test_reset_budget_job.py
@@ -695,7 +695,7 @@ def test_reset_budget_resets_endusers_with_null_budget_id(reset_budget_job, mock
             "object_permission_id": None,
             "object_permission": None,
             "litellm_budget_table": None,
-            "dict": lambda self=None: {
+            "model_dump": lambda self=None: {
                 "spend": 25.0,
                 "user_id": "enduser-implicit",
                 "blocked": False,
diff --git a/tests/test_litellm/proxy/db/mcp_server/test_db.py b/tests/test_litellm/proxy/db/mcp_server/test_db.py
index aa40ec0d76c..e2440e49f19 100644
--- a/tests/test_litellm/proxy/db/mcp_server/test_db.py
+++ b/tests/test_litellm/proxy/db/mcp_server/test_db.py
@@ -4,7 +4,11 @@ from unittest.mock import AsyncMock, MagicMock
 import pytest
 
 
-from litellm.proxy._experimental.mcp_server.db import get_mcp_servers_by_team
+from litellm.proxy._experimental.mcp_server.db import (
+    approve_mcp_server,
+    get_mcp_servers_by_team,
+    reject_mcp_server,
+)
 
 
 def _prisma_client_returning(team_record: object) -> MagicMock:
@@ -38,3 +42,30 @@ async def test_fetch_mcp_servers_by_team(team_record, expected):
         where={"team_id": "team-123"},
         include={"object_permission": True},
     )
+
+
+def _prisma_client_with_missing_mcp_server_row() -> MagicMock:
+    prisma_client = MagicMock()
+    prisma_client.db.litellm_mcpservertable.update = AsyncMock(return_value=None)
+    return prisma_client
+
+
+@pytest.mark.asyncio
+async def test_approve_mcp_server_raises_value_error_when_row_missing():
+    prisma_client = _prisma_client_with_missing_mcp_server_row()
+
+    with pytest.raises(ValueError, match=r"^MCP server not found, passed server_id=server-gone$"):
+        await approve_mcp_server(prisma_client, "server-gone", touched_by="admin")
+
+
+@pytest.mark.asyncio
+async def test_reject_mcp_server_raises_value_error_when_row_missing():
+    prisma_client = _prisma_client_with_missing_mcp_server_row()
+
+    with pytest.raises(ValueError, match=r"^MCP server not found, passed server_id=server-gone$"):
+        await reject_mcp_server(
+            prisma_client,
+            "server-gone",
+            touched_by="admin",
+            review_notes="spam",
+        )
diff --git a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py
index 729dbce6b9a..26b3890464e 100644
--- a/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py
+++ b/tests/test_litellm/proxy/guardrails/test_guardrail_registry.py
@@ -1,8 +1,11 @@
+from unittest.mock import AsyncMock, MagicMock
+
 import pytest
 
 from litellm.integrations.custom_guardrail import CustomGuardrail
 from litellm.proxy.guardrails.guardrail_registry import (
     get_guardrail_initializer_from_hooks,
+    GuardrailRegistry,
     InMemoryGuardrailHandler,
 )
 from litellm.types.guardrails import GuardrailEventHooks, Guardrail, LitellmParams
@@ -657,3 +660,22 @@ class TestScanOnlyToolResultsInitRefusal:
                     "scan_only_tool_results": True,
                 },
             )
+
+
+@pytest.mark.asyncio
+async def test_update_guardrail_in_db_raises_when_row_missing():
+    prisma_client = MagicMock()
+    prisma_client.db.litellm_guardrailstable.update = AsyncMock(return_value=None)
+
+    with pytest.raises(
+        Exception,
+        match=r"^Error updating guardrail in DB: Guardrail not found, passed guardrail_id=missing-guardrail$",
+    ):
+        await GuardrailRegistry().update_guardrail_in_db(
+            guardrail_id="missing-guardrail",
+            guardrail=Guardrail(
+                guardrail_name="missing-guardrail",
+                litellm_params=LitellmParams(guardrail="bedrock", mode="pre_call"),
+            ),
+            prisma_client=prisma_client,
+        )
diff --git a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py
index 5f6c1a2375b..6d8f7f4ccdb 100644
--- a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py
@@ -5446,3 +5446,46 @@ async def test_handle_group_membership_changes_already_in_team_is_noop(mocker):
     )
 
     assert mock_team_member_add.await_count == 2
+
+
+@pytest.mark.asyncio
+async def test_patch_group_404s_when_team_deleted_mid_request(mocker):
+    """A group deleted between the existence check and the write must 404.
+
+    Prisma returns None from both the update and the refresh reads once the row is
+    gone, and patch_group used to dereference that None while building the response.
+    """
+    group_id = "team-gone"
+
+    snapshot_team = LiteLLM_TeamTable(
+        team_id=group_id,
+        team_alias="Group",
+        members_with_roles=[Member(user_id="zed", role="user")],
+        metadata={"externalId": "grp-ext"},
+    )
+
+    patch_ops = SCIMPatchOp(
+        schemas=["urn:ietf:params:scim:api:messages:2.0:PatchOp"],
+        Operations=[SCIMPatchOperation(op="replace", path="displayName", value="Renamed")],
+    )
+
+    mock_prisma_client = mocker.MagicMock()
+    mock_prisma_client.db = mocker.MagicMock()
+    mock_prisma_client.db.litellm_teamtable = mocker.MagicMock()
+    mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock(side_effect=[snapshot_team, None, None])
+    mock_prisma_client.db.litellm_teamtable.update = AsyncMock(return_value=None)
+
+    mocker.patch(  # test-quality-ok: stubs the collaborator so the test pins the endpoint's own error contract
+        "litellm.proxy.management_endpoints.scim.scim_v2._get_prisma_client_or_raise_exception",
+        AsyncMock(return_value=mock_prisma_client),
+    )
+    mocker.patch(  # test-quality-ok: stubs the collaborator so the test pins the endpoint's own error contract
+        "litellm.proxy.management_endpoints.scim.scim_v2._recompute_scim_member_roles",
+        AsyncMock(),
+    )
+
+    with pytest.raises(ProxyException) as exc_info:
+        await patch_group(group_id=group_id, patch_ops=patch_ops)
+
+    assert exc_info.value.code == "404"
+    assert f"Group not found with ID: {group_id}" in exc_info.value.message
diff --git a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py
index f86e17c61b0..8bce967b316 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py
@@ -2398,7 +2398,7 @@ async def test_delete_user_cleans_up_created_by_invitation_links(mocker):
     mock_user_row.user_id = "admin-creator"
     mock_user_row.user_email = "admin@example.com"
     mock_user_row.teams = []
-    mock_user_row.json.return_value = "{}"
+    mock_user_row.model_dump_json.return_value = "{}"
     mock_user_row.model_dump.return_value = {
         "user_id": "admin-creator",
         "user_email": "admin@example.com",
diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py
index 0c615cbaa32..a37c4f72b3d 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py
@@ -7766,6 +7766,45 @@ async def test_validate_key_list_check_key_hash_not_found():
     assert "Key Hash not found" in exc_info.value.message
 
 
+@pytest.mark.asyncio
+async def test_validate_key_list_check_key_hash_row_missing():
+    """A key_hash with no row reaches the same 'Key Hash not found' 403 as a failed
+    lookup, instead of blowing up inside the ownership check on a None row."""
+    mock_prisma_client = AsyncMock()
+    mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(
+        return_value=LiteLLM_UserTable(
+            user_id="test-user",
+            user_email="test@example.com",
+            teams=[],
+            organization_memberships=[],
+        )
+    )
+    mock_prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(
+        return_value=None
+    )
+
+    user_api_key_dict = UserAPIKeyAuth(
+        user_role=LitellmUserRoles.INTERNAL_USER,
+        user_id="test-user",
+        api_key="sk-caller",
+    )
+
+    with pytest.raises(ProxyException) as exc_info:
+        await validate_key_list_check(
+            user_api_key_dict=user_api_key_dict,
+            user_id=None,
+            team_id=None,
+            organization_id=None,
+            key_alias=None,
+            key_hash="hash-of-a-deleted-key",
+            prisma_client=mock_prisma_client,
+        )
+
+    assert exc_info.value.code == "403" or exc_info.value.code == 403
+    assert exc_info.value.param == "key_hash"
+    assert "Key Hash not found" in exc_info.value.message
+
+
 @pytest.mark.asyncio
 async def test_validate_key_list_check_proxy_admin_viewer_skips_db_lookup():
     """proxy_admin_viewer takes the same unscoped read fast-path as proxy_admin, so no
diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py
index 097230108d4..eeb1b2d50e6 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py
@@ -3312,6 +3312,61 @@ class TestPatchModelBlockedAuthGate:
             mock_prisma.db.litellm_proxymodeltable.update.assert_awaited_once()
 
 
+class TestPatchModelRowDeletedBeforeWrite:
+    """A row deleted between the read and the update makes prisma's `update`
+    return None. That must surface patch_model's own 404 not-found contract,
+    not a 500 from dereferencing the missing row."""
+
+    @pytest.mark.asyncio
+    async def test_patch_model_404s_when_update_returns_none(self):
+        from litellm.proxy.management_endpoints.model_management_endpoints import (
+            patch_model,
+        )
+        from litellm.proxy.proxy_server import ProxyException
+
+        admin = UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN)
+        existing_row = MagicMock()
+        existing_row.litellm_params = {"model": "openai/gpt-4o-mini"}
+        existing_row.model_dump.return_value = {
+            "model_name": "gpt-4o-mini",
+            "litellm_params": existing_row.litellm_params,
+            "model_info": {"id": "m1"},
+        }
+        existing_row.model_dump_json.return_value = "{}"
+
+        mock_prisma = MagicMock()
+        mock_prisma.db.litellm_proxymodeltable.find_unique = AsyncMock(
+            return_value=existing_row
+        )
+        mock_prisma.db.litellm_proxymodeltable.update = AsyncMock(return_value=None)
+
+        with (
+            patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),  # test-quality-ok: proxy_server module global is the endpoint's only injection point
+            patch("litellm.proxy.proxy_server.llm_router", MagicMock(**{"get_model_ids.return_value": ["m1"]})),  # test-quality-ok: proxy_server module global is the endpoint's only injection point
+            patch("litellm.proxy.proxy_server.store_model_in_db", True),  # test-quality-ok: proxy_server module global is the endpoint's only injection point
+            patch("litellm.proxy.proxy_server.premium_user", True),  # test-quality-ok: proxy_server module global is the endpoint's only injection point
+            patch(  # test-quality-ok: stubs the auth gate so the test exercises the not-found branch under test
+                "litellm.proxy.management_endpoints.model_management_endpoints.ModelManagementAuthChecks.can_user_make_model_call",
+                new=AsyncMock(return_value=None),
+            ),
+            patch(  # test-quality-ok: stubs the cache write so the test observes only the DB result handling
+                "litellm.proxy.management_endpoints.model_management_endpoints.clear_cache",
+                new=AsyncMock(
+                    return_value=ReconcileOutcome(still_desired=None, live_after=None)
+                ),
+            ),
+        ):
+            with pytest.raises(ProxyException) as exc_info:
+                await patch_model(
+                    model_id="m1",
+                    patch_data=updateDeployment(blocked=True),
+                    user_api_key_dict=admin,
+                )
+
+        assert exc_info.value.code == "404"
+        assert exc_info.value.message == "Model m1 not found on proxy."
+
+
 class TestWriteSurfacesReloadDrop:
     """A model-write endpoint may report success only if every row it wrote is, after the
     reload it triggered, live in this pod's router or deliberately environment-inactive."""
diff --git a/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py
index a62c98e56a7..e2d89a660c2 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py
@@ -1037,3 +1037,29 @@ async def test_get_organization_daily_activity_non_admin_without_org_admin_role_
 
     assert get_daily_activity_mock.call_args.kwargs["entity_id"] == []
     assert org_table_find_many.call_args.kwargs["where"] == {"organization_id": {"in": []}}
+
+
+@pytest.mark.asyncio
+async def test_find_member_if_email_missing_row_raises_documented_400():
+    """A user_email lookup that matches nothing returns None instead of raising, so the
+    only failure the surrounding try/except models is never entered. Without an explicit
+    None guard the next line dereferences None and /organization/member_add answers with
+    an AttributeError-driven 500 rather than the documented 400.
+    """
+    from litellm.proxy.management_endpoints.organization_endpoints import (
+        find_member_if_email,
+    )
+
+    prisma_client = AsyncMock()
+    prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)
+
+    with pytest.raises(HTTPException) as exc_info:
+        await find_member_if_email("missing@example.com", prisma_client)
+
+    assert exc_info.value.status_code == 400
+    assert exc_info.value.detail == {
+        "error": (
+            "Unique user not found for user_email=missing@example.com. Potential duplicate OR "
+            "non-existent user_email in LiteLLM_UserTable. Use 'user_id' instead."
+        )
+    }
diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py
index c2610d88927..08e931e6405 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py
@@ -1169,6 +1169,44 @@ async def test_delete_team_callback_404s_for_unknown_team():
     mock_prisma.db.litellm_teamtable.update.assert_not_called()
 
 
+@pytest.mark.asyncio
+async def test_add_team_callbacks_rejects_team_deleted_before_write():
+    """A team deleted between the existence check and the write must be rejected.
+
+    Prisma's update returns None for a row that is gone, and add_team_callbacks
+    used to hand that None to the cache refresh and report success with a null
+    body. The rejection reuses this endpoint's own missing-team contract, so a
+    caller sees the same 400 whether the team vanished before or after the read.
+    """
+    mock_prisma = _patch_prisma(_team_row(team_id="team-1", metadata={}))
+    mock_prisma.db.litellm_teamtable.update = AsyncMock(return_value=None)
+
+    data = AddTeamCallback(
+        callback_name="langfuse",
+        callback_type="success",
+        callback_vars={
+            "langfuse_public_key": "pk-demo",
+            "langfuse_secret_key": "sk-demo",
+        },
+    )
+
+    with (
+        patch("litellm.proxy.proxy_server.prisma_client", mock_prisma),  # test-quality-ok: proxy_server module global is the endpoint's only injection point
+        patch("litellm.proxy.proxy_server.master_key", None),  # test-quality-ok: proxy_server module global is the endpoint's only injection point
+    ):
+        with pytest.raises(HTTPException) as exc:
+            await add_team_callbacks(
+                data=data,
+                http_request=MagicMock(spec=Request),
+                team_id="team-1",
+                user_api_key_dict=_admin_auth(),
+            )
+
+    mock_prisma.db.litellm_teamtable.update.assert_called_once()
+    assert exc.value.status_code == 400
+    assert exc.value.detail == {"error": "Team id = team-1 does not exist. Please use a different team id."}
+
+
 @pytest.mark.asyncio
 async def test_delete_team_callback_keeps_last_removal_from_reviving_legacy_shape():
     """Removing the last entry must leave metadata["logging"] present and empty.
diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py
index f6d74a189bc..fbe856f4adf 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py
@@ -3,7 +3,7 @@ import json
 from contextlib import asynccontextmanager
 from datetime import datetime, timezone
 from types import SimpleNamespace
-from typing import Optional, cast
+from typing import Final, Optional, cast
 from unittest.mock import AsyncMock, MagicMock, call, patch
 
 import pytest
@@ -2078,6 +2078,100 @@ async def test_team_model_add_delete_refresh_team_cache(endpoint_name):
         assert update_call_kwargs.get("include", {}).get("object_permission") is True
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "endpoint_name",
+    ["team_model_add", "team_model_delete", "update_team_member_permissions"],
+)
+async def test_team_write_404s_when_row_vanishes_before_update(endpoint_name):
+    """A team deleted between the read and the write must 404.
+
+    Prisma's `update` returns None when no row matches `where`, and the team
+    row can be deleted between the read these endpoints do first and the
+    update that follows it. Without the guard, `team_model_add` /
+    `team_model_delete` hand that None to `_refresh_cached_team` (which
+    reads `team_row.team_id`) and `/team/permissions_update` returns None
+    out of a route declared to return a team, so a plain race turns into a
+    500 instead of the 404 every other not-found path in this file raises.
+    """
+    from unittest.mock import AsyncMock, MagicMock, Mock, patch
+
+    from fastapi import Request
+
+    from litellm.proxy._types import (
+        LitellmUserRoles,
+        TeamModelAddRequest,
+        TeamModelDeleteRequest,
+        UserAPIKeyAuth,
+    )
+    from litellm.proxy.management_endpoints.team_endpoints import (
+        team_model_add,
+        team_model_delete,
+        update_team_member_permissions,
+    )
+
+    mock_request = Mock(spec=Request)
+    mock_user_api_key_dict = UserAPIKeyAuth(
+        user_role=LitellmUserRoles.PROXY_ADMIN, user_id="test_user_id"
+    )
+
+    existing_team = MagicMock()
+    existing_team.team_id = "team-1234"
+    existing_team.model_dump.return_value = {
+        "team_id": "team-1234",
+        "models": ["bedrock-claude-sonnet-4", "openai/*"],
+        "team_member_permissions": [],
+        "spend": 0.0,
+    }
+
+    call_endpoint_under_test: Final = {
+        "team_model_add": lambda: team_model_add(
+            data=TeamModelAddRequest(team_id="team-1234", models=["team-byok-1"]),
+            http_request=mock_request,
+            user_api_key_dict=mock_user_api_key_dict,
+        ),
+        "team_model_delete": lambda: team_model_delete(
+            data=TeamModelDeleteRequest(team_id="team-1234", models=["openai/*"]),
+            http_request=mock_request,
+            user_api_key_dict=mock_user_api_key_dict,
+        ),
+        "update_team_member_permissions": lambda: update_team_member_permissions(
+            data=UpdateTeamMemberPermissionsRequest(
+                team_id="team-1234",
+                team_member_permissions=["/key/generate"],
+            ),
+            http_request=mock_request,
+            user_api_key_dict=mock_user_api_key_dict,
+        ),
+    }[endpoint_name]
+
+    with (
+        patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma_client,  # test-quality-ok: proxy_server module global is the endpoint's only injection point
+        patch("litellm.proxy.proxy_server.user_api_key_cache"),  # test-quality-ok: proxy_server module global is the endpoint's only injection point
+        patch("litellm.proxy.proxy_server.proxy_logging_obj"),  # test-quality-ok: proxy_server module global is the endpoint's only injection point
+        patch(  # test-quality-ok: stubs the cache write so the test observes only the DB result handling
+            "litellm.proxy.management_endpoints.team_endpoints._cache_team_object",
+            new_callable=AsyncMock,
+        ),
+        patch(  # test-quality-ok: stubs the collaborator so the test pins the endpoint's own error contract
+            "litellm.proxy.management_endpoints.team_endpoints.get_team_object",
+            new_callable=AsyncMock,
+            return_value=existing_team,
+        ),
+    ):
+        mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock(
+            return_value=existing_team
+        )
+        mock_prisma_client.db.litellm_teamtable.update = AsyncMock(return_value=None)
+        mock_prisma_client.db.execute_raw = AsyncMock(return_value=None)
+
+        with pytest.raises(HTTPException) as exc_info:
+            await call_endpoint_under_test()
+
+    assert exc_info.value.status_code == 404
+    assert exc_info.value.detail == {"error": "Team not found, passed team_id=team-1234"}
+
+
 @pytest.mark.asyncio
 async def test_update_team_team_member_budget_not_passed_to_db(
     disable_audit_logging_for_mocked_team,
diff --git a/tests/test_litellm/proxy/memory/test_memory_endpoints.py b/tests/test_litellm/proxy/memory/test_memory_endpoints.py
index 3d99a600a73..da23e362bae 100644
--- a/tests/test_litellm/proxy/memory/test_memory_endpoints.py
+++ b/tests/test_litellm/proxy/memory/test_memory_endpoints.py
@@ -7,6 +7,7 @@ We patch the endpoint module's `_require_prisma` helper so we never need the
 real proxy_server import chain (which pulls heavy optional deps).
 """
 
+import json
 from datetime import datetime, timezone
 from typing import Any, Dict, List, Optional
 from unittest.mock import MagicMock, patch
@@ -176,14 +177,36 @@ class _InMemoryTeamTable:
         return None
 
 
-def _make_team(team_id: str, *, admin_user_ids: List[str]) -> MagicMock:
-    """Build a team-row stub with `members_with_roles` shaped like Prisma."""
-    members = [MagicMock(user_id=uid, role="admin") for uid in admin_user_ids]
-    team = MagicMock()
-    team.team_id = team_id
-    team.organization_id = None  # skip org-admin path in tests
-    team.members_with_roles = members
-    return team
+def _make_team(team_id: str, *, admin_user_ids: List[str]) -> Any:
+    """Build a real Prisma team row.
+
+    `members_with_roles` is a JSON column, so Prisma deserializes it into plain
+    dicts, not `Member` objects. A stub that hands back attribute-style members
+    would let the router read `member.role` off something Prisma never returns.
+    """
+    from prisma import models as prisma_models
+
+    now = datetime.now(timezone.utc)
+    return prisma_models.LiteLLM_TeamTable(
+        team_id=team_id,
+        organization_id=None,
+        members_with_roles=json.dumps([{"user_id": uid, "role": "admin"} for uid in admin_user_ids]),
+        metadata="{}",
+        models=[],
+        blocked=False,
+        created_at=now,
+        updated_at=now,
+        spend=0.0,
+        model_spend="{}",
+        model_max_budget="{}",
+        admins=[],
+        members=[],
+        team_member_permissions=[],
+        access_group_ids=[],
+        policies=[],
+        default_team_member_models=[],
+        allow_team_guardrail_config=False,
+    )
 
 
 def _make_prisma() -> MagicMock:
@@ -653,6 +676,39 @@ class TestMemoryEndpoints:
         assert resp.json()["value"] == "new"
         assert len(table.rows) == 1
 
+    def test_put_memory_row_deleted_mid_update_returns_404(self):
+        """
+        A concurrent DELETE landing between the visibility read and the write
+        makes Prisma's `update` return None. That must surface the same 404 the
+        read path uses, not an AttributeError bubbling out as an unhandled 500.
+        """
+        table = self.prisma.db.litellm_memorytable
+        table.rows.append(
+            _make_row(
+                memory_id="m1",
+                key="notes",
+                value="old",
+                user_id="user-a",
+                team_id="team-a",
+            )
+        )
+
+        async def vanished(*_args, **_kwargs):
+            return None
+
+        original_update = table.update
+        table.update = vanished
+
+        client = _make_client(_user_auth("user-a", "team-a"))
+        try:
+            with _patch_prisma(self.prisma):
+                resp = client.put("/v1/memory/notes", json={"value": "new"})
+        finally:
+            table.update = original_update
+
+        assert resp.status_code == 404, resp.text
+        assert resp.json()["detail"] == "Memory with key 'notes' not found"
+
     def test_put_memory_explicit_null_metadata_clears_field(self):
         """
         prisma-client-python can't write a true SQL NULL to a `Json?` column
diff --git a/tests/test_litellm/proxy/prompts/test_prompt_endpoints_crud.py b/tests/test_litellm/proxy/prompts/test_prompt_endpoints_crud.py
index 4fb8e54e68d..3e8e1e9dff8 100644
--- a/tests/test_litellm/proxy/prompts/test_prompt_endpoints_crud.py
+++ b/tests/test_litellm/proxy/prompts/test_prompt_endpoints_crud.py
@@ -191,3 +191,58 @@ async def test_get_prompt_info_by_base_id():
             response.prompt_spec.prompt_id == "test_prompt"
         )  # Should return base ID in spec response
         assert response.prompt_spec.version == 3  # Should identify it as version 3
+
+
+@pytest.mark.asyncio
+async def test_patch_prompt_row_deleted_mid_update_returns_404():
+    """
+    A concurrent delete between the version lookup and the write makes Prisma's
+    `update` return None. That must reuse the endpoint's existing not-found 404
+    contract rather than blowing up into an opaque 500.
+    """
+    from fastapi import HTTPException
+
+    from litellm.proxy.prompts.prompt_endpoints import PatchPromptRequest, patch_prompt
+
+    mock_user_auth = UserAPIKeyAuth(
+        api_key="sk-1234", user_role=LitellmUserRoles.PROXY_ADMIN
+    )
+
+    target_row = MagicMock()
+    target_row.id = "row-1"
+    target_row.version = 1
+
+    mock_prisma_client = MagicMock()
+    mock_prisma_client.db.litellm_prompttable.find_many = AsyncMock(
+        return_value=[target_row]
+    )
+    mock_prisma_client.db.litellm_prompttable.update = AsyncMock(return_value=None)
+
+    existing_prompt = PromptSpec(
+        prompt_id="test_prompt.v1",
+        litellm_params=PromptLiteLLMParams(
+            prompt_id="test_prompt", prompt_integration="dotprompt"
+        ),
+        prompt_info=PromptInfo(prompt_type="db"),
+    )
+
+    with (
+        patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client),  # test-quality-ok: proxy_server module global is the endpoint's only injection point
+        patch(  # test-quality-ok: stubs the collaborator so the test pins the endpoint's own error contract
+            "litellm.proxy.prompts.prompt_registry.IN_MEMORY_PROMPT_REGISTRY"
+        ) as mock_registry,
+    ):
+        mock_registry.get_prompt_by_id.return_value = existing_prompt
+
+        with pytest.raises(HTTPException) as exc_info:
+            await patch_prompt(
+                prompt_id="test_prompt",
+                request=PatchPromptRequest(prompt_info=PromptInfo(prompt_type="db")),
+                user_api_key_dict=mock_user_auth,
+            )
+
+    assert exc_info.value.status_code == 404
+    assert (
+        exc_info.value.detail
+        == "Prompt with ID test_prompt not found in environment development"
+    )
diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py
index 31d2a6cef98..d12684beb92 100644
--- a/tests/test_litellm/proxy/test_proxy_server.py
+++ b/tests/test_litellm/proxy/test_proxy_server.py
@@ -1850,6 +1850,39 @@ def test_add_team_models_to_all_models_excludes_other_teams_byok_with_shared_nam
     assert result == {"model-a-id": {"team-a"}}
 
 
+@pytest.mark.asyncio
+async def test_non_admin_all_models_raises_400_when_user_row_missing():
+    """
+    Regression test: a key whose user row no longer exists made find_unique return
+    None, and _check_if_model_is_team_model then dereferenced it
+    (`model_team_id in user_row.teams`) and raised AttributeError, surfacing as a
+    500. The miss must reuse the 400 "User not found" contract the neighbouring
+    except-branch already raises.
+    """
+    from fastapi import HTTPException
+
+    from litellm.proxy.proxy_server import non_admin_all_models
+
+    prisma_client = MagicMock()
+    prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)
+
+    llm_router = MagicMock()
+    llm_router.get_model_list.return_value = [
+        {"model_info": {"id": "gpt-4-model-1", "team_id": "team-a"}},
+    ]
+
+    with pytest.raises(HTTPException) as exc_info:
+        await non_admin_all_models(
+            all_models=[],
+            llm_router=llm_router,
+            user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", user_id="deleted-user"),
+            prisma_client=prisma_client,
+        )
+
+    assert exc_info.value.status_code == 400
+    assert exc_info.value.detail == {"error": "User not found"}
+
+
 @pytest.mark.asyncio
 async def test_apply_search_filter_matches_team_public_model_name():
     """
diff --git a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py
index 905928428b7..eae6f90863a 100644
--- a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py
+++ b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py
@@ -2924,6 +2924,57 @@ class TestUpdateVectorStoreAccessControlAndRedaction:
         assert params["api_key"] == REDACTED_BY_LITELM_STRING
         assert params["api_base"] == "https://api.openai.com/v1"
 
+    @pytest.mark.asyncio
+    async def test_update_row_deleted_mid_update_returns_404(self):
+        """A concurrent delete between the authorization read and the write makes Prisma's
+        ``update`` return None. That must reuse the not-found 404 contract instead of
+        turning an AttributeError into an opaque 500."""
+        from unittest.mock import AsyncMock, MagicMock, patch
+
+        from litellm.proxy._types import UserAPIKeyAuth
+        from litellm.proxy.vector_store_endpoints.management_endpoints import (
+            update_vector_store,
+        )
+        from litellm.types.vector_stores import VectorStoreUpdateRequest
+
+        existing_row = MagicMock()
+        existing_row.model_dump = MagicMock(
+            return_value={"vector_store_id": "vs_owned", "team_id": "team-A"}
+        )
+
+        mock_prisma_client = MagicMock()
+        mock_prisma_client.db.litellm_managedvectorstorestable.find_unique = AsyncMock(
+            return_value=existing_row
+        )
+        mock_prisma_client.db.litellm_managedvectorstorestable.update = AsyncMock(
+            return_value=None
+        )
+
+        with (
+            patch(  # test-quality-ok: stubs the auth gate so the test exercises the not-found branch under test
+                "litellm.proxy.vector_store_endpoints.management_endpoints.check_feature_access_for_user",
+                new_callable=AsyncMock,
+            ),
+            patch(  # test-quality-ok: stubs the auth gate so the test exercises the not-found branch under test
+                "litellm.proxy.vector_store_endpoints.management_endpoints._check_vector_store_access",
+                new_callable=AsyncMock,
+                return_value=True,
+            ),
+            patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client),  # test-quality-ok: proxy_server module global is the endpoint's only injection point
+            patch("litellm.vector_store_registry", None),  # test-quality-ok: litellm module global is the only injection point for the registry
+        ):
+            with pytest.raises(HTTPException) as exc_info:
+                await update_vector_store(
+                    data=VectorStoreUpdateRequest(
+                        vector_store_id="vs_owned",
+                        vector_store_description="new desc",
+                    ),
+                    user_api_key_dict=UserAPIKeyAuth(user_id="owner", team_id="team-A"),
+                )
+
+        assert exc_info.value.status_code == 404
+        assert exc_info.value.detail == "Vector store with ID vs_owned not found"
+
 
 class TestAzureAIDocumentWritePassthroughPermission:
     """Regression tests for the Azure AI Search passthrough write mapping.
diff --git a/type-discipline-budget.json b/type-discipline-budget.json
index 627811a7f1d..72fdfbef8c5 100644
--- a/type-discipline-budget.json
+++ b/type-discipline-budget.json
@@ -1,9 +1,9 @@
 {
   "LIT001": {
-    "limit": 22805
+    "limit": 22749
   },
   "LIT002": {
-    "limit": 26873
+    "limit": 26867
   },
   "LIT003": {
     "limit": 269
@@ -15,22 +15,22 @@
     "limit": 0
   },
   "LIT006": {
-    "limit": 1069
+    "limit": 1066
   },
   "LIT007": {
     "limit": 0
   },
   "LIT008": {
-    "limit": 950
+    "limit": 948
   },
   "LIT009": {
     "limit": 0
   },
   "LIT010": {
-    "limit": 16673
+    "limit": 16655
   },
   "LIT011": {
-    "limit": 5588
+    "limit": 5586
   },
   "LIT012": {
     "limit": 4510
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index 87f050e417f..c0509ec3358 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -26685,7 +26685,7 @@ export interface components {
              * Admins
              * @default []
              */
-            admins: unknown[];
+            admins: string[];
             /**
              * Allow Team Guardrail Config
              * @default false
@@ -26725,7 +26725,7 @@ export interface components {
              * Members
              * @default []
              */
-            members: unknown[];
+            members: string[];
             /**
              * Members With Roles
              * @default []
@@ -26755,7 +26755,7 @@ export interface components {
              * Models
              * @default []
              */
-            models: unknown[];
+            models: string[];
             object_permission?: components["schemas"]["LiteLLM_ObjectPermissionTable"] | null;
             /** Object Permission Id */
             object_permission_id?: string | null;
@@ -27961,7 +27961,7 @@ export interface components {
              * Admins
              * @default []
              */
-            admins: unknown[];
+            admins: string[];
             /**
              * Allow Team Guardrail Config
              * @default false
@@ -27991,7 +27991,7 @@ export interface components {
              * Members
              * @default []
              */
-            members: unknown[];
+            members: string[];
             /**
              * Members With Roles
              * @default []
@@ -28021,7 +28021,7 @@ export interface components {
              * Models
              * @default []
              */
-            models: unknown[];
+            models: string[];
             object_permission?: components["schemas"]["LiteLLM_ObjectPermissionTable"] | null;
             /** Object Permission Id */
             object_permission_id?: string | null;
@@ -30065,7 +30065,7 @@ export interface components {
              * Admins
              * @default []
              */
-            admins: unknown[];
+            admins: string[];
             /** Allowed Passthrough Routes */
             allowed_passthrough_routes?: unknown[] | null;
             /** Allowed Vector Store Indexes */
@@ -30109,7 +30109,7 @@ export interface components {
              * Members
              * @default []
              */
-            members: unknown[];
+            members: string[];
             /**
              * Members With Roles
              * @default []
@@ -30135,7 +30135,7 @@ export interface components {
              * Models
              * @default []
              */
-            models: unknown[];
+            models: string[];
             object_permission?: components["schemas"]["LiteLLM_ObjectPermissionBase"] | null;
             /** Organization Id */
             organization_id?: string | null;
@@ -33913,7 +33913,7 @@ export interface components {
              * Admins
              * @default []
              */
-            admins: unknown[];
+            admins: string[];
             /**
              * Allow Team Guardrail Config
              * @default false
@@ -33943,7 +33943,7 @@ export interface components {
              * Members
              * @default []
              */
-            members: unknown[];
+            members: string[];
             /**
              * Members With Roles
              * @default []
@@ -33973,7 +33973,7 @@ export interface components {
              * Models
              * @default []
              */
-            models: unknown[];
+            models: string[];
             object_permission?: components["schemas"]["LiteLLM_ObjectPermissionTable"] | null;
             /** Object Permission Id */
             object_permission_id?: string | null;
@@ -34043,7 +34043,7 @@ export interface components {
              * Admins
              * @default []
              */
-            admins: unknown[];
+            admins: string[];
             /**
              * Allow Team Guardrail Config
              * @default false
@@ -34078,7 +34078,7 @@ export interface components {
              * Members
              * @default []
              */
-            members: unknown[];
+            members: string[];
             /**
              * Members Count
              * @default 0
@@ -34113,7 +34113,7 @@ export interface components {
              * Models
              * @default []
              */
-            models: unknown[];
+            models: string[];
             object_permission?: components["schemas"]["LiteLLM_ObjectPermissionTable"] | null;
             /** Object Permission Id */
             object_permission_id?: string | null;

From fc810484f790f283e2f0a897fa972783e707431c Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 09:28:55 -0700
Subject: [PATCH 026/105] Revert "ci: raise three unit shard job timeouts to
 satisfy the startup safety gate"

This reverts commit b71b574af410f436954f9e6fcb28b44eff6c1d34.
---
 .github/workflows/test-unit.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml
index 2dfca3d308f..a7c67f2b35d 100644
--- a/.github/workflows/test-unit.yml
+++ b/.github/workflows/test-unit.yml
@@ -211,7 +211,7 @@ jobs:
             workers: 2
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 60
+            job-timeout-minutes: 55
 
           - shard: proxy-extras
             artifact-name: proxy-extras
@@ -219,7 +219,7 @@ jobs:
             workers: 2
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 60
+            job-timeout-minutes: 55
 
           - shard: enterprise-package
             artifact-name: enterprise-package
@@ -227,7 +227,7 @@ jobs:
             workers: 4
             reruns: 2
             timeout-minutes: 20
-            job-timeout-minutes: 60
+            job-timeout-minutes: 55
 
           - shard: responses-caching-types
             artifact-name: responses-caching-types

From 9224b2ce5d3e0327bcc0b02e6f111794574716d6 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 09:50:23 -0700
Subject: [PATCH 027/105] fix(router): freeze reasoning effort flag mappings

---
 .../router_utils/reasoning_effort_capability.py  | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/litellm/router_utils/reasoning_effort_capability.py b/litellm/router_utils/reasoning_effort_capability.py
index d20b1151803..3e4478e5e21 100644
--- a/litellm/router_utils/reasoning_effort_capability.py
+++ b/litellm/router_utils/reasoning_effort_capability.py
@@ -25,12 +25,14 @@ it above.
 """
 
 from collections.abc import Mapping, Sequence
+from types import MappingProxyType
 from typing import Final, get_args
 
 import litellm
 from litellm.types.llms.openai import REASONING_EFFORT
 
 REASONING_EFFORT_ADVERTISEMENT_ORDER: Final = get_args(REASONING_EFFORT)
+_EMPTY_ENTRY: Final[Mapping[str, object]] = MappingProxyType({})
 
 _EFFORT_FLAGS: Final = (
     ("none", "supports_none_reasoning_effort"),
@@ -52,17 +54,19 @@ def _bare_model_entry(model_info: Mapping[str, object]) -> Mapping[str, object]:
     key: Final = model_info.get("key")
     provider: Final = model_info.get("litellm_provider")
     if not isinstance(key, str) or not isinstance(provider, str) or not key.startswith(f"{provider}/"):
-        return {}
+        return _EMPTY_ENTRY
     entry: Final[Mapping[str, object] | None] = litellm.model_cost.get(key.removeprefix(f"{provider}/"))
-    return entry if entry is not None else {}
+    return entry if entry is not None else _EMPTY_ENTRY
 
 
 def _declared_effort_flags(model_info: Mapping[str, object]) -> Mapping[str, object]:
     bare: Final = _bare_model_entry(model_info)
-    return {
-        effort: model_info.get(flag) if model_info.get(flag) is not None else bare.get(flag)
-        for effort, flag in _EFFORT_FLAGS
-    }
+    return MappingProxyType(
+        {
+            effort: model_info.get(flag) if model_info.get(flag) is not None else bare.get(flag)
+            for effort, flag in _EFFORT_FLAGS
+        }
+    )
 
 
 def _supports_none_reasoning_effort(model_info: Mapping[str, object], flag: object) -> bool:

From ed28581d791e289c2a5b20e0f91678232d5d17ee Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 09:57:25 -0700
Subject: [PATCH 028/105] fix(bedrock_mantle): normalize Codex input item types
 Mantle rejects

Mantle 400s ("Invalid 'input': value did not match any expected variant")
on the Codex history item types agent_message, context_compaction, and
local_shell_call, killing every Codex multi-agent session on the first
sub-agent turn. Rewrite agent_message into an assistant output_text message
(preserving encrypted_content slot payloads, which carry the plaintext task
through Mantle), context_compaction into Mantle's supported compaction
spelling, and local_shell_call into the function_call its recorded
function_call_output already pairs with.
---
 .../responses/transformation.py               | 118 +++++++++++-
 ...bedrock_mantle_responses_transformation.py | 176 ++++++++++++++++++
 2 files changed, 293 insertions(+), 1 deletion(-)

diff --git a/litellm/llms/bedrock_mantle/responses/transformation.py b/litellm/llms/bedrock_mantle/responses/transformation.py
index 92da5835b2d..9068a641940 100644
--- a/litellm/llms/bedrock_mantle/responses/transformation.py
+++ b/litellm/llms/bedrock_mantle/responses/transformation.py
@@ -15,8 +15,12 @@ role / access key / profile / web identity), signed via the shared
 BaseAWSLLM._sign_request after the request body is finalized.
 """
 
+import json
+from collections.abc import Mapping
 from typing import Any, Final
 
+from typing_extensions import ReadOnly, TypedDict
+
 import litellm
 from litellm._logging import verbose_logger
 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
@@ -50,6 +54,33 @@ _BEDROCK_MANTLE_SUPPORTED_SERVICE_TIERS: Final = frozenset({"auto", "default"})
 
 _CODEX_ADDITIONAL_TOOLS_INPUT_ITEM_TYPE: Final = "additional_tools"
 
+_CODEX_AGENT_MESSAGE_INPUT_ITEM_TYPE: Final = "agent_message"
+_CODEX_CONTEXT_COMPACTION_INPUT_ITEM_TYPE: Final = "context_compaction"
+_CODEX_LOCAL_SHELL_CALL_INPUT_ITEM_TYPE: Final = "local_shell_call"
+
+
+class _RewrittenOutputTextBlock(TypedDict):
+    type: ReadOnly[str]
+    text: ReadOnly[str]
+
+
+class _RewrittenAssistantMessageItem(TypedDict):
+    type: ReadOnly[str]
+    role: ReadOnly[str]
+    content: ReadOnly[tuple[_RewrittenOutputTextBlock, ...]]
+
+
+class _RewrittenCompactionItem(TypedDict):
+    type: ReadOnly[str]
+    encrypted_content: ReadOnly[str]
+
+
+class _RewrittenFunctionCallItem(TypedDict):
+    type: ReadOnly[str]
+    call_id: ReadOnly[str]
+    name: ReadOnly[str]
+    arguments: ReadOnly[str]
+
 
 class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPIConfig):
     def __init__(
@@ -155,6 +186,7 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
         headers: dict,
     ) -> dict:
         remaining_input, hoisted_tools = self._hoist_codex_additional_tools(input)
+        normalized_input: Final = self._normalize_codex_input_items(remaining_input)
         request_params: Final = (
             {
                 **response_api_optional_request_params,
@@ -168,7 +200,7 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
         )
         return super().transform_responses_api_request(
             model=model,
-            input=remaining_input,
+            input=normalized_input,
             response_api_optional_request_params=request_params,
             litellm_params=litellm_params,
             headers=headers,
@@ -210,6 +242,90 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
         )
         return remaining_input, cls._filter_unsupported_tools(hoisted_tools)
 
+    @staticmethod
+    def _agent_message_text(item: "Mapping[str, Any]") -> str:
+        content: Final = item.get("content")
+        if not isinstance(content, list):
+            return ""
+        return "".join(
+            str(block.get("text") or block.get("encrypted_content") or "")
+            for block in content
+            if isinstance(block, dict)
+        )
+
+    @classmethod
+    def _normalize_agent_message_item(cls, item: "Mapping[str, Any]") -> "_RewrittenAssistantMessageItem | None":
+        text: Final = cls._agent_message_text(item)
+        if not text:
+            return None
+        rewritten: Final[_RewrittenAssistantMessageItem] = {
+            "type": "message",
+            "role": "assistant",
+            "content": ({"type": "output_text", "text": text},),
+        }
+        return rewritten
+
+    @staticmethod
+    def _normalize_context_compaction_item(item: "Mapping[str, Any]") -> "_RewrittenCompactionItem | None":
+        encrypted_content: Final = item.get("encrypted_content")
+        if not isinstance(encrypted_content, str) or not encrypted_content:
+            return None
+        rewritten: Final[_RewrittenCompactionItem] = {"type": "compaction", "encrypted_content": encrypted_content}
+        return rewritten
+
+    @staticmethod
+    def _normalize_local_shell_call_item(item: "Mapping[str, Any]") -> "_RewrittenFunctionCallItem | None":
+        call_id: Final = item.get("call_id")
+        if not isinstance(call_id, str) or not call_id:
+            return None
+        action: Final = item.get("action")
+        rewritten: Final[_RewrittenFunctionCallItem] = {
+            "type": "function_call",
+            "call_id": call_id,
+            "name": "local_shell",
+            "arguments": json.dumps(action) if isinstance(action, dict) else "{}",
+        }
+        return rewritten
+
+    @classmethod
+    def _normalize_codex_input_item(cls, item: object) -> "tuple[Any, str | None]":
+        """Returns (normalized item or None to drop it, original type when rewritten)."""
+        if not isinstance(item, dict):
+            return item, None
+        item_type: Final = item.get("type")
+        if item_type == _CODEX_AGENT_MESSAGE_INPUT_ITEM_TYPE:
+            return cls._normalize_agent_message_item(item), item_type
+        if item_type == _CODEX_CONTEXT_COMPACTION_INPUT_ITEM_TYPE:
+            return cls._normalize_context_compaction_item(item), item_type
+        if item_type == _CODEX_LOCAL_SHELL_CALL_INPUT_ITEM_TYPE:
+            return cls._normalize_local_shell_call_item(item), item_type
+        return item, None
+
+    @classmethod
+    def _normalize_codex_input_items(
+        cls,
+        input: "str | ResponseInputParam",
+    ) -> "str | ResponseInputParam":
+        """Rewrite Codex history item types Mantle rejects with 400 "Invalid
+        'input': value did not match any expected variant" into supported
+        equivalents. `agent_message` (Codex multi-agent traffic; its
+        encrypted_content slot carries the plaintext payload when the model
+        never issued encrypted args) becomes an assistant message,
+        `context_compaction` becomes the `compaction` spelling Mantle accepts,
+        and `local_shell_call` becomes the function_call its recorded
+        function_call_output already pairs with.
+        """
+        if not isinstance(input, list):
+            return input
+        normalized: Final = tuple(cls._normalize_codex_input_item(item) for item in input)
+        rewritten_types: Final = sorted(frozenset(item_type for _, item_type in normalized if item_type is not None))
+        if rewritten_types:
+            verbose_logger.warning(
+                "Bedrock Mantle Responses API: rewrote Codex input item type(s) %s that Mantle rejects.",
+                rewritten_types,
+            )
+        return [item for item, _ in normalized if item is not None]  # mutable-ok: ResponseInputParam is a list
+
     def map_openai_params(
         self,
         response_api_optional_params: ResponsesAPIOptionalRequestParams,
diff --git a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py
index 9e05d48a18f..984ff997292 100644
--- a/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py
+++ b/tests/test_litellm/llms/bedrock_mantle/test_bedrock_mantle_responses_transformation.py
@@ -8,6 +8,7 @@ gate, the URL construction for both paths, and the shared Bearer auth.
 """
 
 import copy
+import logging
 
 
 import pytest
@@ -623,6 +624,181 @@ class TestBedrockMantleCodexAdditionalTools:
         assert "additional_tools" in str(mock_debug.call_args)
 
 
+class TestBedrockMantleCodexInputItemNormalization:
+    """Mantle 400s ("Invalid 'input': value did not match any expected variant")
+    on the Codex history item types agent_message, context_compaction, and
+    local_shell_call (verified against bedrock-mantle.us-east-1.api.aws with
+    openai.gpt-5.6-sol), so the config must rewrite them into supported
+    equivalents. agent_message is what every Codex multi-agent v2 session sends,
+    and its encrypted_content slot carries the verbatim plaintext payload when
+    the upstream model never issued encrypted args, so that slot must be
+    preserved, not dropped. Mantle also rejects assistant messages with
+    input_text content, so the rewrite must use output_text."""
+
+    _USER_MESSAGE = {
+        "type": "message",
+        "role": "user",
+        "content": [{"type": "input_text", "text": "Continue."}],
+    }
+
+    def _transform(self, input):
+        cfg = BedrockMantleResponsesAPIConfig()
+        return cfg.transform_responses_api_request(
+            model="openai.gpt-5.6-sol",
+            input=input,
+            response_api_optional_request_params={},
+            litellm_params=GenericLiteLLMParams(),
+            headers={},
+        )
+
+    def test_plaintext_agent_message_becomes_assistant_output_text_message(self):
+        body = self._transform(
+            input=[
+                self._USER_MESSAGE,
+                {
+                    "type": "agent_message",
+                    "id": "amsg_1",
+                    "author": "/root/arithmetic",
+                    "recipient": "/root",
+                    "content": [{"type": "input_text", "text": "Message Type: FINAL_ANSWER\nPayload:\n2+2 is 4."}],
+                },
+            ]
+        )
+        assert body["input"] == [
+            self._USER_MESSAGE,
+            {
+                "type": "message",
+                "role": "assistant",
+                "content": ({"type": "output_text", "text": "Message Type: FINAL_ANSWER\nPayload:\n2+2 is 4."},),
+            },
+        ]
+
+    def test_agent_message_encrypted_content_payload_is_preserved(self):
+        body = self._transform(
+            input=[
+                {
+                    "type": "agent_message",
+                    "author": "/root",
+                    "recipient": "/root/arithmetic",
+                    "content": [
+                        {"type": "input_text", "text": "Message Type: NEW_TASK\nPayload:\n"},
+                        {"type": "encrypted_content", "encrypted_content": "Answer the question 'what is 2+2'."},
+                    ],
+                },
+                self._USER_MESSAGE,
+            ]
+        )
+        assert body["input"][0] == {
+            "type": "message",
+            "role": "assistant",
+            "content": (
+                {
+                    "type": "output_text",
+                    "text": "Message Type: NEW_TASK\nPayload:\nAnswer the question 'what is 2+2'.",
+                },
+            ),
+        }
+
+    def test_agent_message_without_any_text_is_dropped(self):
+        body = self._transform(
+            input=[
+                {"type": "agent_message", "author": "/root", "recipient": "/root/a", "content": []},
+                self._USER_MESSAGE,
+            ]
+        )
+        assert body["input"] == [self._USER_MESSAGE]
+
+    def test_context_compaction_becomes_compaction_with_same_ciphertext(self):
+        body = self._transform(
+            input=[
+                {"type": "context_compaction", "id": "cc_1", "encrypted_content": "smry_abc123"},
+                self._USER_MESSAGE,
+            ]
+        )
+        assert body["input"] == [
+            {"type": "compaction", "encrypted_content": "smry_abc123"},
+            self._USER_MESSAGE,
+        ]
+
+    def test_context_compaction_without_ciphertext_is_dropped(self):
+        body = self._transform(
+            input=[
+                {"type": "context_compaction", "id": "cc_1"},
+                self._USER_MESSAGE,
+            ]
+        )
+        assert body["input"] == [self._USER_MESSAGE]
+
+    def test_local_shell_call_becomes_function_call_keeping_call_id_pairing(self):
+        body = self._transform(
+            input=[
+                {
+                    "type": "local_shell_call",
+                    "id": "lsh_1",
+                    "call_id": "call_1",
+                    "status": "completed",
+                    "action": {"type": "exec", "command": ["echo", "hi"]},
+                },
+                {"type": "function_call_output", "call_id": "call_1", "output": "hi\n"},
+                self._USER_MESSAGE,
+            ]
+        )
+        assert body["input"] == [
+            {
+                "type": "function_call",
+                "call_id": "call_1",
+                "name": "local_shell",
+                "arguments": '{"type": "exec", "command": ["echo", "hi"]}',
+            },
+            {"type": "function_call_output", "call_id": "call_1", "output": "hi\n"},
+            self._USER_MESSAGE,
+        ]
+
+    def test_local_shell_call_without_call_id_is_dropped(self):
+        body = self._transform(
+            input=[
+                {"type": "local_shell_call", "status": "completed", "action": {"type": "exec", "command": ["ls"]}},
+                self._USER_MESSAGE,
+            ]
+        )
+        assert body["input"] == [self._USER_MESSAGE]
+
+    def test_mantle_supported_item_types_pass_through_untouched(self):
+        supported_items = [
+            self._USER_MESSAGE,
+            {"type": "compaction", "encrypted_content": "smry_abc123"},
+            {"type": "function_call", "name": "shell", "arguments": "{}", "call_id": "call_2"},
+            {"type": "function_call_output", "call_id": "call_2", "output": "ok"},
+            {"type": "tool_search_call", "call_id": "call_3", "execution": "server", "arguments": {"query": "x"}},
+            {"type": "tool_search_output", "call_id": "call_3", "status": "completed", "execution": "server", "tools": []},
+            {"type": "compaction_trigger"},
+        ]
+        body = self._transform(input=copy.deepcopy(supported_items))
+        assert body["input"] == supported_items
+
+    def test_string_input_passes_through(self):
+        body = self._transform(input="Say hi.")
+        assert body["input"] == "Say hi."
+
+    def test_rewrite_is_logged_as_warning_naming_the_types(self, caplog):
+        with caplog.at_level(logging.WARNING, logger="LiteLLM"):
+            body = self._transform(
+                input=[
+                    {"type": "agent_message", "author": "a", "recipient": "b", "content": [{"type": "input_text", "text": "hi"}]},
+                    self._USER_MESSAGE,
+                ]
+            )
+        assert body["input"][0]["role"] == "assistant"
+        rewrite_warnings = [
+            record.getMessage()
+            for record in caplog.records
+            if record.levelno == logging.WARNING and "rewrote Codex input item type" in record.getMessage()
+        ]
+        assert rewrite_warnings == [
+            "Bedrock Mantle Responses API: rewrote Codex input item type(s) ['agent_message'] that Mantle rejects."
+        ]
+
+
 class TestBedrockMantleResponsesRegistry:
     def test_registry_returns_config_for_gpt_5_5(self, local_cost_map):
         # gpt-5.x advertises /v1/responses in supported_endpoints (capability)

From 44c7cb20aee5832734f626ad522c867b851fde40 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 10:10:07 -0700
Subject: [PATCH 029/105] feat(models): add missing Together AI serverless
 models to the cost map

Backfill 21 serverless chat models, the multilingual-e5 embedding model, and
Llama-Guard-4-12B from the live Together catalog with per-token pricing and
capability flags. Mark 25 delisted together_ai entries with their documented
deprecation_date and point superseded models at a live successor via metadata.
Reprice Llama-3.3-70B-Instruct-Turbo to Together's current rate.
---
 ...odel_prices_and_context_window_backup.json | 349 +++++++++++++++++-
 model_prices_and_context_window.json          | 349 +++++++++++++++++-
 .../test_together_ai_model_metadata.py        | 149 ++++++++
 3 files changed, 843 insertions(+), 4 deletions(-)
 create mode 100644 tests/test_litellm/test_together_ai_model_metadata.py

diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 9f953e11df1..9b2b910e007 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -37886,6 +37886,7 @@
         "output_cost_per_token": 1e-07
     },
     "together_ai/Qwen/Qwen2.5-72B-Instruct-Turbo": {
+        "deprecation_date": "2026-02-06",
         "litellm_provider": "together_ai",
         "mode": "chat",
         "supports_function_calling": true,
@@ -37902,6 +37903,7 @@
         "supports_tool_choice": true
     },
     "together_ai/Qwen/Qwen3-235B-A22B-Instruct-2507-tput": {
+        "deprecation_date": "2026-07-10",
         "input_cost_per_token": 2e-07,
         "litellm_provider": "together_ai",
         "max_input_tokens": 262000,
@@ -37914,6 +37916,7 @@
         "supports_tool_choice": true
     },
     "together_ai/Qwen/Qwen3-235B-A22B-Thinking-2507": {
+        "deprecation_date": "2026-04-16",
         "input_cost_per_token": 6.5e-07,
         "litellm_provider": "together_ai",
         "max_input_tokens": 256000,
@@ -37926,6 +37929,7 @@
         "supports_tool_choice": true
     },
     "together_ai/Qwen/Qwen3-235B-A22B-fp8-tput": {
+        "deprecation_date": "2026-02-06",
         "input_cost_per_token": 2e-07,
         "litellm_provider": "together_ai",
         "max_input_tokens": 40000,
@@ -37937,6 +37941,7 @@
         "supports_tool_choice": false
     },
     "together_ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": {
+        "deprecation_date": "2026-06-04",
         "input_cost_per_token": 2e-06,
         "litellm_provider": "together_ai",
         "max_input_tokens": 256000,
@@ -37949,11 +37954,15 @@
         "supports_tool_choice": true
     },
     "together_ai/deepseek-ai/DeepSeek-R1": {
+        "deprecation_date": "2026-05-14",
         "input_cost_per_token": 3e-06,
         "litellm_provider": "together_ai",
         "max_input_tokens": 128000,
         "max_output_tokens": 20480,
         "max_tokens": 20480,
+        "metadata": {
+            "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro"
+        },
         "mode": "chat",
         "output_cost_per_token": 7e-06,
         "supports_function_calling": true,
@@ -37962,6 +37971,7 @@
         "supports_tool_choice": true
     },
     "together_ai/deepseek-ai/DeepSeek-R1-0528-tput": {
+        "deprecation_date": "2026-02-03",
         "input_cost_per_token": 5.5e-07,
         "litellm_provider": "together_ai",
         "max_input_tokens": 128000,
@@ -37979,6 +37989,9 @@
         "max_input_tokens": 65536,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
+        "metadata": {
+            "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro"
+        },
         "mode": "chat",
         "output_cost_per_token": 1.25e-06,
         "supports_function_calling": true,
@@ -37987,9 +38000,13 @@
         "supports_tool_choice": true
     },
     "together_ai/deepseek-ai/DeepSeek-V3.1": {
+        "deprecation_date": "2026-05-14",
         "input_cost_per_token": 6e-07,
         "litellm_provider": "together_ai",
         "max_tokens": 16384,
+        "metadata": {
+            "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro"
+        },
         "mode": "chat",
         "output_cost_per_token": 1.7e-06,
         "source": "https://www.together.ai/models/deepseek-v3-1",
@@ -38001,6 +38018,7 @@
         "max_output_tokens": 16384
     },
     "together_ai/meta-llama/Llama-3.2-3B-Instruct-Turbo": {
+        "deprecation_date": "2026-03-06",
         "litellm_provider": "together_ai",
         "mode": "chat",
         "supports_function_calling": true,
@@ -38009,16 +38027,21 @@
         "supports_tool_choice": true
     },
     "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo": {
-        "input_cost_per_token": 8.8e-07,
+        "input_cost_per_token": 1.04e-06,
         "litellm_provider": "together_ai",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
         "mode": "chat",
-        "output_cost_per_token": 8.8e-07,
+        "output_cost_per_token": 1.04e-06,
+        "source": "https://docs.together.ai/docs/serverless-models",
         "supports_function_calling": true,
         "supports_parallel_function_calling": true,
         "supports_response_schema": true,
         "supports_tool_choice": true
     },
     "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo-Free": {
+        "deprecation_date": "2025-11-13",
         "input_cost_per_token": 0,
         "litellm_provider": "together_ai",
         "mode": "chat",
@@ -38029,6 +38052,7 @@
         "supports_tool_choice": true
     },
     "together_ai/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8": {
+        "deprecation_date": "2026-03-31",
         "input_cost_per_token": 2.7e-07,
         "litellm_provider": "together_ai",
         "mode": "chat",
@@ -38039,6 +38063,7 @@
         "supports_tool_choice": true
     },
     "together_ai/meta-llama/Llama-4-Scout-17B-16E-Instruct": {
+        "deprecation_date": "2026-02-06",
         "input_cost_per_token": 1.8e-07,
         "litellm_provider": "together_ai",
         "mode": "chat",
@@ -38049,6 +38074,7 @@
         "supports_tool_choice": true
     },
     "together_ai/meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo": {
+        "deprecation_date": "2026-02-06",
         "input_cost_per_token": 3.5e-06,
         "litellm_provider": "together_ai",
         "mode": "chat",
@@ -38059,6 +38085,7 @@
         "supports_tool_choice": true
     },
     "together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo": {
+        "deprecation_date": "2026-02-25",
         "input_cost_per_token": 8.8e-07,
         "litellm_provider": "together_ai",
         "mode": "chat",
@@ -38069,6 +38096,7 @@
         "supports_tool_choice": true
     },
     "together_ai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo": {
+        "deprecation_date": "2026-03-06",
         "input_cost_per_token": 1.8e-07,
         "litellm_provider": "together_ai",
         "mode": "chat",
@@ -38079,6 +38107,7 @@
         "supports_tool_choice": true
     },
     "together_ai/mistralai/Mistral-7B-Instruct-v0.1": {
+        "deprecation_date": "2025-11-13",
         "litellm_provider": "together_ai",
         "mode": "chat",
         "supports_function_calling": true,
@@ -38087,6 +38116,7 @@
         "supports_tool_choice": true
     },
     "together_ai/mistralai/Mistral-Small-24B-Instruct-2501": {
+        "deprecation_date": "2026-04-02",
         "litellm_provider": "together_ai",
         "mode": "chat",
         "supports_function_calling": true,
@@ -38094,6 +38124,7 @@
         "supports_tool_choice": true
     },
     "together_ai/mistralai/Mixtral-8x7B-Instruct-v0.1": {
+        "deprecation_date": "2026-04-16",
         "input_cost_per_token": 6e-07,
         "litellm_provider": "together_ai",
         "mode": "chat",
@@ -38106,6 +38137,9 @@
     "together_ai/moonshotai/Kimi-K2-Instruct": {
         "input_cost_per_token": 1e-06,
         "litellm_provider": "together_ai",
+        "metadata": {
+            "successor": "together_ai/moonshotai/Kimi-K3"
+        },
         "mode": "chat",
         "output_cost_per_token": 3e-06,
         "source": "https://www.together.ai/models/kimi-k2-instruct",
@@ -38149,6 +38183,7 @@
         "supports_tool_choice": true
     },
     "together_ai/zai-org/GLM-4.5-Air-FP8": {
+        "deprecation_date": "2026-04-02",
         "input_cost_per_token": 2e-07,
         "litellm_provider": "together_ai",
         "max_input_tokens": 128000,
@@ -38166,6 +38201,9 @@
         "max_input_tokens": 200000,
         "max_output_tokens": 200000,
         "max_tokens": 200000,
+        "metadata": {
+            "successor": "together_ai/zai-org/GLM-5.2"
+        },
         "mode": "chat",
         "output_cost_per_token": 2.2e-06,
         "source": "https://www.together.ai/models/glm-4-6",
@@ -38175,11 +38213,15 @@
         "supports_tool_choice": true
     },
     "together_ai/zai-org/GLM-4.7": {
+        "deprecation_date": "2026-04-02",
         "input_cost_per_token": 4.5e-07,
         "litellm_provider": "together_ai",
         "max_input_tokens": 200000,
         "max_output_tokens": 200000,
         "max_tokens": 200000,
+        "metadata": {
+            "successor": "together_ai/zai-org/GLM-5.2"
+        },
         "mode": "chat",
         "output_cost_per_token": 2e-06,
         "source": "https://www.together.ai/models/glm-4-7",
@@ -38189,11 +38231,15 @@
         "supports_tool_choice": true
     },
     "together_ai/moonshotai/Kimi-K2.5": {
+        "deprecation_date": "2026-05-21",
         "input_cost_per_token": 5e-07,
         "litellm_provider": "together_ai",
         "max_input_tokens": 256000,
         "max_output_tokens": 256000,
         "max_tokens": 256000,
+        "metadata": {
+            "successor": "together_ai/moonshotai/Kimi-K3"
+        },
         "mode": "chat",
         "output_cost_per_token": 2.8e-06,
         "source": "https://www.together.ai/models/kimi-k2-5",
@@ -38203,9 +38249,13 @@
         "supports_reasoning": true
     },
     "together_ai/moonshotai/Kimi-K2-Instruct-0905": {
+        "deprecation_date": "2026-03-06",
         "input_cost_per_token": 1e-06,
         "litellm_provider": "together_ai",
         "max_input_tokens": 262144,
+        "metadata": {
+            "successor": "together_ai/moonshotai/Kimi-K3"
+        },
         "mode": "chat",
         "output_cost_per_token": 3e-06,
         "source": "https://www.together.ai/models/kimi-k2-0905",
@@ -38214,9 +38264,13 @@
         "supports_tool_choice": true
     },
     "together_ai/Qwen/Qwen3-Next-80B-A3B-Instruct": {
+        "deprecation_date": "2026-04-02",
         "input_cost_per_token": 1.5e-07,
         "litellm_provider": "together_ai",
         "max_input_tokens": 262144,
+        "metadata": {
+            "successor": "together_ai/Qwen/Qwen3.7-Plus"
+        },
         "mode": "chat",
         "output_cost_per_token": 1.5e-06,
         "source": "https://www.together.ai/models/qwen3-next-80b-a3b-instruct",
@@ -38226,9 +38280,13 @@
         "supports_tool_choice": true
     },
     "together_ai/Qwen/Qwen3-Next-80B-A3B-Thinking": {
+        "deprecation_date": "2026-02-25",
         "input_cost_per_token": 1.5e-07,
         "litellm_provider": "together_ai",
         "max_input_tokens": 262144,
+        "metadata": {
+            "successor": "together_ai/Qwen/Qwen3.6-Plus"
+        },
         "mode": "chat",
         "output_cost_per_token": 1.5e-06,
         "source": "https://www.together.ai/models/qwen3-next-80b-a3b-thinking",
@@ -38238,6 +38296,7 @@
         "supports_tool_choice": true
     },
     "together_ai/Qwen/Qwen3.5-397B-A17B": {
+        "deprecation_date": "2026-06-29",
         "input_cost_per_token": 6e-07,
         "litellm_provider": "together_ai",
         "max_input_tokens": 262144,
@@ -38249,6 +38308,292 @@
         "supports_response_schema": true,
         "supports_tool_choice": true
     },
+    "together_ai/MiniMaxAI/MiniMax-M3": {
+        "input_cost_per_token": 3e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 524288,
+        "max_output_tokens": 524288,
+        "max_tokens": 524288,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-06,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "together_ai/Prism-ML/Ternary-Bonsai-27B": {
+        "input_cost_per_token": 0.0,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 0.0,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/Qwen/Qwen3.5-9B": {
+        "input_cost_per_token": 1.7e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 2.5e-07,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "together_ai/Qwen/Qwen3.6-Plus": {
+        "input_cost_per_token": 5e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 1000000,
+        "max_tokens": 1000000,
+        "mode": "chat",
+        "output_cost_per_token": 3e-06,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_reasoning": true
+    },
+    "together_ai/Qwen/Qwen3.7-Max": {
+        "input_cost_per_token": 1.25e-06,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 1000000,
+        "max_tokens": 1000000,
+        "mode": "chat",
+        "output_cost_per_token": 3.75e-06,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/Qwen/Qwen3.7-Plus": {
+        "input_cost_per_token": 3.2e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 1000000,
+        "max_tokens": 1000000,
+        "mode": "chat",
+        "output_cost_per_token": 1.28e-06,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/Qwen/Qwen3.8-2.4T-A95B": {
+        "input_cost_per_token": 2.5e-06,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 1010000,
+        "max_output_tokens": 1010000,
+        "max_tokens": 1010000,
+        "mode": "chat",
+        "output_cost_per_token": 6.25e-06,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/arize-ai/qwen-2-1.5b-instruct": {
+        "input_cost_per_token": 1e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 32768,
+        "max_output_tokens": 32768,
+        "max_tokens": 32768,
+        "mode": "chat",
+        "output_cost_per_token": 1e-07,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/deepseek-ai/DeepSeek-V4-Flash-0731": {
+        "input_cost_per_token": 1.4e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 1048576,
+        "max_tokens": 1048576,
+        "mode": "chat",
+        "output_cost_per_token": 2.8e-07,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
+    "together_ai/deepseek-ai/DeepSeek-V4-Pro": {
+        "input_cost_per_token": 1.74e-06,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 512000,
+        "max_output_tokens": 512000,
+        "max_tokens": 512000,
+        "mode": "chat",
+        "output_cost_per_token": 3.48e-06,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
+    "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813": {
+        "input_cost_per_token": 1.32e-06,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 1048576,
+        "max_tokens": 1048576,
+        "mode": "chat",
+        "output_cost_per_token": 3.96e-06,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
+    "together_ai/google/gemma-3n-E4B-it": {
+        "input_cost_per_token": 6e-08,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 32768,
+        "max_output_tokens": 32768,
+        "max_tokens": 32768,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-07,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/google/gemma-4-31B-it": {
+        "input_cost_per_token": 3.9e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 9.7e-07,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "together_ai/intfloat/multilingual-e5-large-instruct": {
+        "input_cost_per_token": 2e-08,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 514,
+        "max_tokens": 514,
+        "mode": "embedding",
+        "output_cost_per_token": 2e-08,
+        "output_vector_size": 1024,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/meta-llama/Llama-Guard-4-12B": {
+        "input_cost_per_token": 2e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 1048576,
+        "max_tokens": 1048576,
+        "mode": "chat",
+        "output_cost_per_token": 2e-07,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/meta-models/Muse-Glimmer-30B": {
+        "input_cost_per_token": 3.5e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-06,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/moonshotai/Kimi-K2.7-Code": {
+        "input_cost_per_token": 9.5e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 4e-06,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "together_ai/moonshotai/Kimi-K3": {
+        "input_cost_per_token": 3e-06,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 1048576,
+        "max_tokens": 1048576,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "together_ai/nvidia/nemotron-3-ultra-550b-a55b": {
+        "input_cost_per_token": 6e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 512288,
+        "max_output_tokens": 512288,
+        "max_tokens": 512288,
+        "mode": "chat",
+        "output_cost_per_token": 3.6e-06,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
+    "together_ai/pearl-ai/gemma-4-31b-it": {
+        "input_cost_per_token": 2.8e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 8.6e-07,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/thinkingmachines/Inkling": {
+        "input_cost_per_token": 1e-06,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 524288,
+        "max_output_tokens": 524288,
+        "max_tokens": 524288,
+        "mode": "chat",
+        "output_cost_per_token": 4.05e-06,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
+    "together_ai/thinkingmachines/Inkling-Small": {
+        "input_cost_per_token": 5e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 524288,
+        "max_output_tokens": 524288,
+        "max_tokens": 524288,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-06,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/zai-org/GLM-5.2": {
+        "input_cost_per_token": 1.4e-06,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 1048575,
+        "max_output_tokens": 1048575,
+        "max_tokens": 1048575,
+        "mode": "chat",
+        "output_cost_per_token": 4.4e-06,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
     "tts-1": {
         "input_cost_per_character": 1.5e-05,
         "litellm_provider": "openai",
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index 9f953e11df1..9b2b910e007 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -37886,6 +37886,7 @@
         "output_cost_per_token": 1e-07
     },
     "together_ai/Qwen/Qwen2.5-72B-Instruct-Turbo": {
+        "deprecation_date": "2026-02-06",
         "litellm_provider": "together_ai",
         "mode": "chat",
         "supports_function_calling": true,
@@ -37902,6 +37903,7 @@
         "supports_tool_choice": true
     },
     "together_ai/Qwen/Qwen3-235B-A22B-Instruct-2507-tput": {
+        "deprecation_date": "2026-07-10",
         "input_cost_per_token": 2e-07,
         "litellm_provider": "together_ai",
         "max_input_tokens": 262000,
@@ -37914,6 +37916,7 @@
         "supports_tool_choice": true
     },
     "together_ai/Qwen/Qwen3-235B-A22B-Thinking-2507": {
+        "deprecation_date": "2026-04-16",
         "input_cost_per_token": 6.5e-07,
         "litellm_provider": "together_ai",
         "max_input_tokens": 256000,
@@ -37926,6 +37929,7 @@
         "supports_tool_choice": true
     },
     "together_ai/Qwen/Qwen3-235B-A22B-fp8-tput": {
+        "deprecation_date": "2026-02-06",
         "input_cost_per_token": 2e-07,
         "litellm_provider": "together_ai",
         "max_input_tokens": 40000,
@@ -37937,6 +37941,7 @@
         "supports_tool_choice": false
     },
     "together_ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": {
+        "deprecation_date": "2026-06-04",
         "input_cost_per_token": 2e-06,
         "litellm_provider": "together_ai",
         "max_input_tokens": 256000,
@@ -37949,11 +37954,15 @@
         "supports_tool_choice": true
     },
     "together_ai/deepseek-ai/DeepSeek-R1": {
+        "deprecation_date": "2026-05-14",
         "input_cost_per_token": 3e-06,
         "litellm_provider": "together_ai",
         "max_input_tokens": 128000,
         "max_output_tokens": 20480,
         "max_tokens": 20480,
+        "metadata": {
+            "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro"
+        },
         "mode": "chat",
         "output_cost_per_token": 7e-06,
         "supports_function_calling": true,
@@ -37962,6 +37971,7 @@
         "supports_tool_choice": true
     },
     "together_ai/deepseek-ai/DeepSeek-R1-0528-tput": {
+        "deprecation_date": "2026-02-03",
         "input_cost_per_token": 5.5e-07,
         "litellm_provider": "together_ai",
         "max_input_tokens": 128000,
@@ -37979,6 +37989,9 @@
         "max_input_tokens": 65536,
         "max_output_tokens": 8192,
         "max_tokens": 8192,
+        "metadata": {
+            "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro"
+        },
         "mode": "chat",
         "output_cost_per_token": 1.25e-06,
         "supports_function_calling": true,
@@ -37987,9 +38000,13 @@
         "supports_tool_choice": true
     },
     "together_ai/deepseek-ai/DeepSeek-V3.1": {
+        "deprecation_date": "2026-05-14",
         "input_cost_per_token": 6e-07,
         "litellm_provider": "together_ai",
         "max_tokens": 16384,
+        "metadata": {
+            "successor": "together_ai/deepseek-ai/DeepSeek-V4-Pro"
+        },
         "mode": "chat",
         "output_cost_per_token": 1.7e-06,
         "source": "https://www.together.ai/models/deepseek-v3-1",
@@ -38001,6 +38018,7 @@
         "max_output_tokens": 16384
     },
     "together_ai/meta-llama/Llama-3.2-3B-Instruct-Turbo": {
+        "deprecation_date": "2026-03-06",
         "litellm_provider": "together_ai",
         "mode": "chat",
         "supports_function_calling": true,
@@ -38009,16 +38027,21 @@
         "supports_tool_choice": true
     },
     "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo": {
-        "input_cost_per_token": 8.8e-07,
+        "input_cost_per_token": 1.04e-06,
         "litellm_provider": "together_ai",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
         "mode": "chat",
-        "output_cost_per_token": 8.8e-07,
+        "output_cost_per_token": 1.04e-06,
+        "source": "https://docs.together.ai/docs/serverless-models",
         "supports_function_calling": true,
         "supports_parallel_function_calling": true,
         "supports_response_schema": true,
         "supports_tool_choice": true
     },
     "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo-Free": {
+        "deprecation_date": "2025-11-13",
         "input_cost_per_token": 0,
         "litellm_provider": "together_ai",
         "mode": "chat",
@@ -38029,6 +38052,7 @@
         "supports_tool_choice": true
     },
     "together_ai/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8": {
+        "deprecation_date": "2026-03-31",
         "input_cost_per_token": 2.7e-07,
         "litellm_provider": "together_ai",
         "mode": "chat",
@@ -38039,6 +38063,7 @@
         "supports_tool_choice": true
     },
     "together_ai/meta-llama/Llama-4-Scout-17B-16E-Instruct": {
+        "deprecation_date": "2026-02-06",
         "input_cost_per_token": 1.8e-07,
         "litellm_provider": "together_ai",
         "mode": "chat",
@@ -38049,6 +38074,7 @@
         "supports_tool_choice": true
     },
     "together_ai/meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo": {
+        "deprecation_date": "2026-02-06",
         "input_cost_per_token": 3.5e-06,
         "litellm_provider": "together_ai",
         "mode": "chat",
@@ -38059,6 +38085,7 @@
         "supports_tool_choice": true
     },
     "together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo": {
+        "deprecation_date": "2026-02-25",
         "input_cost_per_token": 8.8e-07,
         "litellm_provider": "together_ai",
         "mode": "chat",
@@ -38069,6 +38096,7 @@
         "supports_tool_choice": true
     },
     "together_ai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo": {
+        "deprecation_date": "2026-03-06",
         "input_cost_per_token": 1.8e-07,
         "litellm_provider": "together_ai",
         "mode": "chat",
@@ -38079,6 +38107,7 @@
         "supports_tool_choice": true
     },
     "together_ai/mistralai/Mistral-7B-Instruct-v0.1": {
+        "deprecation_date": "2025-11-13",
         "litellm_provider": "together_ai",
         "mode": "chat",
         "supports_function_calling": true,
@@ -38087,6 +38116,7 @@
         "supports_tool_choice": true
     },
     "together_ai/mistralai/Mistral-Small-24B-Instruct-2501": {
+        "deprecation_date": "2026-04-02",
         "litellm_provider": "together_ai",
         "mode": "chat",
         "supports_function_calling": true,
@@ -38094,6 +38124,7 @@
         "supports_tool_choice": true
     },
     "together_ai/mistralai/Mixtral-8x7B-Instruct-v0.1": {
+        "deprecation_date": "2026-04-16",
         "input_cost_per_token": 6e-07,
         "litellm_provider": "together_ai",
         "mode": "chat",
@@ -38106,6 +38137,9 @@
     "together_ai/moonshotai/Kimi-K2-Instruct": {
         "input_cost_per_token": 1e-06,
         "litellm_provider": "together_ai",
+        "metadata": {
+            "successor": "together_ai/moonshotai/Kimi-K3"
+        },
         "mode": "chat",
         "output_cost_per_token": 3e-06,
         "source": "https://www.together.ai/models/kimi-k2-instruct",
@@ -38149,6 +38183,7 @@
         "supports_tool_choice": true
     },
     "together_ai/zai-org/GLM-4.5-Air-FP8": {
+        "deprecation_date": "2026-04-02",
         "input_cost_per_token": 2e-07,
         "litellm_provider": "together_ai",
         "max_input_tokens": 128000,
@@ -38166,6 +38201,9 @@
         "max_input_tokens": 200000,
         "max_output_tokens": 200000,
         "max_tokens": 200000,
+        "metadata": {
+            "successor": "together_ai/zai-org/GLM-5.2"
+        },
         "mode": "chat",
         "output_cost_per_token": 2.2e-06,
         "source": "https://www.together.ai/models/glm-4-6",
@@ -38175,11 +38213,15 @@
         "supports_tool_choice": true
     },
     "together_ai/zai-org/GLM-4.7": {
+        "deprecation_date": "2026-04-02",
         "input_cost_per_token": 4.5e-07,
         "litellm_provider": "together_ai",
         "max_input_tokens": 200000,
         "max_output_tokens": 200000,
         "max_tokens": 200000,
+        "metadata": {
+            "successor": "together_ai/zai-org/GLM-5.2"
+        },
         "mode": "chat",
         "output_cost_per_token": 2e-06,
         "source": "https://www.together.ai/models/glm-4-7",
@@ -38189,11 +38231,15 @@
         "supports_tool_choice": true
     },
     "together_ai/moonshotai/Kimi-K2.5": {
+        "deprecation_date": "2026-05-21",
         "input_cost_per_token": 5e-07,
         "litellm_provider": "together_ai",
         "max_input_tokens": 256000,
         "max_output_tokens": 256000,
         "max_tokens": 256000,
+        "metadata": {
+            "successor": "together_ai/moonshotai/Kimi-K3"
+        },
         "mode": "chat",
         "output_cost_per_token": 2.8e-06,
         "source": "https://www.together.ai/models/kimi-k2-5",
@@ -38203,9 +38249,13 @@
         "supports_reasoning": true
     },
     "together_ai/moonshotai/Kimi-K2-Instruct-0905": {
+        "deprecation_date": "2026-03-06",
         "input_cost_per_token": 1e-06,
         "litellm_provider": "together_ai",
         "max_input_tokens": 262144,
+        "metadata": {
+            "successor": "together_ai/moonshotai/Kimi-K3"
+        },
         "mode": "chat",
         "output_cost_per_token": 3e-06,
         "source": "https://www.together.ai/models/kimi-k2-0905",
@@ -38214,9 +38264,13 @@
         "supports_tool_choice": true
     },
     "together_ai/Qwen/Qwen3-Next-80B-A3B-Instruct": {
+        "deprecation_date": "2026-04-02",
         "input_cost_per_token": 1.5e-07,
         "litellm_provider": "together_ai",
         "max_input_tokens": 262144,
+        "metadata": {
+            "successor": "together_ai/Qwen/Qwen3.7-Plus"
+        },
         "mode": "chat",
         "output_cost_per_token": 1.5e-06,
         "source": "https://www.together.ai/models/qwen3-next-80b-a3b-instruct",
@@ -38226,9 +38280,13 @@
         "supports_tool_choice": true
     },
     "together_ai/Qwen/Qwen3-Next-80B-A3B-Thinking": {
+        "deprecation_date": "2026-02-25",
         "input_cost_per_token": 1.5e-07,
         "litellm_provider": "together_ai",
         "max_input_tokens": 262144,
+        "metadata": {
+            "successor": "together_ai/Qwen/Qwen3.6-Plus"
+        },
         "mode": "chat",
         "output_cost_per_token": 1.5e-06,
         "source": "https://www.together.ai/models/qwen3-next-80b-a3b-thinking",
@@ -38238,6 +38296,7 @@
         "supports_tool_choice": true
     },
     "together_ai/Qwen/Qwen3.5-397B-A17B": {
+        "deprecation_date": "2026-06-29",
         "input_cost_per_token": 6e-07,
         "litellm_provider": "together_ai",
         "max_input_tokens": 262144,
@@ -38249,6 +38308,292 @@
         "supports_response_schema": true,
         "supports_tool_choice": true
     },
+    "together_ai/MiniMaxAI/MiniMax-M3": {
+        "input_cost_per_token": 3e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 524288,
+        "max_output_tokens": 524288,
+        "max_tokens": 524288,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-06,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "together_ai/Prism-ML/Ternary-Bonsai-27B": {
+        "input_cost_per_token": 0.0,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 0.0,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/Qwen/Qwen3.5-9B": {
+        "input_cost_per_token": 1.7e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 2.5e-07,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "together_ai/Qwen/Qwen3.6-Plus": {
+        "input_cost_per_token": 5e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 1000000,
+        "max_tokens": 1000000,
+        "mode": "chat",
+        "output_cost_per_token": 3e-06,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_reasoning": true
+    },
+    "together_ai/Qwen/Qwen3.7-Max": {
+        "input_cost_per_token": 1.25e-06,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 1000000,
+        "max_tokens": 1000000,
+        "mode": "chat",
+        "output_cost_per_token": 3.75e-06,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/Qwen/Qwen3.7-Plus": {
+        "input_cost_per_token": 3.2e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 1000000,
+        "max_tokens": 1000000,
+        "mode": "chat",
+        "output_cost_per_token": 1.28e-06,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/Qwen/Qwen3.8-2.4T-A95B": {
+        "input_cost_per_token": 2.5e-06,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 1010000,
+        "max_output_tokens": 1010000,
+        "max_tokens": 1010000,
+        "mode": "chat",
+        "output_cost_per_token": 6.25e-06,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/arize-ai/qwen-2-1.5b-instruct": {
+        "input_cost_per_token": 1e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 32768,
+        "max_output_tokens": 32768,
+        "max_tokens": 32768,
+        "mode": "chat",
+        "output_cost_per_token": 1e-07,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/deepseek-ai/DeepSeek-V4-Flash-0731": {
+        "input_cost_per_token": 1.4e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 1048576,
+        "max_tokens": 1048576,
+        "mode": "chat",
+        "output_cost_per_token": 2.8e-07,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
+    "together_ai/deepseek-ai/DeepSeek-V4-Pro": {
+        "input_cost_per_token": 1.74e-06,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 512000,
+        "max_output_tokens": 512000,
+        "max_tokens": 512000,
+        "mode": "chat",
+        "output_cost_per_token": 3.48e-06,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
+    "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813": {
+        "input_cost_per_token": 1.32e-06,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 1048576,
+        "max_tokens": 1048576,
+        "mode": "chat",
+        "output_cost_per_token": 3.96e-06,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
+    "together_ai/google/gemma-3n-E4B-it": {
+        "input_cost_per_token": 6e-08,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 32768,
+        "max_output_tokens": 32768,
+        "max_tokens": 32768,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-07,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/google/gemma-4-31B-it": {
+        "input_cost_per_token": 3.9e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 9.7e-07,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "together_ai/intfloat/multilingual-e5-large-instruct": {
+        "input_cost_per_token": 2e-08,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 514,
+        "max_tokens": 514,
+        "mode": "embedding",
+        "output_cost_per_token": 2e-08,
+        "output_vector_size": 1024,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/meta-llama/Llama-Guard-4-12B": {
+        "input_cost_per_token": 2e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 1048576,
+        "max_tokens": 1048576,
+        "mode": "chat",
+        "output_cost_per_token": 2e-07,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/meta-models/Muse-Glimmer-30B": {
+        "input_cost_per_token": 3.5e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 131072,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-06,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/moonshotai/Kimi-K2.7-Code": {
+        "input_cost_per_token": 9.5e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 4e-06,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "together_ai/moonshotai/Kimi-K3": {
+        "input_cost_per_token": 3e-06,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 1048576,
+        "max_tokens": 1048576,
+        "mode": "chat",
+        "output_cost_per_token": 1.5e-05,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "together_ai/nvidia/nemotron-3-ultra-550b-a55b": {
+        "input_cost_per_token": 6e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 512288,
+        "max_output_tokens": 512288,
+        "max_tokens": 512288,
+        "mode": "chat",
+        "output_cost_per_token": 3.6e-06,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
+    "together_ai/pearl-ai/gemma-4-31b-it": {
+        "input_cost_per_token": 2.8e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 262144,
+        "max_tokens": 262144,
+        "mode": "chat",
+        "output_cost_per_token": 8.6e-07,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/thinkingmachines/Inkling": {
+        "input_cost_per_token": 1e-06,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 524288,
+        "max_output_tokens": 524288,
+        "max_tokens": 524288,
+        "mode": "chat",
+        "output_cost_per_token": 4.05e-06,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
+    "together_ai/thinkingmachines/Inkling-Small": {
+        "input_cost_per_token": 5e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 524288,
+        "max_output_tokens": 524288,
+        "max_tokens": 524288,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-06,
+        "source": "https://docs.together.ai/docs/serverless-models"
+    },
+    "together_ai/zai-org/GLM-5.2": {
+        "input_cost_per_token": 1.4e-06,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 1048575,
+        "max_output_tokens": 1048575,
+        "max_tokens": 1048575,
+        "mode": "chat",
+        "output_cost_per_token": 4.4e-06,
+        "source": "https://docs.together.ai/docs/serverless-models",
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
     "tts-1": {
         "input_cost_per_character": 1.5e-05,
         "litellm_provider": "openai",
diff --git a/tests/test_litellm/test_together_ai_model_metadata.py b/tests/test_litellm/test_together_ai_model_metadata.py
new file mode 100644
index 00000000000..7d7712f3c56
--- /dev/null
+++ b/tests/test_litellm/test_together_ai_model_metadata.py
@@ -0,0 +1,149 @@
+import json
+from pathlib import Path
+from typing import Final
+
+import pytest
+
+from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
+
+REPO_ROOT: Final = Path(__file__).parents[2]
+
+SERVERLESS_CHAT_MODELS: Final = (
+    "together_ai/moonshotai/Kimi-K3",
+    "together_ai/zai-org/GLM-5.2",
+    "together_ai/deepseek-ai/DeepSeek-V4-Pro",
+    "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813",
+    "together_ai/deepseek-ai/DeepSeek-V4-Flash-0731",
+    "together_ai/moonshotai/Kimi-K2.7-Code",
+    "together_ai/MiniMaxAI/MiniMax-M3",
+    "together_ai/thinkingmachines/Inkling",
+    "together_ai/thinkingmachines/Inkling-Small",
+    "together_ai/Qwen/Qwen3.8-2.4T-A95B",
+    "together_ai/Qwen/Qwen3.7-Max",
+    "together_ai/Qwen/Qwen3.7-Plus",
+    "together_ai/Qwen/Qwen3.6-Plus",
+    "together_ai/Qwen/Qwen3.5-9B",
+    "together_ai/nvidia/nemotron-3-ultra-550b-a55b",
+    "together_ai/meta-models/Muse-Glimmer-30B",
+    "together_ai/google/gemma-4-31B-it",
+    "together_ai/pearl-ai/gemma-4-31b-it",
+    "together_ai/google/gemma-3n-E4B-it",
+    "together_ai/arize-ai/qwen-2-1.5b-instruct",
+    "together_ai/Prism-ML/Ternary-Bonsai-27B",
+    "together_ai/meta-llama/Llama-Guard-4-12B",
+    "together_ai/openai/gpt-oss-120b",
+    "together_ai/openai/gpt-oss-20b",
+    "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo",
+)
+
+DEPRECATED_MODELS: Final = {
+    "together_ai/Qwen/Qwen3-235B-A22B-Instruct-2507-tput": "2026-07-10",
+    "together_ai/Qwen/Qwen3.5-397B-A17B": "2026-06-29",
+    "together_ai/Qwen/Qwen3-Coder-480B-A35B-Instruct-FP8": "2026-06-04",
+    "together_ai/moonshotai/Kimi-K2.5": "2026-05-21",
+    "together_ai/deepseek-ai/DeepSeek-R1": "2026-05-14",
+    "together_ai/deepseek-ai/DeepSeek-V3.1": "2026-05-14",
+    "together_ai/Qwen/Qwen3-235B-A22B-Thinking-2507": "2026-04-16",
+    "together_ai/mistralai/Mixtral-8x7B-Instruct-v0.1": "2026-04-16",
+    "together_ai/zai-org/GLM-4.5-Air-FP8": "2026-04-02",
+    "together_ai/zai-org/GLM-4.7": "2026-04-02",
+    "together_ai/mistralai/Mistral-Small-24B-Instruct-2501": "2026-04-02",
+    "together_ai/Qwen/Qwen3-Next-80B-A3B-Instruct": "2026-04-02",
+    "together_ai/meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8": "2026-03-31",
+    "together_ai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo": "2026-03-06",
+    "together_ai/moonshotai/Kimi-K2-Instruct-0905": "2026-03-06",
+    "together_ai/meta-llama/Llama-3.2-3B-Instruct-Turbo": "2026-03-06",
+    "together_ai/Qwen/Qwen3-Next-80B-A3B-Thinking": "2026-02-25",
+    "together_ai/meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo": "2026-02-25",
+    "together_ai/Qwen/Qwen3-235B-A22B-fp8-tput": "2026-02-06",
+    "together_ai/meta-llama/Llama-4-Scout-17B-16E-Instruct": "2026-02-06",
+    "together_ai/Qwen/Qwen2.5-72B-Instruct-Turbo": "2026-02-06",
+    "together_ai/meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo": "2026-02-06",
+    "together_ai/deepseek-ai/DeepSeek-R1-0528-tput": "2026-02-03",
+    "together_ai/mistralai/Mistral-7B-Instruct-v0.1": "2025-11-13",
+    "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo-Free": "2025-11-13",
+}
+
+
+@pytest.fixture(scope="module")
+def cost_map() -> dict:
+    with open(REPO_ROOT / "model_prices_and_context_window.json") as f:
+        return json.load(f)
+
+
+@pytest.mark.parametrize("model", SERVERLESS_CHAT_MODELS)
+def test_together_serverless_chat_model_is_mapped(cost_map: dict, model: str):
+    info = cost_map.get(model)
+    assert info is not None, f"{model} missing from model_prices_and_context_window.json"
+    assert info["litellm_provider"] == "together_ai"
+    assert info["mode"] == "chat"
+    assert info["input_cost_per_token"] >= 0
+    assert info["output_cost_per_token"] >= info["input_cost_per_token"]
+    assert "deprecation_date" not in info
+
+    routed_model, provider, _, _ = get_llm_provider(model=model)
+    assert routed_model == model.removeprefix("together_ai/")
+    assert provider == "together_ai"
+
+
+def test_together_kimi_k3_pricing_and_capabilities(cost_map: dict):
+    info = cost_map["together_ai/moonshotai/Kimi-K3"]
+    assert info["input_cost_per_token"] == 3e-06
+    assert info["output_cost_per_token"] == 1.5e-05
+    assert info["max_input_tokens"] == 1048576
+    assert info["supports_function_calling"] is True
+    assert info["supports_tool_choice"] is True
+    assert info["supports_response_schema"] is True
+    assert info["supports_vision"] is True
+    assert info["supports_reasoning"] is True
+
+
+def test_together_glm_52_pricing(cost_map: dict):
+    info = cost_map["together_ai/zai-org/GLM-5.2"]
+    assert info["input_cost_per_token"] == 1.4e-06
+    assert info["output_cost_per_token"] == 4.4e-06
+    assert info["supports_function_calling"] is True
+    assert info["supports_reasoning"] is True
+
+
+def test_together_multilingual_e5_embedding_entry(cost_map: dict):
+    info = cost_map["together_ai/intfloat/multilingual-e5-large-instruct"]
+    assert info["mode"] == "embedding"
+    assert info["input_cost_per_token"] == 2e-08
+    assert info["max_input_tokens"] == 514
+    assert info["output_vector_size"] == 1024
+
+
+def test_together_llama_33_70b_repriced_to_current_together_rate(cost_map: dict):
+    info = cost_map["together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo"]
+    assert info["input_cost_per_token"] == 1.04e-06
+    assert info["output_cost_per_token"] == 1.04e-06
+    assert info["max_input_tokens"] == 131072
+
+
+@pytest.mark.parametrize("model", sorted(DEPRECATED_MODELS))
+def test_together_deprecated_model_carries_deprecation_date(cost_map: dict, model: str):
+    info = cost_map.get(model)
+    assert info is not None, f"{model} missing from model_prices_and_context_window.json"
+    assert info.get("deprecation_date") == DEPRECATED_MODELS[model]
+
+
+def test_together_successor_metadata_points_at_live_models(cost_map: dict):
+    successors = {
+        model: info["metadata"]["successor"]
+        for model, info in cost_map.items()
+        if model.startswith("together_ai/") and "successor" in info.get("metadata", {})
+    }
+    assert len(successors) >= 10
+    for model, successor in successors.items():
+        target = cost_map.get(successor)
+        assert target is not None, f"{model} names successor {successor} that is not in the map"
+        assert "deprecation_date" not in target, f"{model} names deprecated successor {successor}"
+
+
+def test_together_backup_cost_map_in_sync(cost_map: dict):
+    with open(REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json") as f:
+        backup = json.load(f)
+    together_main = {k: v for k, v in cost_map.items() if k.startswith("together_ai/")}
+    together_backup = {k: v for k, v in backup.items() if k.startswith("together_ai/")}
+    assert together_backup == together_main

From 4b5e3db8906625ba2128d8702d37e8d4ee95995e Mon Sep 17 00:00:00 2001
From: mateo-berri <mateowang.dev@users.noreply.github.com>
Date: Tue, 25 Aug 2026 10:15:29 -0700
Subject: [PATCH 030/105] test(e2e): cover the Bedrock provider-feature cells
 customers run

Adds live e2e coverage for the Bedrock combinations behind recent customer
incidents: llm_provider-* response-header forwarding on /chat/completions
(nonstream and stream), regional us.anthropic.* inference-profile ids over
the invoke route, and the Admin UI Test Connection probe for a
responses-mode Bedrock Mantle deployment. Registers the matching cells in
the coverage registry and publishes the provider x feature matrix table in
its README.
---
 tests/e2e/coverage_registry/README.md         |  18 ++
 .../coverage_registry/llm_conversational.yaml |   4 +
 tests/e2e/coverage_registry/mgmt.yaml         |   1 +
 tests/e2e/coverage_registry/schema.py         |   1 +
 .../test_bedrock_provider_matrix_e2e.py       | 157 ++++++++++++++++++
 tests/e2e/management/management_client.py     |  13 ++
 .../test_model_test_connection_e2e.py         |  42 +++++
 tests/e2e/models.py                           |  20 +++
 8 files changed, 256 insertions(+)
 create mode 100644 tests/e2e/llm_translation/test_bedrock_provider_matrix_e2e.py
 create mode 100644 tests/e2e/management/test_model_test_connection_e2e.py

diff --git a/tests/e2e/coverage_registry/README.md b/tests/e2e/coverage_registry/README.md
index 5627c88dee4..da6aee84cc4 100644
--- a/tests/e2e/coverage_registry/README.md
+++ b/tests/e2e/coverage_registry/README.md
@@ -77,6 +77,24 @@ Strict mode exits non-zero on `@pytest.mark.covers(...)` ids that are not checke
 the registry. Add `--fail-on-collection-errors` when the job should also fail on pytest
 collection errors.
 
+## Provider x feature matrix: customer-run Bedrock combinations
+
+The provider and feature combinations customers actually run get explicit cells, expanded
+here as incidents surface new ones. The current Bedrock set, seeded from a customer's
+production shape (regional `us.anthropic.*` inference-profile ids over both chat routes,
+provider response headers for AWS-side correlation, and the Test Connection probe for a
+responses-mode Bedrock Mantle deployment):
+
+| Cell | Feature | Covering test |
+|------|---------|---------------|
+| `llm.chat_completions.bedrock_converse.basic.nonstream.works` | regional `us.` id, Converse | `llm_translation/test_chat_completions_regression_e2e.py` |
+| `llm.chat_completions.bedrock_converse.basic.stream.works` | regional `us.` id, Converse stream | `llm_translation/test_chat_completions_regression_e2e.py` |
+| `llm.chat_completions.bedrock_invoke.basic.nonstream.works` | regional `us.` id, Invoke | `llm_translation/test_bedrock_provider_matrix_e2e.py` |
+| `llm.chat_completions.bedrock_invoke.basic.stream.works` | regional `us.` id, Invoke stream | `llm_translation/test_bedrock_provider_matrix_e2e.py` |
+| `llm.chat_completions.bedrock_converse.response_headers.nonstream.works` | `llm_provider-*` headers | `llm_translation/test_bedrock_provider_matrix_e2e.py` |
+| `llm.chat_completions.bedrock_converse.response_headers.stream.works` | `llm_provider-*` headers, stream | `llm_translation/test_bedrock_provider_matrix_e2e.py` |
+| `mgmt.model.test_connection.happy_path` | Test Connection, Bedrock Mantle | `management/test_model_test_connection_e2e.py` |
+
 ## Status: this is a draft for review
 
 The cells were enumerated from the codebase and the tiers are a first proposal. Known
diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml
index 1d4e1e028ca..d13f17e7eb6 100644
--- a/tests/e2e/coverage_registry/llm_conversational.yaml
+++ b/tests/e2e/coverage_registry/llm_conversational.yaml
@@ -29,6 +29,10 @@
 - {id: llm.chat_completions.bedrock_converse.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_converse, capability: vision, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Bedrock vision (Anthropic/Nova)"}
 - {id: llm.chat_completions.bedrock_converse.prompt_cache_5m.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: bedrock_converse, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Anthropic-on-Bedrock caching"}
 - {id: llm.chat_completions.bedrock_converse.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: bedrock_converse, capability: thinking, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Anthropic thinking on Bedrock"}
+- {id: llm.chat_completions.bedrock_converse.response_headers.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_converse, capability: response_headers, streaming: nonstream, assertions: [works], source: "llms/bedrock/chat/converse_handler.py:248", rationale: "Bedrock request ids must surface as llm_provider-* response headers on /chat/completions so callers can correlate calls with AWS-side logs (#37003)", fail_before_fix: proven}
+- {id: llm.chat_completions.bedrock_converse.response_headers.stream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_converse, capability: response_headers, streaming: stream, assertions: [works], source: "llms/bedrock/chat/converse_handler.py:154", rationale: "The llm_provider-* headers must also surface on streaming /chat/completions, where CustomStreamWrapper carries them instead of the nonstream setter"}
+- {id: llm.chat_completions.bedrock_invoke.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_invoke, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "Regional inference-profile ids (us.anthropic.*) over the invoke route, the deployment shape behind a customer timeout report on v1.90.0"}
+- {id: llm.chat_completions.bedrock_invoke.basic.stream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: bedrock_invoke, capability: basic, streaming: stream, assertions: [works], source: "proxy_server.py:8455", rationale: "Streaming with regional inference-profile ids over the invoke route"}
 - {id: llm.chat_completions.vertex.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "P0 route; Vertex AI"}
 - {id: llm.chat_completions.gemini.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: gemini, capability: basic, streaming: nonstream, assertions: [works], source: "test_chat_completions_regression_e2e.py", rationale: "Gemini OpenAI-compatible chat translation"}
 - {id: llm.chat_completions.gemini.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: chat_completions, route: gemini, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "test_chat_completions_regression_e2e.py", rationale: "Gemini chat cost lands in SpendLogs"}
diff --git a/tests/e2e/coverage_registry/mgmt.yaml b/tests/e2e/coverage_registry/mgmt.yaml
index d8788d7fcb0..1e6de0c3d6a 100644
--- a/tests/e2e/coverage_registry/mgmt.yaml
+++ b/tests/e2e/coverage_registry/mgmt.yaml
@@ -75,3 +75,4 @@
 - {id: mgmt.workflow.list.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "workflow_management_endpoints.py", rationale: "Workflow tracking (smoke)"}
 - {id: mgmt.credential_migration.check.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "key_management_endpoints.py:4252", rationale: "Encryption migration (smoke)"}
 - {id: mgmt.credential.new.serves_request, module: mgmt, tier: P1, surface: api, assertions: [serves_request], source: "credential_endpoints/endpoints.py:42", rationale: "Stored credential resolves into a deployment and serves a live /messages request"}
+- {id: mgmt.model.test_connection.happy_path, module: mgmt, tier: P0, surface: api, assertions: [happy_path], source: "_health_endpoints.py:1785", rationale: "Test Connection for a responses-mode Bedrock Mantle deployment reaches the live provider and reports success; this exact shape 500ed on an acompletion partial before v1.91.0", fail_before_fix: proven}
diff --git a/tests/e2e/coverage_registry/schema.py b/tests/e2e/coverage_registry/schema.py
index a5c723f8965..03d15f532b8 100644
--- a/tests/e2e/coverage_registry/schema.py
+++ b/tests/e2e/coverage_registry/schema.py
@@ -71,6 +71,7 @@ LlmCapability = Literal[
     "pdf_input",
     "prompt_cache_1h",
     "prompt_cache_5m",
+    "response_headers",
     "service_tier",
     "structured_output",
     "thinking",
diff --git a/tests/e2e/llm_translation/test_bedrock_provider_matrix_e2e.py b/tests/e2e/llm_translation/test_bedrock_provider_matrix_e2e.py
new file mode 100644
index 00000000000..3c6aaa75ab3
--- /dev/null
+++ b/tests/e2e/llm_translation/test_bedrock_provider_matrix_e2e.py
@@ -0,0 +1,157 @@
+"""Live e2e for the Bedrock cells of the provider-feature matrix: provider
+response headers on /chat/completions and regional inference-profile model ids
+(us.anthropic.*) over the invoke route.
+
+Header forwarding is the #37003 contract: the proxy surfaces Bedrock's response
+headers prefixed llm_provider- (llm_provider-x-amzn-requestid above all) so a
+caller can hand AWS support the request id behind a completion. Regional
+inference-profile ids are the deployment shape most Bedrock customers run; a
+v1.90.0 regression timed them out, and the Converse route keeps them covered in
+test_chat_completions_regression_e2e.py, so the invoke route carries its own
+rows here.
+"""
+
+from __future__ import annotations
+
+import pytest
+from pydantic import BaseModel
+
+from e2e_config import unique_marker
+from e2e_http import StreamingResponse, unwrap
+from lifecycle import ResourceManager
+from models import ChatBody, ChatMessage, ChatResponse, LiteLLMParamsBody
+from passthrough_client import PassthroughClient
+
+pytestmark = pytest.mark.e2e
+
+CONVERSE_REGIONAL_BACKEND = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0"
+INVOKE_REGIONAL_BACKEND = "bedrock/invoke/us.anthropic.claude-haiku-4-5-20251001-v1:0"
+PROVIDER_HEADER_PREFIX = "llm_provider-"
+BEDROCK_REQUEST_ID_HEADER = "llm_provider-x-amzn-requestid"
+
+
+class _StreamDelta(BaseModel):
+    content: str | None = None
+
+
+class _StreamChoice(BaseModel):
+    delta: _StreamDelta = _StreamDelta()
+
+
+class _StreamChunk(BaseModel):
+    choices: list[_StreamChoice] = []
+
+
+def _streamed_text(events: list[str]) -> str:
+    chunks = [_StreamChunk.model_validate_json(event) for event in events]
+    return "".join(choice.delta.content or "" for chunk in chunks for choice in chunk.choices)
+
+
+def _assert_streamed_completion(result: StreamingResponse) -> None:
+    assert result.ok and result.is_streaming, f"stream was not established: {result}"
+    assert result.stream_error is None, f"stream carried an error event: {result.stream_error}"
+    assert len(result.stream_events) > 1, f"stream did not deliver multiple data events: {result}"
+    assert _streamed_text(result.stream_events).strip(), (
+        f"stream completed with no content deltas: {result.stream_events[:3]}"
+    )
+
+
+def _assert_request_id_header(result: StreamingResponse) -> None:
+    forwarded = [name for name in result.headers if name.startswith(PROVIDER_HEADER_PREFIX)]
+    assert result.headers.get(BEDROCK_REQUEST_ID_HEADER), (
+        f"missing {BEDROCK_REQUEST_ID_HEADER}; forwarded provider headers: {forwarded}"
+    )
+
+
+def _assert_completion(response: ChatResponse) -> None:
+    assert response.choices, f"completion returned no choices: {response}"
+    message = response.choices[0].message
+    content = (message.content if message else None) or ""
+    assert content.strip(), f"completion carried no content: {response}"
+
+
+def _register_bedrock_model(
+    client: PassthroughClient, resources: ResourceManager, prefix: str, backend: str
+) -> str:
+    model = f"{prefix}-{unique_marker()}"
+    model_id = client.proxy.create_model(
+        model,
+        LiteLLMParamsBody(
+            model=backend,
+            aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
+            aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
+            aws_region_name="os.environ/AWS_REGION",
+        ),
+    )
+    resources.defer(lambda: client.proxy.delete_model(model_id))
+    return model
+
+
+def _prompt() -> list[ChatMessage]:
+    return [ChatMessage(role="user", content="reply with one word")]
+
+
+class TestBedrockResponseHeaders:
+    @pytest.mark.covers(
+        "llm.chat_completions.bedrock_converse.response_headers.nonstream.works",
+        exercised_on=[],
+    )
+    def test_bedrock_request_id_header_surfaces(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        model = _register_bedrock_model(client, resources, "e2e-bedrock-headers", CONVERSE_REGIONAL_BACKEND)
+        key = resources.key()
+
+        result = client.proxy.transport.send(
+            "/chat/completions",
+            headers=client.proxy.transport.bearer(key),
+            json=ChatBody(model=model, messages=_prompt(), max_tokens=64),
+        )
+
+        assert result.ok, f"chat call failed: {result.status_code} {result.body[:300]}"
+        _assert_request_id_header(result)
+
+    @pytest.mark.covers(
+        "llm.chat_completions.bedrock_converse.response_headers.stream.works",
+        exercised_on=[],
+    )
+    def test_bedrock_request_id_header_surfaces_on_stream(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        model = _register_bedrock_model(
+            client, resources, "e2e-bedrock-headers-stream", CONVERSE_REGIONAL_BACKEND
+        )
+        key = resources.key()
+
+        result = client.proxy.chat_stream(
+            key, ChatBody(model=model, messages=_prompt(), stream=True, max_tokens=64)
+        )
+
+        _assert_streamed_completion(result)
+        _assert_request_id_header(result)
+
+
+class TestBedrockInvokeRegionalModelIds:
+    @pytest.mark.covers("llm.chat_completions.bedrock_invoke.basic.nonstream.works", exercised_on=[])
+    def test_invoke_regional_id_completes(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        model = _register_bedrock_model(client, resources, "e2e-bedrock-invoke", INVOKE_REGIONAL_BACKEND)
+        key = resources.key()
+
+        response = unwrap(client.proxy.chat(key, ChatBody(model=model, messages=_prompt(), max_tokens=64)))
+
+        _assert_completion(response)
+
+    @pytest.mark.covers("llm.chat_completions.bedrock_invoke.basic.stream.works", exercised_on=[])
+    def test_invoke_regional_id_streams(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        model = _register_bedrock_model(client, resources, "e2e-bedrock-invoke-stream", INVOKE_REGIONAL_BACKEND)
+        key = resources.key()
+
+        result = client.proxy.chat_stream(
+            key, ChatBody(model=model, messages=_prompt(), stream=True, max_tokens=64)
+        )
+
+        _assert_streamed_completion(result)
diff --git a/tests/e2e/management/management_client.py b/tests/e2e/management/management_client.py
index cdc31aeea79..b2bd41e19ba 100644
--- a/tests/e2e/management/management_client.py
+++ b/tests/e2e/management/management_client.py
@@ -14,6 +14,8 @@ from e2e_http import NoBody, ProbeResult, Result, StreamingResponse, Success, Un
 from models import (
     ChatBody,
     ChatMessage,
+    ConnectionTestBody,
+    ConnectionTestResponse,
     CustomerDeleteBody,
     CustomerInfoParams,
     CustomerNewBody,
@@ -118,6 +120,17 @@ class ManagementClient:
             )
         )
 
+    def connection_test(self, body: ConnectionTestBody) -> Result[ConnectionTestResponse]:
+        """POST /health/test_connection, the call behind the Admin UI's Test
+        Connection button, probing the live provider with the supplied params."""
+        return self.proxy.transport.post(
+            "/health/test_connection",
+            headers=self.proxy.transport.master,
+            json=body,
+            response_type=ConnectionTestResponse,
+            timeout=120.0,
+        )
+
     def block_key(self, key: str) -> None:
         _ = unwrap(
             self.proxy.transport.post(
diff --git a/tests/e2e/management/test_model_test_connection_e2e.py b/tests/e2e/management/test_model_test_connection_e2e.py
new file mode 100644
index 00000000000..a1f714df4c8
--- /dev/null
+++ b/tests/e2e/management/test_model_test_connection_e2e.py
@@ -0,0 +1,42 @@
+"""Live e2e for POST /health/test_connection, the API behind the Admin UI's
+Test Connection button on the add-model form.
+
+The covered cell is a responses-mode Bedrock Mantle deployment: exactly this
+shape 500ed on a functools.partial acompletion conflict before v1.91.0 while
+every chat-mode probe stayed green, so the happy path asserts a real success
+verdict from the live provider rather than just a 200 envelope. The region is a
+literal because the endpoint rejects request-supplied os.environ/ references;
+credentials fall through to the proxy's own environment (bearer token locally,
+pod identity in CI).
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from e2e_http import unwrap
+from management_client import ManagementClient
+from models import ConnectionTestBody, LiteLLMParamsBody
+
+pytestmark = pytest.mark.e2e
+
+MANTLE_RESPONSES_BACKEND = "bedrock_mantle/openai.gpt-5.6-luna"
+MANTLE_REGION = "us-east-1"
+
+
+class TestModelTestConnection:
+    @pytest.mark.covers("mgmt.model.test_connection.happy_path")
+    def test_bedrock_mantle_responses_connection_succeeds(self, client: ManagementClient) -> None:
+        response = unwrap(
+            client.connection_test(
+                ConnectionTestBody(
+                    litellm_params=LiteLLMParamsBody(
+                        model=MANTLE_RESPONSES_BACKEND, aws_region_name=MANTLE_REGION
+                    ),
+                    mode="responses",
+                )
+            )
+        )
+
+        error = response.result.error if response.result else None
+        assert response.status == "success", f"test_connection reported an error: {error}"
diff --git a/tests/e2e/models.py b/tests/e2e/models.py
index 5e2cb90958e..e6bde9770a6 100644
--- a/tests/e2e/models.py
+++ b/tests/e2e/models.py
@@ -820,6 +820,26 @@ class ModelDeleteBody(BaseModel):
     id: str
 
 
+class ConnectionTestBody(BaseModel):
+    """POST /health/test_connection body, the API behind the Admin UI's Test
+    Connection button: the deployment params as typed into the add-model form and
+    the health-check mode picking which endpoint the probe calls. The endpoint
+    rejects `os.environ/` references, so credentials are either literal values or
+    omitted to fall through to the proxy's own environment."""
+
+    litellm_params: LiteLLMParamsBody
+    mode: Literal["chat", "completion", "embedding", "responses"]
+
+
+class ConnectionTestResult(BaseModel):
+    error: str | None = None
+
+
+class ConnectionTestResponse(BaseModel):
+    status: Literal["success", "error"]
+    result: ConnectionTestResult | None = None
+
+
 class CredentialCreateBody(BaseModel):
     credential_name: str
     credential_values: dict[str, str]

From fd1dca05deb0fd4d50153b42412f716e341236c3 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 10:19:56 -0700
Subject: [PATCH 031/105] fix(bedrock_mantle): type the codex item dispatcher
 without Any

---
 litellm/llms/bedrock_mantle/responses/transformation.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/litellm/llms/bedrock_mantle/responses/transformation.py b/litellm/llms/bedrock_mantle/responses/transformation.py
index 9068a641940..3e5dd4ff87d 100644
--- a/litellm/llms/bedrock_mantle/responses/transformation.py
+++ b/litellm/llms/bedrock_mantle/responses/transformation.py
@@ -288,7 +288,7 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
         return rewritten
 
     @classmethod
-    def _normalize_codex_input_item(cls, item: object) -> "tuple[Any, str | None]":
+    def _normalize_codex_input_item(cls, item: object) -> "tuple[object, str | None]":
         """Returns (normalized item or None to drop it, original type when rewritten)."""
         if not isinstance(item, dict):
             return item, None
@@ -324,7 +324,8 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
                 "Bedrock Mantle Responses API: rewrote Codex input item type(s) %s that Mantle rejects.",
                 rewritten_types,
             )
-        return [item for item, _ in normalized if item is not None]  # mutable-ok: ResponseInputParam is a list
+        kept: Final = [item for item, _ in normalized if item is not None]  # mutable-ok: ResponseInputParam is a list
+        return kept  # pyright: ignore[reportReturnType]  # Codex passthrough items sit outside the OpenAI input union
 
     def map_openai_params(
         self,

From db8c49305ead5151c2c9b7d9bdfb5cf388b1c140 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 10:24:28 -0700
Subject: [PATCH 032/105] test: type the cost map fixture instead of bare dict

---
 .../test_together_ai_model_metadata.py        | 38 ++++++++++++-------
 1 file changed, 25 insertions(+), 13 deletions(-)

diff --git a/tests/test_litellm/test_together_ai_model_metadata.py b/tests/test_litellm/test_together_ai_model_metadata.py
index 7d7712f3c56..5a0aadf4737 100644
--- a/tests/test_litellm/test_together_ai_model_metadata.py
+++ b/tests/test_litellm/test_together_ai_model_metadata.py
@@ -3,11 +3,15 @@ from pathlib import Path
 from typing import Final
 
 import pytest
+from pydantic import TypeAdapter
 
 from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
 
 REPO_ROOT: Final = Path(__file__).parents[2]
 
+CostMap = dict[str, dict[str, object]]
+COST_MAP_ADAPTER: Final = TypeAdapter(CostMap)
+
 SERVERLESS_CHAT_MODELS: Final = (
     "together_ai/moonshotai/Kimi-K3",
     "together_ai/zai-org/GLM-5.2",
@@ -66,13 +70,13 @@ DEPRECATED_MODELS: Final = {
 
 
 @pytest.fixture(scope="module")
-def cost_map() -> dict:
+def cost_map() -> CostMap:
     with open(REPO_ROOT / "model_prices_and_context_window.json") as f:
-        return json.load(f)
+        return COST_MAP_ADAPTER.validate_python(json.load(f))
 
 
 @pytest.mark.parametrize("model", SERVERLESS_CHAT_MODELS)
-def test_together_serverless_chat_model_is_mapped(cost_map: dict, model: str):
+def test_together_serverless_chat_model_is_mapped(cost_map: CostMap, model: str):
     info = cost_map.get(model)
     assert info is not None, f"{model} missing from model_prices_and_context_window.json"
     assert info["litellm_provider"] == "together_ai"
@@ -86,7 +90,7 @@ def test_together_serverless_chat_model_is_mapped(cost_map: dict, model: str):
     assert provider == "together_ai"
 
 
-def test_together_kimi_k3_pricing_and_capabilities(cost_map: dict):
+def test_together_kimi_k3_pricing_and_capabilities(cost_map: CostMap):
     info = cost_map["together_ai/moonshotai/Kimi-K3"]
     assert info["input_cost_per_token"] == 3e-06
     assert info["output_cost_per_token"] == 1.5e-05
@@ -98,7 +102,7 @@ def test_together_kimi_k3_pricing_and_capabilities(cost_map: dict):
     assert info["supports_reasoning"] is True
 
 
-def test_together_glm_52_pricing(cost_map: dict):
+def test_together_glm_52_pricing(cost_map: CostMap):
     info = cost_map["together_ai/zai-org/GLM-5.2"]
     assert info["input_cost_per_token"] == 1.4e-06
     assert info["output_cost_per_token"] == 4.4e-06
@@ -106,7 +110,7 @@ def test_together_glm_52_pricing(cost_map: dict):
     assert info["supports_reasoning"] is True
 
 
-def test_together_multilingual_e5_embedding_entry(cost_map: dict):
+def test_together_multilingual_e5_embedding_entry(cost_map: CostMap):
     info = cost_map["together_ai/intfloat/multilingual-e5-large-instruct"]
     assert info["mode"] == "embedding"
     assert info["input_cost_per_token"] == 2e-08
@@ -114,7 +118,7 @@ def test_together_multilingual_e5_embedding_entry(cost_map: dict):
     assert info["output_vector_size"] == 1024
 
 
-def test_together_llama_33_70b_repriced_to_current_together_rate(cost_map: dict):
+def test_together_llama_33_70b_repriced_to_current_together_rate(cost_map: CostMap):
     info = cost_map["together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo"]
     assert info["input_cost_per_token"] == 1.04e-06
     assert info["output_cost_per_token"] == 1.04e-06
@@ -122,17 +126,25 @@ def test_together_llama_33_70b_repriced_to_current_together_rate(cost_map: dict)
 
 
 @pytest.mark.parametrize("model", sorted(DEPRECATED_MODELS))
-def test_together_deprecated_model_carries_deprecation_date(cost_map: dict, model: str):
+def test_together_deprecated_model_carries_deprecation_date(cost_map: CostMap, model: str):
     info = cost_map.get(model)
     assert info is not None, f"{model} missing from model_prices_and_context_window.json"
     assert info.get("deprecation_date") == DEPRECATED_MODELS[model]
 
 
-def test_together_successor_metadata_points_at_live_models(cost_map: dict):
+def _successor(info: dict[str, object]) -> str | None:
+    metadata = info.get("metadata")
+    if not isinstance(metadata, dict):
+        return None
+    successor = metadata.get("successor")
+    return successor if isinstance(successor, str) else None
+
+
+def test_together_successor_metadata_points_at_live_models(cost_map: CostMap):
     successors = {
-        model: info["metadata"]["successor"]
+        model: successor
         for model, info in cost_map.items()
-        if model.startswith("together_ai/") and "successor" in info.get("metadata", {})
+        if model.startswith("together_ai/") and (successor := _successor(info)) is not None
     }
     assert len(successors) >= 10
     for model, successor in successors.items():
@@ -141,9 +153,9 @@ def test_together_successor_metadata_points_at_live_models(cost_map: dict):
         assert "deprecation_date" not in target, f"{model} names deprecated successor {successor}"
 
 
-def test_together_backup_cost_map_in_sync(cost_map: dict):
+def test_together_backup_cost_map_in_sync(cost_map: CostMap):
     with open(REPO_ROOT / "litellm" / "model_prices_and_context_window_backup.json") as f:
-        backup = json.load(f)
+        backup = COST_MAP_ADAPTER.validate_python(json.load(f))
     together_main = {k: v for k, v in cost_map.items() if k.startswith("together_ai/")}
     together_backup = {k: v for k, v in backup.items() if k.startswith("together_ai/")}
     assert together_backup == together_main

From 367a6e5dc5fc1e23d31b7395d96fa2a5332fd6d2 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 10:37:48 -0700
Subject: [PATCH 033/105] test(router): pin the guard that keeps a junk-typed
 operator effort value out of model group info

---
 tests/test_litellm/test_router.py | 34 +++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py
index 5f1941574eb..e8830d05065 100644
--- a/tests/test_litellm/test_router.py
+++ b/tests/test_litellm/test_router.py
@@ -9092,6 +9092,40 @@ def test_model_group_info_reasoning_efforts_ignore_a_value_declared_in_model_inf
     assert result.supported_reasoning_efforts == ("none", "minimal", "low", "medium", "high")
 
 
+def test_model_group_info_survives_a_junk_typed_operator_effort_value():
+    """A deployment's registered model_info reads back with whatever the operator wrote under any
+    key, so a wrong-typed supported_reasoning_efforts must not fail the group's info. Only the
+    constructor's trailing override keeps the junk away from ModelGroupInfo validation."""
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "junk-declared-group",
+                "litellm_params": {"model": "openai/lone-reasoner"},
+                "model_info": {"id": "junk-deployment"},
+            },
+        ]
+    )
+
+    def _model_info(model_id: str, model_name: str):
+        return {
+            "key": model_name,
+            "litellm_provider": "openai",
+            "mode": "chat",
+            "supports_reasoning": True,
+            "supports_none_reasoning_effort": True,
+            "supported_reasoning_efforts": "high",
+        }
+
+    with patch.object(router, "get_deployment_model_info", side_effect=_model_info):
+        result = router._set_model_group_info(
+            model_group="junk-declared-group",
+            user_facing_model_group_name="junk-declared-group",
+        )
+
+    assert result is not None
+    assert result.supported_reasoning_efforts == ("none", "minimal", "low", "medium", "high")
+
+
 def test_model_group_info_reasoning_efforts_ignore_a_mode_the_operator_declared():
     """A deployment is registered in the cost map under its own id with whatever model_info the
     operator wrote, so a mode they set themselves reads back exactly like one the map supplied. Only

From 0ea6f5e159350aa75e9118ba027b7286074cb0fe Mon Sep 17 00:00:00 2001
From: ryan-crabbe-berri <ryan@berri.ai>
Date: Tue, 25 Aug 2026 10:50:11 -0700
Subject: [PATCH 034/105] fix(azure_ai): stamp the model router's selected
 model instead of matching on the model name

The model Azure Model Router served was recovered by checking whether the text
"model_router" or "model-router" appeared in a model string. Spend logs applied that
check to the litellm model path, where the route prefix guarantees a match, but the
proxy applied it to the client's model group alias, which carries no prefix. A model
group named anything else therefore lost the selected model in both the response and
the spend row.

AzureModelRouterConfig now stamps the served model onto _hidden_params, and the spend
log payload and the proxy's response restamping read that stamp. The name heuristic
survives as a fallback for callers with no response in hand, routed through
get_azure_ai_route so it lives in one place.
---
 litellm/litellm_core_utils/litellm_logging.py |  12 +-
 .../azure_model_router/transformation.py      |  23 +-
 litellm/llms/azure_ai/common_utils.py         |  36 ++
 litellm/proxy/common_request_processing.py    |  21 +-
 .../test_litellm_logging.py                   | 472 +++++++-----------
 .../chat/test_azure_ai_transformation.py      |  97 +++-
 .../proxy/test_common_request_processing.py   | 443 +++++++---------
 7 files changed, 538 insertions(+), 566 deletions(-)

diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py
index 9b7707eabe1..01802c3bbb3 100644
--- a/litellm/litellm_core_utils/litellm_logging.py
+++ b/litellm/litellm_core_utils/litellm_logging.py
@@ -5762,11 +5762,15 @@ def get_standard_logging_object_payload(
             response_model_name = final_response_obj.get("model")
 
         # For Azure Model Router, preserve the actual model in the top-level standard
-        # logging payload only when the user has opted in.
+        # logging payload.
+        from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
+
         requested_model: Final = kwargs.get("model")
-        if (
-            isinstance(requested_model, str)
-            and ("model_router" in requested_model.lower() or "model-router" in requested_model.lower())
+        stamped_selected_model: Final = AzureFoundryModelInfo.get_model_router_selected_model(hidden_params)
+        if stamped_selected_model is not None:
+            model_name = stamped_selected_model
+        elif (
+            AzureFoundryModelInfo.is_model_router_call(model=requested_model, hidden_params=hidden_params)
             and isinstance(response_model_name, str)
             and response_model_name
         ):
diff --git a/litellm/llms/azure_ai/azure_model_router/transformation.py b/litellm/llms/azure_ai/azure_model_router/transformation.py
index 1a924088390..d33564c0f9a 100644
--- a/litellm/llms/azure_ai/azure_model_router/transformation.py
+++ b/litellm/llms/azure_ai/azure_model_router/transformation.py
@@ -65,15 +65,24 @@ class AzureModelRouterConfig(AzureAIStudioConfig):
 
         Extracts the actual model used from the Azure response (e.g., gpt-5-nano-2025-08-07)
         and returns it with the azure_ai/ prefix for proper display and cost tracking.
+
+        Also stamps that model onto ``_hidden_params`` so downstream consumers (spend logs,
+        response restamping) can read it instead of guessing the route from the model string.
         """
-        from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
+        from litellm.llms.azure_ai.common_utils import (
+            AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY,
+            AzureFoundryModelInfo,
+        )
+        from litellm.router_utils.add_retry_fallback_headers import (
+            get_hidden_params_dict,
+        )
 
         # Get base model for the parent call (strips routing prefixes for API compatibility)
         base_model: Final[str] = AzureFoundryModelInfo.get_base_model(model)
 
         # Call parent transform_response first - this will extract the actual model
         # from the raw response (e.g., "gpt-5-nano-2025-08-07")
-        model_response = super().transform_response(
+        transformed_response: Final = super().transform_response(
             model=base_model,
             raw_response=raw_response,
             model_response=model_response,
@@ -86,7 +95,15 @@ class AzureModelRouterConfig(AzureAIStudioConfig):
             api_key=api_key,
             json_mode=json_mode,
         )
-        return model_response
+        selected_model: Final = transformed_response.model
+        if selected_model:
+            # Rebuilt rather than mutated in place: ModelResponseBase declares _hidden_params as a
+            # class-level dict, so an in-place write can bleed into unrelated responses.
+            transformed_response._hidden_params = {  # pyright: ignore[reportPrivateUsage]  # ModelResponse exposes no public hidden-params setter
+                **get_hidden_params_dict(transformed_response),
+                AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY: selected_model,
+            }
+        return transformed_response
 
     def calculate_additional_costs(self, model: str, prompt_tokens: int, completion_tokens: int) -> dict | None:
         """
diff --git a/litellm/llms/azure_ai/common_utils.py b/litellm/llms/azure_ai/common_utils.py
index d25a8fd6561..9d37d8d1185 100644
--- a/litellm/llms/azure_ai/common_utils.py
+++ b/litellm/llms/azure_ai/common_utils.py
@@ -1,3 +1,4 @@
+from collections.abc import Mapping
 from typing import Final, Literal
 
 import litellm
@@ -5,6 +6,8 @@ from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import AllMessageValues
 
+AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY: Final = "azure_model_router_selected_model"
+
 
 class AzureFoundryModelInfo(BaseLLMModelInfo):
     """Model info for Azure AI / Azure Foundry models."""
@@ -37,6 +40,39 @@ class AzureFoundryModelInfo(BaseLLMModelInfo):
             return "model_router"
         return "default"
 
+    @staticmethod
+    def get_model_router_selected_model(hidden_params: Mapping[str, object] | None) -> str | None:
+        """The model Azure Model Router actually served, stamped by ``AzureModelRouterConfig``.
+
+        Reading this beats re-deriving the route from a model string: the stamp is set on the
+        code path that was actually taken, so it holds no matter what the caller named the model.
+        """
+        if not hidden_params:
+            return None
+        selected: Final = hidden_params.get(AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY)
+        if isinstance(selected, str) and selected:
+            return selected
+        return None
+
+    @staticmethod
+    def is_model_router_call(
+        model: str | None = None,
+        hidden_params: Mapping[str, object] | None = None,
+    ) -> bool:
+        """Whether a request went down the Azure Model Router route.
+
+        Prefers the response stamp, then the deployment's litellm model path, and only then the
+        caller-supplied name. The last two go through ``get_azure_ai_route`` so the model-router
+        name heuristic lives in exactly one place.
+        """
+        if AzureFoundryModelInfo.get_model_router_selected_model(hidden_params) is not None:
+            return True
+        deployment_model: Final = (hidden_params or {}).get("litellm_model_name") or (hidden_params or {}).get("model")
+        return any(
+            isinstance(candidate, str) and AzureFoundryModelInfo.get_azure_ai_route(candidate) == "model_router"
+            for candidate in (deployment_model, model)
+        )
+
     @staticmethod
     def get_api_base(api_base: str | None = None) -> str | None:
         return api_base or litellm.api_base or get_secret_str("AZURE_AI_API_BASE")
diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py
index e5714ef66fb..37a947f7f19 100644
--- a/litellm/proxy/common_request_processing.py
+++ b/litellm/proxy/common_request_processing.py
@@ -1136,24 +1136,25 @@ async def open_sse_before_first_byte(
     )
 
 
-def _is_azure_model_router_request(model: str) -> bool:
+def _is_azure_model_router_request(model: str, hidden_params: Mapping[str, object] | None = None) -> bool:
     """
-    Check if the requested model is an Azure Model Router.
+    Check if a request went down the Azure Model Router route.
 
-    Azure Model Router models follow the pattern:
-    - azure_ai/model_router/<deployment-name>
-    - azure_ai/model-router
-    - model_router/<deployment-name>
-    - model-router
+    ``model`` here is what the *client* sent, a model group alias with no ``model_router/``
+    prefix, so matching on it alone only works when the operator happened to put "model-router"
+    in the alias. Where the response is in hand its stamp answers this outright, so callers
+    should pass ``hidden_params``.
 
     Args:
         model: The requested model name
+        hidden_params: ``_hidden_params`` from the response, when the caller has it
 
     Returns:
         bool: True if this is an Azure Model Router request
     """
-    model_lower: Final = model.lower()
-    return "model-router" in model_lower or "model_router" in model_lower
+    from litellm.llms.azure_ai.common_utils import AzureFoundryModelInfo
+
+    return AzureFoundryModelInfo.is_model_router_call(model=model, hidden_params=hidden_params)
 
 
 def _override_openai_response_model(
@@ -1221,7 +1222,7 @@ def _override_openai_response_model(
                 return
 
     # Check if this is an Azure Model Router request - if so, preserve the actual model used
-    if _is_azure_model_router_request(requested_model):
+    if _is_azure_model_router_request(requested_model, hidden_params):
         verbose_proxy_logger.debug(
             "%s: Azure Model Router detected - preserving actual model used from response instead of overriding to router model.",
             log_context,
diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
index 82de634b488..a0f7de6b320 100644
--- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
+++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
@@ -6,9 +6,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
 
-sys.path.insert(
-    0, os.path.abspath("../../..")
-)  # Adds the parent directory to the system path
+sys.path.insert(0, os.path.abspath("../../.."))  # Adds the parent directory to the system path
 
 import time
 
@@ -277,9 +275,7 @@ def test_response_cost_calculator_uses_router_model_id_from_litellm_metadata():
 
         assert cost is not None, "Cost should not be None"
         expected_cost = (10 * custom_input_cost) + (5 * custom_output_cost)
-        assert cost == pytest.approx(
-            expected_cost
-        ), f"Expected {expected_cost}, got {cost}"
+        assert cost == pytest.approx(expected_cost), f"Expected {expected_cost}, got {cost}"
     finally:
         litellm.model_cost.pop(custom_model_id, None)
 
@@ -876,13 +872,8 @@ async def test_datadog_logger_not_shadowed_by_llm_obs(monkeypatch):
 
         # Regression check: we expect a distinct DataDogLogger, not the LLM Obs logger
         assert type(datadog_logger) is DataDogLogger
-        assert any(
-            isinstance(cb, DataDogLLMObsLogger)
-            for cb in logging_module._in_memory_loggers
-        )
-        assert any(
-            type(cb) is DataDogLogger for cb in logging_module._in_memory_loggers
-        )
+        assert any(isinstance(cb, DataDogLLMObsLogger) for cb in logging_module._in_memory_loggers)
+        assert any(type(cb) is DataDogLogger for cb in logging_module._in_memory_loggers)
     finally:
         logging_module._in_memory_loggers.clear()
 
@@ -893,9 +884,7 @@ async def test_logfire_logger_accepts_env_vars_for_base_url(monkeypatch):
 
     # Required env vars for Logfire integration
     monkeypatch.setenv("LOGFIRE_TOKEN", "test-token")
-    monkeypatch.setenv(
-        "LOGFIRE_BASE_URL", "https://logfire-api-custom.pydantic.dev"
-    )  # no trailing slash on purpose
+    monkeypatch.setenv("LOGFIRE_BASE_URL", "https://logfire-api-custom.pydantic.dev")  # no trailing slash on purpose
 
     # Import after env vars are set (important if module-level caching exists)
     from litellm.integrations.opentelemetry import OpenTelemetry  # logger class
@@ -914,9 +903,7 @@ async def test_logfire_logger_accepts_env_vars_for_base_url(monkeypatch):
 
         # Sanity: we got the right logger type and it is cached
         assert type(logger) is OpenTelemetry
-        assert any(
-            type(cb) is OpenTelemetry for cb in logging_module._in_memory_loggers
-        )
+        assert any(type(cb) is OpenTelemetry for cb in logging_module._in_memory_loggers)
 
         # Core regression check: base URL env var should influence the exporter endpoint.
         #
@@ -927,9 +914,7 @@ async def test_logfire_logger_accepts_env_vars_for_base_url(monkeypatch):
             or getattr(logger, "config", None)
             or getattr(logger, "_otel_config", None)
         )
-        assert (
-            cfg is not None
-        ), "Expected OpenTelemetry logger to keep an otel config on the instance"
+        assert cfg is not None, "Expected OpenTelemetry logger to keep an otel config on the instance"
 
         endpoint = getattr(cfg, "endpoint", None) or getattr(cfg, "otlp_endpoint", None)
         assert endpoint is not None, "Expected otel config to expose the OTLP endpoint"
@@ -1087,9 +1072,7 @@ async def test_logging_non_streaming_request():
 
             # Use the filtered call for assertions
             call_args = calls_with_expected_input[0]
-            standard_logging_object = call_args.kwargs["kwargs"][
-                "standard_logging_object"
-            ]
+            standard_logging_object = call_args.kwargs["kwargs"]["standard_logging_object"]
             assert standard_logging_object["stream"] is not True
     finally:
         # Restore original callbacks to ensure test isolation
@@ -1107,18 +1090,14 @@ async def test_logging_non_streaming_request():
         "agenerate_content_stream",
     ],
 )
-def test_success_handler_skips_sync_callbacks_for_async_requests(
-    logging_obj, async_flag
-):
+def test_success_handler_skips_sync_callbacks_for_async_requests(logging_obj, async_flag):
     """Ensure sync success callbacks are skipped when async call type flags are set."""
     from litellm.integrations.custom_logger import CustomLogger
 
     class DummyLogger(CustomLogger):
         pass
 
-    logging_obj.stream = (
-        False  # simulate non-streaming request where sync callbacks would normally run
-    )
+    logging_obj.stream = False  # simulate non-streaming request where sync callbacks would normally run
     logging_obj.model_call_details["litellm_params"] = {async_flag: True}
     logging_obj.litellm_params = logging_obj.model_call_details["litellm_params"]
 
@@ -1194,21 +1173,11 @@ def test_success_handler_runs_sync_callbacks_for_sync_requests(logging_obj, call
 def test_is_sync_litellm_request():
     assert LitellmLogging._is_sync_litellm_request({}) is True
     assert LitellmLogging._is_sync_litellm_request({"acompletion": True}) is False
-    assert (
-        LitellmLogging._is_sync_litellm_request({"allm_passthrough_route": True})
-        is False
-    )
-    assert (
-        LitellmLogging._is_sync_litellm_request({"aanthropic_messages": True}) is False
-    )
+    assert LitellmLogging._is_sync_litellm_request({"allm_passthrough_route": True}) is False
+    assert LitellmLogging._is_sync_litellm_request({"aanthropic_messages": True}) is False
     assert LitellmLogging._is_sync_litellm_request({"agenerate_content": True}) is False
-    assert (
-        LitellmLogging._is_sync_litellm_request({"agenerate_content_stream": True})
-        is False
-    )
-    assert (
-        LitellmLogging._is_sync_litellm_request({"aanthropic_messages": False}) is True
-    )
+    assert LitellmLogging._is_sync_litellm_request({"agenerate_content_stream": True}) is False
+    assert LitellmLogging._is_sync_litellm_request({"aanthropic_messages": False}) is True
 
 
 def test_get_litellm_params_propagates_allm_passthrough_route():
@@ -1255,9 +1224,7 @@ async def test_dispatch_success_handlers_invokes_callbacks_once_for_final_stream
         logging_obj.model_call_details["litellm_params"] = {"acompletion": True}
 
         with (
-            patch.object(
-                mock_callback, "async_log_success_event", new_callable=AsyncMock
-            ) as mock_async_log,
+            patch.object(mock_callback, "async_log_success_event", new_callable=AsyncMock) as mock_async_log,
             patch.object(mock_callback, "log_success_event") as mock_sync_log,
             patch.object(
                 logging_obj,
@@ -1318,9 +1285,7 @@ async def test_dispatch_success_handlers_sync_path_invokes_callback_once_for_fin
 
         with (
             patch.object(mock_callback, "log_success_event") as mock_sync_log,
-            patch.object(
-                mock_callback, "async_log_success_event", new_callable=AsyncMock
-            ) as mock_async_log,
+            patch.object(mock_callback, "async_log_success_event", new_callable=AsyncMock) as mock_async_log,
             patch.object(
                 logging_obj,
                 "_success_handler_helper_fn",
@@ -1362,20 +1327,14 @@ async def test_dispatch_prefer_async_handlers_runs_legacy_callbacks(
     logging_obj.model_call_details["litellm_params"] = {}
 
     with (
-        patch.object(
-            logging_obj, "async_success_handler", new_callable=AsyncMock
-        ) as mock_async,
-        patch.object(
-            logging_obj, "success_handler", new_callable=MagicMock
-        ) as mock_sync,
+        patch.object(logging_obj, "async_success_handler", new_callable=AsyncMock) as mock_async,
+        patch.object(logging_obj, "success_handler", new_callable=MagicMock) as mock_sync,
         patch.object(
             logging_obj,
             "_should_run_sync_callbacks_for_async_calls",
             return_value=True,
         ),
-        patch(
-            "litellm.litellm_core_utils.litellm_logging.executor.submit"
-        ) as mock_submit,
+        patch("litellm.litellm_core_utils.litellm_logging.executor.submit") as mock_submit,
     ):
         await logging_obj.dispatch_success_handlers(
             result=result,
@@ -1409,9 +1368,7 @@ async def test_dispatch_success_handlers_invokes_async_callback_for_pass_through
 
     try:
         with (
-            patch.object(
-                mock_callback, "async_log_success_event", new_callable=AsyncMock
-            ) as mock_async_log,
+            patch.object(mock_callback, "async_log_success_event", new_callable=AsyncMock) as mock_async_log,
             patch.object(mock_callback, "log_success_event") as mock_sync_log,
         ):
             await logging_obj.dispatch_success_handlers(result={"id": "pt-1"})
@@ -1438,20 +1395,14 @@ async def test_dispatch_failure_handlers_prefer_async_does_not_submit_sync_handl
     logging_obj.model_call_details["litellm_params"] = {}
 
     with (
-        patch.object(
-            logging_obj, "async_failure_handler", new_callable=AsyncMock
-        ) as mock_async,
-        patch.object(
-            logging_obj, "failure_handler", new_callable=MagicMock
-        ) as mock_sync,
+        patch.object(logging_obj, "async_failure_handler", new_callable=AsyncMock) as mock_async,
+        patch.object(logging_obj, "failure_handler", new_callable=MagicMock) as mock_sync,
         patch.object(
             logging_obj,
             "_should_run_sync_failure_callbacks_for_async_calls",
             return_value=False,
         ),
-        patch(
-            "litellm.litellm_core_utils.litellm_logging.executor.submit"
-        ) as mock_submit,
+        patch("litellm.litellm_core_utils.litellm_logging.executor.submit") as mock_submit,
     ):
         await logging_obj.dispatch_failure_handlers(
             exception,
@@ -1534,12 +1485,8 @@ async def test_dispatch_failure_handlers_submits_sync_handler_for_failure_only_c
         patch.object(litellm, "success_callback", []),
         patch.object(litellm, "failure_callback", [_sync_failure_callback]),
         patch.object(logging_obj, "async_failure_handler", new_callable=AsyncMock),
-        patch.object(
-            logging_obj, "failure_handler", new_callable=MagicMock
-        ) as mock_sync,
-        patch(
-            "litellm.litellm_core_utils.litellm_logging.executor.submit"
-        ) as mock_submit,
+        patch.object(logging_obj, "failure_handler", new_callable=MagicMock) as mock_sync,
+        patch("litellm.litellm_core_utils.litellm_logging.executor.submit") as mock_submit,
     ):
         await logging_obj.dispatch_failure_handlers(
             exception,
@@ -1566,15 +1513,9 @@ async def test_dispatch_failure_handlers_sync_sdk_shortcut_runs_sync_handler_inl
     logging_obj.model_call_details["litellm_params"] = {}
 
     with (
-        patch.object(
-            logging_obj, "async_failure_handler", new_callable=AsyncMock
-        ) as mock_async,
-        patch.object(
-            logging_obj, "failure_handler", new_callable=MagicMock
-        ) as mock_sync,
-        patch(
-            "litellm.litellm_core_utils.litellm_logging.executor.submit"
-        ) as mock_submit,
+        patch.object(logging_obj, "async_failure_handler", new_callable=AsyncMock) as mock_async,
+        patch.object(logging_obj, "failure_handler", new_callable=MagicMock) as mock_sync,
+        patch("litellm.litellm_core_utils.litellm_logging.executor.submit") as mock_submit,
     ):
         await logging_obj.dispatch_failure_handlers(
             exception,
@@ -1621,14 +1562,10 @@ def test_success_handler_skips_guardrail_logging_hook_when_disabled(logging_obj)
         event_hook=GuardrailEventHooks.logging_only,
     )
     guardrail.should_run_guardrail = MagicMock(return_value=False)
-    guardrail.logging_hook = MagicMock(
-        return_value=(logging_obj.model_call_details, model_response)
-    )
+    guardrail.logging_hook = MagicMock(return_value=(logging_obj.model_call_details, model_response))
 
     dummy_logger = DummyLogger()
-    dummy_logger.logging_hook = MagicMock(
-        return_value=(logging_obj.model_call_details, model_response)
-    )
+    dummy_logger.logging_hook = MagicMock(return_value=(logging_obj.model_call_details, model_response))
 
     with patch.object(
         logging_obj,
@@ -1762,11 +1699,7 @@ def test_get_request_tags_from_metadata_and_litellm_metadata():
 
     # Test case 2: Tags in litellm_metadata only
     tags = StandardLoggingPayloadSetup._get_request_tags(
-        litellm_params={
-            "litellm_metadata": {
-                "tags": ["litellm-metadata-tag-1", "litellm-metadata-tag-2"]
-            }
-        },
+        litellm_params={"litellm_metadata": {"tags": ["litellm-metadata-tag-1", "litellm-metadata-tag-2"]}},
         proxy_server_request={},
     )
     assert "litellm-metadata-tag-1" in tags
@@ -1871,15 +1804,9 @@ def test_get_request_tags_does_not_mutate_original_tags():
     user_agent_count_2 = len([t for t in tags2 if t.startswith("User-Agent:")])
     user_agent_count_3 = len([t for t in tags3 if t.startswith("User-Agent:")])
 
-    assert (
-        user_agent_count_1 == 2
-    ), f"Expected 2 User-Agent tags, got {user_agent_count_1}"
-    assert (
-        user_agent_count_2 == 2
-    ), f"Expected 2 User-Agent tags, got {user_agent_count_2}"
-    assert (
-        user_agent_count_3 == 2
-    ), f"Expected 2 User-Agent tags, got {user_agent_count_3}"
+    assert user_agent_count_1 == 2, f"Expected 2 User-Agent tags, got {user_agent_count_1}"
+    assert user_agent_count_2 == 2, f"Expected 2 User-Agent tags, got {user_agent_count_2}"
+    assert user_agent_count_3 == 2, f"Expected 2 User-Agent tags, got {user_agent_count_3}"
 
     # Verify all returned lists are independent (different objects)
     assert tags1 is not tags2
@@ -1912,9 +1839,7 @@ def test_get_extra_header_tags():
 
         # Test case 3: Extra headers configured but request has no headers dict
         litellm.extra_spend_tag_headers = ["x-custom", "x-tenant"]
-        result = StandardLoggingPayloadSetup._get_extra_header_tags(
-            proxy_server_request={"headers": "not-a-dict"}
-        )
+        result = StandardLoggingPayloadSetup._get_extra_header_tags(proxy_server_request={"headers": "not-a-dict"})
         assert result is None
 
         # Test case 4: Extra headers configured but none match request headers
@@ -2215,9 +2140,7 @@ def test_get_masked_values():
         "presidio_anonymizer_api_base": None,
         "vertex_credentials": "{sensitive_api_key}",
     }
-    masked_values = _get_masked_values(
-        sensitive_object, unmasked_length=4, number_of_asterisks=4
-    )
+    masked_values = _get_masked_values(sensitive_object, unmasked_length=4, number_of_asterisks=4)
     assert masked_values["presidio_anonymizer_api_base"] is None
     assert masked_values["vertex_credentials"] == "{s****y}"
 
@@ -2242,9 +2165,7 @@ async def test_e2e_generate_cold_storage_object_key_successful():
         patch("litellm.integrations.s3.get_s3_object_key") as mock_get_s3_key,
     ):
         # Mock the S3 object key generation to return a predictable result
-        mock_get_s3_key.return_value = (
-            "2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json"
-        )
+        mock_get_s3_key.return_value = "2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json"
 
         # Call the function
         result = StandardLoggingPayloadSetup._generate_cold_storage_object_key(
@@ -2285,16 +2206,12 @@ async def test_e2e_generate_cold_storage_object_key_with_custom_logger_s3_path()
 
     with (
         patch("litellm.cold_storage_custom_logger", "s3_v2"),
-        patch(
-            "litellm.logging_callback_manager.get_active_custom_logger_for_callback_name"
-        ) as mock_get_logger,
+        patch("litellm.logging_callback_manager.get_active_custom_logger_for_callback_name") as mock_get_logger,
         patch("litellm.integrations.s3.get_s3_object_key") as mock_get_s3_key,
     ):
         # Setup mocks
         mock_get_logger.return_value = mock_custom_logger
-        mock_get_s3_key.return_value = (
-            "storage/2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json"
-        )
+        mock_get_s3_key.return_value = "storage/2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json"
 
         # Call the function
         result = StandardLoggingPayloadSetup._generate_cold_storage_object_key(
@@ -2313,9 +2230,7 @@ async def test_e2e_generate_cold_storage_object_key_with_custom_logger_s3_path()
         )
 
         # Verify the result
-        assert (
-            result == "storage/2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json"
-        )
+        assert result == "storage/2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json"
 
 
 @pytest.mark.asyncio
@@ -2338,16 +2253,12 @@ async def test_e2e_generate_cold_storage_object_key_with_logger_no_s3_path():
 
     with (
         patch("litellm.cold_storage_custom_logger", "s3_v2"),
-        patch(
-            "litellm.logging_callback_manager.get_active_custom_logger_for_callback_name"
-        ) as mock_get_logger,
+        patch("litellm.logging_callback_manager.get_active_custom_logger_for_callback_name") as mock_get_logger,
         patch("litellm.integrations.s3.get_s3_object_key") as mock_get_s3_key,
     ):
         # Setup mocks
         mock_get_logger.return_value = mock_custom_logger
-        mock_get_s3_key.return_value = (
-            "2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json"
-        )
+        mock_get_s3_key.return_value = "2025-01-15/time-10-30-45-123456_chatcmpl-test-12345.json"
 
         # Call the function
         result = StandardLoggingPayloadSetup._generate_cold_storage_object_key(
@@ -2463,9 +2374,7 @@ def test_get_usage_as_dict():
     assert result == {"prompt_tokens": 20, "completion_tokens": 30}
 
     # Test case 5: response_obj with no usage key returns empty
-    result = StandardLoggingPayloadSetup.get_usage_as_dict(
-        response_obj={"id": "resp-1", "choices": []}
-    )
+    result = StandardLoggingPayloadSetup.get_usage_as_dict(response_obj={"id": "resp-1", "choices": []})
     assert result == {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
 
 
@@ -2478,26 +2387,20 @@ def test_append_system_prompt_messages():
     # Test case 1: system in kwargs with existing messages
     kwargs = {"system": "You are a helpful assistant"}
     messages = [{"role": "user", "content": "Hello"}]
-    result = StandardLoggingPayloadSetup.append_system_prompt_messages(
-        kwargs=kwargs, messages=messages
-    )
+    result = StandardLoggingPayloadSetup.append_system_prompt_messages(kwargs=kwargs, messages=messages)
     assert len(result) == 2
     assert result[0] == {"role": "system", "content": "You are a helpful assistant"}
     assert result[1] == {"role": "user", "content": "Hello"}
 
     # Test case 2: system in kwargs with None messages
     kwargs = {"system": "You are a helpful assistant"}
-    result = StandardLoggingPayloadSetup.append_system_prompt_messages(
-        kwargs=kwargs, messages=None
-    )
+    result = StandardLoggingPayloadSetup.append_system_prompt_messages(kwargs=kwargs, messages=None)
     assert len(result) == 1
     assert result[0] == {"role": "system", "content": "You are a helpful assistant"}
 
     # Test case 3: system in kwargs with empty messages list
     kwargs = {"system": "You are a helpful assistant"}
-    result = StandardLoggingPayloadSetup.append_system_prompt_messages(
-        kwargs=kwargs, messages=[]
-    )
+    result = StandardLoggingPayloadSetup.append_system_prompt_messages(kwargs=kwargs, messages=[])
     assert len(result) == 1
     assert result[0] == {"role": "system", "content": "You are a helpful assistant"}
 
@@ -2507,24 +2410,18 @@ def test_append_system_prompt_messages():
         {"role": "system", "content": "You are a helpful assistant"},
         {"role": "user", "content": "Hello"},
     ]
-    result = StandardLoggingPayloadSetup.append_system_prompt_messages(
-        kwargs=kwargs, messages=messages
-    )
+    result = StandardLoggingPayloadSetup.append_system_prompt_messages(kwargs=kwargs, messages=messages)
     assert len(result) == 2
     assert result[0] == {"role": "system", "content": "You are a helpful assistant"}
 
     # Test case 5: no system in kwargs returns messages unchanged
     kwargs = {}
     messages = [{"role": "user", "content": "Hello"}]
-    result = StandardLoggingPayloadSetup.append_system_prompt_messages(
-        kwargs=kwargs, messages=messages
-    )
+    result = StandardLoggingPayloadSetup.append_system_prompt_messages(kwargs=kwargs, messages=messages)
     assert result == messages
 
     # Test case 6: None kwargs returns messages unchanged
-    result = StandardLoggingPayloadSetup.append_system_prompt_messages(
-        kwargs=None, messages=messages
-    )
+    result = StandardLoggingPayloadSetup.append_system_prompt_messages(kwargs=None, messages=messages)
     assert result == messages
 
 
@@ -2585,12 +2482,11 @@ async def test_async_success_handler_sets_standard_logging_object_for_pass_throu
 
     # Verify that standard_logging_object was set
     assert "standard_logging_object" in logging_obj.model_call_details, (
-        "standard_logging_object should be set for pass-through endpoints "
-        "even when complete_streaming_response is None"
+        "standard_logging_object should be set for pass-through endpoints even when complete_streaming_response is None"
+    )
+    assert logging_obj.model_call_details["standard_logging_object"] is not None, (
+        "standard_logging_object should not be None for pass-through endpoints"
     )
-    assert (
-        logging_obj.model_call_details["standard_logging_object"] is not None
-    ), "standard_logging_object should not be None for pass-through endpoints"
 
     # Verify that async_complete_streaming_response was set to prevent re-processing
     # This is consistent with the existing code pattern for regular streaming
@@ -2598,15 +2494,13 @@ async def test_async_success_handler_sets_standard_logging_object_for_pass_throu
         "async_complete_streaming_response should be set to prevent re-processing, "
         "consistent with the existing code pattern"
     )
-    assert (
-        logging_obj.model_call_details["async_complete_streaming_response"] is result
-    ), "async_complete_streaming_response should be set to the result"
+    assert logging_obj.model_call_details["async_complete_streaming_response"] is result, (
+        "async_complete_streaming_response should be set to the result"
+    )
 
     # Verify that response_cost is set to None (cost calculation not possible for pass-through)
     # This is consistent with the error handling in the non-pass-through code path
-    assert (
-        "response_cost" in logging_obj.model_call_details
-    ), "response_cost should be set for pass-through endpoints"
+    assert "response_cost" in logging_obj.model_call_details, "response_cost should be set for pass-through endpoints"
     assert logging_obj.model_call_details["response_cost"] is None, (
         "response_cost should be None for pass-through endpoints since "
         "StandardPassThroughResponseObject doesn't have standard usage info"
@@ -2665,14 +2559,10 @@ async def test_async_success_handler_prevents_reprocessing_for_pass_through_endp
     # Verify first call set the values
     assert "standard_logging_object" in logging_obj.model_call_details
     assert "async_complete_streaming_response" in logging_obj.model_call_details
-    first_standard_logging_object = logging_obj.model_call_details[
-        "standard_logging_object"
-    ]
+    first_standard_logging_object = logging_obj.model_call_details["standard_logging_object"]
 
     # Second call - should return early due to async_complete_streaming_response guard
-    with patch.object(
-        logging_obj, "get_combined_callback_list", return_value=[]
-    ) as mock_callbacks:
+    with patch.object(logging_obj, "get_combined_callback_list", return_value=[]) as mock_callbacks:
         await logging_obj.async_success_handler(
             result=result,
             start_time=start_time,
@@ -2683,10 +2573,9 @@ async def test_async_success_handler_prevents_reprocessing_for_pass_through_endp
         mock_callbacks.assert_not_called()
 
     # Verify standard_logging_object wasn't modified by second call
-    assert (
-        logging_obj.model_call_details["standard_logging_object"]
-        is first_standard_logging_object
-    ), "standard_logging_object should not be modified on re-processing"
+    assert logging_obj.model_call_details["standard_logging_object"] is first_standard_logging_object, (
+        "standard_logging_object should not be modified on re-processing"
+    )
 
 
 @pytest.mark.asyncio
@@ -2725,9 +2614,7 @@ async def test_async_success_handler_sets_standard_logging_object_for_streaming_
     }
 
     # Create a pass-through response object (simulating unparseable streaming response)
-    result = StandardPassThroughResponseObject(
-        response='data: {"chunk": 1}\ndata: {"chunk": 2}\ndata: [DONE]'
-    )
+    result = StandardPassThroughResponseObject(response='data: {"chunk": 1}\ndata: {"chunk": 2}\ndata: [DONE]')
 
     start_time = datetime.now()
     end_time = datetime.now()
@@ -2747,9 +2634,9 @@ async def test_async_success_handler_sets_standard_logging_object_for_streaming_
         "standard_logging_object should be set for streaming pass-through endpoints "
         "even when the response cannot be parsed into a ModelResponse"
     )
-    assert (
-        logging_obj.model_call_details["standard_logging_object"] is not None
-    ), "standard_logging_object should not be None for streaming pass-through endpoints"
+    assert logging_obj.model_call_details["standard_logging_object"] is not None, (
+        "standard_logging_object should not be None for streaming pass-through endpoints"
+    )
 
 
 def test_get_error_information_error_code_priority():
@@ -2791,30 +2678,22 @@ def test_get_error_information_error_code_priority():
             self.message = message
             super().__init__(message)
 
-    both_exception = BothAttributesException(
-        code="400", status_code=500, message="Bad Request"
-    )
+    both_exception = BothAttributesException(code="400", status_code=500, message="Bad Request")
     result = StandardLoggingPayloadSetup.get_error_information(both_exception)
     assert result["error_code"] == "400"  # Should prefer 'code' over 'status_code'
 
     # Test case 4: Exception with 'code' as empty string - should fall back to 'status_code'
-    empty_code_exception = BothAttributesException(
-        code="", status_code=404, message="Not Found"
-    )
+    empty_code_exception = BothAttributesException(code="", status_code=404, message="Not Found")
     result = StandardLoggingPayloadSetup.get_error_information(empty_code_exception)
     assert result["error_code"] == "404"  # Should fall back to status_code
 
     # Test case 5: Exception with 'code' as "None" string - should fall back to 'status_code'
-    none_string_exception = BothAttributesException(
-        code="None", status_code=503, message="Service Unavailable"
-    )
+    none_string_exception = BothAttributesException(code="None", status_code=503, message="Service Unavailable")
     result = StandardLoggingPayloadSetup.get_error_information(none_string_exception)
     assert result["error_code"] == "503"  # Should fall back to status_code
 
     # Test case 6: Exception with 'code' as None - should fall back to 'status_code'
-    none_code_exception = BothAttributesException(
-        code=None, status_code=401, message="Unauthorized"
-    )
+    none_code_exception = BothAttributesException(code=None, status_code=401, message="Unauthorized")
     result = StandardLoggingPayloadSetup.get_error_information(none_code_exception)
     assert result["error_code"] == "401"  # Should fall back to status_code
 
@@ -2863,9 +2742,7 @@ def test_get_error_information_prefers_message_attribute_over_str():
     )
 
     result = StandardLoggingPayloadSetup.get_error_information(exc)
-    assert (
-        result["error_message"] == msg
-    ), f"expected message from .message attribute, got {result['error_message']!r}"
+    assert result["error_message"] == msg, f"expected message from .message attribute, got {result['error_message']!r}"
     assert result["error_code"] == "401"
     assert result["error_class"] == "ProxyExceptionLike"
 
@@ -2940,8 +2817,7 @@ def test_get_error_information_preserves_explicit_empty_message():
     exc = ProxyExceptionLike(message="", code=500)
     result = StandardLoggingPayloadSetup.get_error_information(exc)
     assert result["error_message"] == "", (
-        "explicit empty .message must survive verbatim; got "
-        f"{result['error_message']!r}"
+        f"explicit empty .message must survive verbatim; got {result['error_message']!r}"
     )
 
 
@@ -3204,9 +3080,7 @@ def test_process_hidden_params_recalculates_cost_after_failure_handler_zero():
         choices=[{"message": {"role": "assistant", "content": "ok"}}],
         usage=Usage(prompt_tokens=9698, completion_tokens=30, total_tokens=9728),
     )
-    logging_obj._process_hidden_params_and_response_cost(
-        result, datetime.now(), datetime.now()
-    )
+    logging_obj._process_hidden_params_and_response_cost(result, datetime.now(), datetime.now())
 
     cost = logging_obj.model_call_details.get("response_cost")
     assert cost is not None and cost > 0
@@ -3230,9 +3104,7 @@ def test_process_hidden_params_preserves_zero_cost_in_hidden_params():
         litellm_call_id="test-hidden-zero-cost",
         function_id="test-hidden-zero-cost",
     )
-    logging_obj.model_call_details["litellm_params"] = {
-        "model": "gemini-2.5-flash-lite"
-    }
+    logging_obj.model_call_details["litellm_params"] = {"model": "gemini-2.5-flash-lite"}
     logging_obj.optional_params = {}
 
     result = ModelResponse(
@@ -3242,9 +3114,7 @@ def test_process_hidden_params_preserves_zero_cost_in_hidden_params():
     )
     result._hidden_params = {"response_cost": 0.0}
 
-    logging_obj._process_hidden_params_and_response_cost(
-        result, datetime.now(), datetime.now()
-    )
+    logging_obj._process_hidden_params_and_response_cost(result, datetime.now(), datetime.now())
 
     assert logging_obj.model_call_details.get("response_cost") == 0.0
     slo = logging_obj.model_call_details.get("standard_logging_object") or {}
@@ -3293,9 +3163,7 @@ def test_process_hidden_params_uses_hidden_params_cost_after_failure_handler_zer
     )
     result._hidden_params = {"response_cost": passthrough_cost}
 
-    logging_obj._process_hidden_params_and_response_cost(
-        result, datetime.now(), datetime.now()
-    )
+    logging_obj._process_hidden_params_and_response_cost(result, datetime.now(), datetime.now())
 
     assert logging_obj.model_call_details.get("response_cost") == passthrough_cost
     slo = logging_obj.model_call_details.get("standard_logging_object") or {}
@@ -3352,9 +3220,7 @@ def test_function_setup_litellm_metadata_populates_metadata():
     assert litellm_metadata.get("user_api_key_hash") == test_api_key_hash
 
     # metadata should be a COPY, not an alias — mutating one must not affect the other
-    assert (
-        metadata is not litellm_metadata
-    ), "litellm_params['metadata'] should be a copy, not the same object"
+    assert metadata is not litellm_metadata, "litellm_params['metadata'] should be a copy, not the same object"
 
 
 def test_function_setup_litellm_metadata_guardrail_writes_visible_after_setup():
@@ -3399,9 +3265,9 @@ def test_function_setup_litellm_metadata_guardrail_writes_visible_after_setup():
     litellm_params = logging_obj.model_call_details.get("litellm_params", {})
     litellm_metadata = litellm_params.get("litellm_metadata")
     assert litellm_metadata is not None
-    assert litellm_metadata.get("standard_logging_guardrail_information") == [
-        guardrail_entry
-    ], "guardrail writes after function_setup must be visible to the logging object"
+    assert litellm_metadata.get("standard_logging_guardrail_information") == [guardrail_entry], (
+        "guardrail writes after function_setup must be visible to the logging object"
+    )
     assert litellm_metadata.get("applied_guardrails") == ["pam-ethical-request"]
 
     merged = StandardLoggingPayloadSetup.merge_litellm_metadata(litellm_params)
@@ -3570,9 +3436,7 @@ def test_failure_handler_skips_sync_callbacks_for_pass_through_requests(logging_
 
 
 @pytest.mark.parametrize("call_type", ["completion", "acompletion"])
-def test_failure_handler_runs_sync_callbacks_for_non_pass_through_requests(
-    logging_obj, call_type
-):
+def test_failure_handler_runs_sync_callbacks_for_non_pass_through_requests(logging_obj, call_type):
     """Ensure sync failure callbacks still fire for normal (non-pass-through) requests."""
     from litellm.integrations.custom_logger import CustomLogger
 
@@ -3733,9 +3597,7 @@ def test_standard_logging_hidden_params_backfills_response_cost_without_mutating
     )
     response._hidden_params = {"response_cost": None, "model_id": "mid-test"}
 
-    payload = logging_obj._build_standard_logging_payload(
-        response, datetime.now(), datetime.now()
-    )
+    payload = logging_obj._build_standard_logging_payload(response, datetime.now(), datetime.now())
 
     assert payload is not None
     assert payload["hidden_params"]["response_cost"] == 0.002
@@ -3789,10 +3651,7 @@ def test_merge_hidden_params_from_response_into_metadata_no_op_when_empty():
         _hidden_params = {}
 
     logging_obj._merge_hidden_params_from_response_into_metadata(_NoHp())
-    assert (
-        "hidden_params"
-        not in logging_obj.model_call_details["litellm_params"]["metadata"]
-    )
+    assert "hidden_params" not in logging_obj.model_call_details["litellm_params"]["metadata"]
 
 
 # ── StandardLoggingPayloadSetup.get_additional_headers ───────────────────────
@@ -3870,6 +3729,82 @@ def test_get_standard_logging_object_payload_includes_litellm_call_id(logging_ob
     assert payload["litellm_call_id"] == call_id
 
 
+# ── Azure Model Router selected-model attribution ────────────────────────────
+
+
+def _model_router_response(selected_model: str, stamp: bool):
+    """A ModelResponse as AzureModelRouterConfig hands it back, with or without the stamp."""
+    from litellm.llms.azure_ai.common_utils import (
+        AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY,
+    )
+    from litellm.types.utils import ModelResponse
+
+    response = ModelResponse(model=selected_model)
+    response._hidden_params = {AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY: selected_model} if stamp else {}
+    return response
+
+
+def test_standard_logging_payload_uses_stamped_model_router_model(logging_obj):
+    """
+    The selected model must win off the stamp, not off "model-router" appearing in the
+    requested model. An operator whose model group is named anything else was invisible
+    to the name check, so their logs and spend rows named the router instead.
+    """
+    import datetime
+
+    from litellm.litellm_core_utils.litellm_logging import (
+        get_standard_logging_object_payload,
+    )
+
+    now = datetime.datetime.now()
+    payload = get_standard_logging_object_payload(
+        kwargs={
+            "model": "azure_ai/smart-pick",
+            "custom_llm_provider": "azure_ai",
+            "messages": [],
+            "litellm_params": {"metadata": {}},
+        },
+        init_response_obj=_model_router_response("azure_ai/grok-4-1-fast-reasoning", stamp=True),
+        start_time=now,
+        end_time=now,
+        logging_obj=logging_obj,
+        status="success",
+    )
+
+    assert payload is not None
+    assert payload["model"] == "azure_ai/grok-4-1-fast-reasoning"
+
+
+def test_standard_logging_payload_keeps_requested_model_without_router_stamp(logging_obj):
+    """
+    Control for the test above: an ordinary azure_ai deployment is unaffected, so the stamp
+    is what redirects attribution rather than the response model winning unconditionally.
+    """
+    import datetime
+
+    from litellm.litellm_core_utils.litellm_logging import (
+        get_standard_logging_object_payload,
+    )
+
+    now = datetime.datetime.now()
+    payload = get_standard_logging_object_payload(
+        kwargs={
+            "model": "azure_ai/smart-pick",
+            "custom_llm_provider": "azure_ai",
+            "messages": [],
+            "litellm_params": {"metadata": {}},
+        },
+        init_response_obj=_model_router_response("azure_ai/grok-4-1-fast-reasoning", stamp=False),
+        start_time=now,
+        end_time=now,
+        logging_obj=logging_obj,
+        status="success",
+    )
+
+    assert payload is not None
+    assert payload["model"] == "azure_ai/smart-pick"
+
+
 def _make_dict_logging_obj():
     """Build a Logging instance configured for a non-streaming dict result."""
     obj = LitellmLogging(
@@ -3905,9 +3840,7 @@ def test_success_handler_computes_cost_for_dict_response():
             "_build_standard_logging_payload",
             return_value={"response_cost": expected_cost},
         ),
-        patch(
-            "litellm.litellm_core_utils.litellm_logging.emit_standard_logging_payload"
-        ),
+        patch("litellm.litellm_core_utils.litellm_logging.emit_standard_logging_payload"),
         patch.object(
             logging_obj,
             "_is_recognized_call_type_for_logging",
@@ -3944,9 +3877,7 @@ def test_success_handler_preserves_precomputed_cost_for_dict_response():
             "_build_standard_logging_payload",
             return_value={"response_cost": precomputed_cost},
         ),
-        patch(
-            "litellm.litellm_core_utils.litellm_logging.emit_standard_logging_payload"
-        ),
+        patch("litellm.litellm_core_utils.litellm_logging.emit_standard_logging_payload"),
         patch.object(
             logging_obj,
             "_is_recognized_call_type_for_logging",
@@ -3985,9 +3916,7 @@ def test_success_handler_unified_helper_runs_for_typed_results():
             "_build_standard_logging_payload",
             return_value={"response_cost": expected_cost},
         ),
-        patch(
-            "litellm.litellm_core_utils.litellm_logging.emit_standard_logging_payload"
-        ),
+        patch("litellm.litellm_core_utils.litellm_logging.emit_standard_logging_payload"),
         patch.object(
             logging_obj,
             "_is_recognized_call_type_for_logging",
@@ -4042,9 +3971,7 @@ class TestFirstApiCallStartTimeSetOnce:
         assert first == obj.model_call_details["api_call_start_time"]
         # Set on the logging object only — user metadata untouched.
         assert user_meta == {}
-        assert (
-            "first_api_call_start_time" not in obj.model_call_details["litellm_params"]
-        )
+        assert "first_api_call_start_time" not in obj.model_call_details["litellm_params"]
 
         time.sleep(0.002)  # ensure a distinct retry timestamp
         obj.pre_call(input="hi", api_key="sk-test")
@@ -4061,18 +3988,16 @@ def test_get_error_information_for_logging_payload_ignores_spoofed_disconnect_wi
     baseline = StandardLoggingPayloadSetup.get_error_information(
         original_exception=ValueError("provider failure"),
     )
-    error_information, error_str = (
-        StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
-            metadata={
-                "error_information": {
-                    "error_code": "499",
-                    "error_message": "Client disconnected the request",
-                    "error_class": "ClientDisconnected",
-                }
-            },
-            original_exception=ValueError("provider failure"),
-            error_str="provider failure",
-        )
+    error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
+        metadata={
+            "error_information": {
+                "error_code": "499",
+                "error_message": "Client disconnected the request",
+                "error_class": "ClientDisconnected",
+            }
+        },
+        original_exception=ValueError("provider failure"),
+        error_str="provider failure",
     )
     assert error_information == baseline
     assert error_str == "provider failure"
@@ -4086,22 +4011,18 @@ def test_get_error_information_for_logging_payload_client_disconnect():
         "error_message": "Client disconnected the request",
         "error_class": "ClientDisconnected",
     }
-    error_information, error_str = (
-        StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
-            metadata={"client_disconnected": True, "error_information": custom_error},
-            original_exception=None,
-            error_str=None,
-        )
+    error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
+        metadata={"client_disconnected": True, "error_information": custom_error},
+        original_exception=None,
+        error_str=None,
     )
     assert error_information == custom_error
     assert error_str == "Client disconnected the request"
 
-    error_information, error_str = (
-        StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
-            metadata={"client_disconnected": True},
-            original_exception=None,
-            error_str="existing error",
-        )
+    error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
+        metadata={"client_disconnected": True},
+        original_exception=None,
+        error_str="existing error",
     )
     assert error_information["error_code"] == "499"
     assert error_str == "existing error"
@@ -4109,12 +4030,10 @@ def test_get_error_information_for_logging_payload_client_disconnect():
     baseline = StandardLoggingPayloadSetup.get_error_information(
         original_exception=None,
     )
-    error_information, error_str = (
-        StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
-            metadata={},
-            original_exception=None,
-            error_str=None,
-        )
+    error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
+        metadata={},
+        original_exception=None,
+        error_str=None,
     )
     assert error_information == baseline
     assert error_str is None
@@ -4149,9 +4068,7 @@ def test_get_error_information_prefers_message_attribute_over_empty_str():
         def __str__(self):
             return ""
 
-    info = StandardLoggingPayloadSetup.get_error_information(
-        original_exception=_SilentExc()
-    )
+    info = StandardLoggingPayloadSetup.get_error_information(original_exception=_SilentExc())
     assert info["error_message"] == "real failure detail"
     assert info["error_code"] == "401"
 
@@ -4182,9 +4099,7 @@ def _responses_api_response_with_text(text="hello world"):
                 type="message",
                 role="assistant",
                 status="completed",
-                content=[
-                    ResponseOutputText(annotations=[], text=text, type="output_text")
-                ],
+                content=[ResponseOutputText(annotations=[], text=text, type="output_text")],
             )
         ],
         usage=ResponseAPIUsage(input_tokens=11, output_tokens=7, total_tokens=18),
@@ -4199,9 +4114,7 @@ def _responses_api_response_with_text(text="hello world"):
         ("ResponseFailedEvent", "response.failed"),
     ],
 )
-def test_handle_anthropic_messages_response_logging_translates_terminal_responses_api_event(
-    event_cls, event_type
-):
+def test_handle_anthropic_messages_response_logging_translates_terminal_responses_api_event(event_cls, event_type):
     """Regression for #28595 / #28943. When anthropic_messages routes to the OpenAI
     Responses backend and stream=True, success_handler receives a terminal Responses
     API event. The handler must translate it to a ModelResponse whose choices carry
@@ -4240,10 +4153,7 @@ def test_handle_anthropic_messages_response_logging_passes_model_response_throug
     """Anthropic-native path already yields a ModelResponse; it must be returned unchanged."""
     logging_obj = _anthropic_messages_logging_obj()
     model_response = ModelResponse()
-    assert (
-        logging_obj._handle_anthropic_messages_response_logging(result=model_response)
-        is model_response
-    )
+    assert logging_obj._handle_anthropic_messages_response_logging(result=model_response) is model_response
 
 
 def test_handle_anthropic_messages_response_logging_degrades_on_unparseable_responses_payload():
@@ -4539,9 +4449,7 @@ def test_non_image_response_has_no_output_image_count(logging_obj):
 
 def test_zero_token_video_usage_preserves_duration_seconds(logging_obj):
     """Video usage bills by duration; the payload must keep duration_seconds even with zero tokens."""
-    payload = _build_payload_for_media_response(
-        logging_obj, {"id": "video-1", "usage": {"duration_seconds": 4.0}}
-    )
+    payload = _build_payload_for_media_response(logging_obj, {"id": "video-1", "usage": {"duration_seconds": 4.0}})
 
     assert payload is not None
     assert payload["metadata"]["usage_object"]["duration_seconds"] == 4.0
diff --git a/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py
index 900372f3e54..1f684ecc3b5 100644
--- a/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py
+++ b/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py
@@ -5,9 +5,7 @@ from unittest.mock import MagicMock, patch
 
 import pytest
 
-sys.path.insert(
-    0, os.path.abspath("../../../../..")
-)  # Adds the parent directory to the system path
+sys.path.insert(0, os.path.abspath("../../../../.."))  # Adds the parent directory to the system path
 from litellm.llms.azure_ai.azure_model_router.transformation import (
     AzureModelRouterConfig,
 )
@@ -120,9 +118,7 @@ def test_azure_ai_grok_stop_parameter_handling():
     # Test supported parameters for Grok models
     for model in ("grok-4-fast", "grok-4.3"):
         grok_params = config.get_supported_openai_params(model)
-        assert (
-            "stop" not in grok_params
-        ), "Grok models should not support stop parameter"
+        assert "stop" not in grok_params, "Grok models should not support stop parameter"
 
     # Test supported parameters for non-Grok models
     gpt_params = config.get_supported_openai_params("gpt-4")
@@ -201,11 +197,84 @@ def test_azure_model_router_response_shows_actual_model():
 
     # Verify that the response contains the actual model used, not the router model
     assert result.model == "azure_ai/gpt-5-nano-2025-08-07", (
-        f"Expected model to be 'azure_ai/gpt-5-nano-2025-08-07' (actual model used), "
-        f"but got '{result.model}'"
+        f"Expected model to be 'azure_ai/gpt-5-nano-2025-08-07' (actual model used), but got '{result.model}'"
     )
 
 
+def test_azure_model_router_stamps_selected_model_on_hidden_params():
+    """
+    The selected model must be stamped on _hidden_params, not left for downstream code to
+    re-derive by looking for "model-router" in the model string. Deployments whose alias
+    does not contain that text are invisible to the string check.
+    """
+    from httpx import Response
+
+    from litellm.llms.azure_ai.common_utils import (
+        AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY,
+        AzureFoundryModelInfo,
+    )
+    from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
+    from litellm.types.utils import ModelResponse
+
+    raw_response_json = {
+        "id": "chatcmpl-test456",
+        "object": "chat.completion",
+        "created": 1234567890,
+        "model": "grok-4-1-fast-reasoning",
+        "choices": [
+            {
+                "index": 0,
+                "message": {"role": "assistant", "content": "pong"},
+                "finish_reason": "stop",
+            }
+        ],
+        "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
+    }
+
+    mock_response = MagicMock(spec=Response)
+    mock_response.json.return_value = raw_response_json
+    mock_response.text = json.dumps(raw_response_json)
+    mock_response.headers = {}
+
+    logging_obj = MagicMock(spec=LiteLLMLoggingObj)
+    logging_obj.post_call = MagicMock()
+    logging_obj.model_call_details = {}
+
+    result = AzureModelRouterConfig().transform_response(
+        model="smart-pick",
+        raw_response=mock_response,
+        model_response=ModelResponse(),
+        logging_obj=logging_obj,
+        request_data={},
+        messages=[{"role": "user", "content": "Reply with just pong"}],
+        optional_params={},
+        litellm_params={"model": "azure_ai/model_router/smart-pick"},
+        encoding=None,
+        api_key="test-key",
+        json_mode=False,
+    )
+
+    assert result._hidden_params[AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY] == result.model
+    assert result._hidden_params[AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY] == "azure_ai/grok-4-1-fast-reasoning"
+    assert AzureFoundryModelInfo.get_model_router_selected_model(result._hidden_params) == (
+        "azure_ai/grok-4-1-fast-reasoning"
+    )
+    assert AzureFoundryModelInfo.is_model_router_call(model="smart-pick", hidden_params=result._hidden_params) is True
+
+
+def test_azure_model_router_stamp_does_not_leak_across_responses():
+    """
+    ModelResponse declares _hidden_params as a class-level dict, so the stamp has to be written
+    as a fresh dict. Mutating in place would bleed the selected model into unrelated responses.
+    """
+    from litellm.llms.azure_ai.common_utils import AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY
+    from litellm.types.utils import ModelResponse
+
+    untouched = ModelResponse()
+
+    assert AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY not in (untouched._hidden_params or {})
+
+
 def test_drop_tool_level_extra_fields_strips_copilot_mcp_server_name():
     """
     Regression test: Azure AI returns 400 when tools contain copilot_mcp_server_name.
@@ -226,14 +295,10 @@ def test_drop_tool_level_extra_fields_strips_copilot_mcp_server_name():
     mock_response.text = error_text
     mock_response.json.return_value = json.loads(error_text)
     mock_response.status_code = 400
-    e = httpx.HTTPStatusError(
-        message="400", request=MagicMock(), response=mock_response
-    )
+    e = httpx.HTTPStatusError(message="400", request=MagicMock(), response=mock_response)
 
     assert config._error_has_tool_level_extra_fields(error_text) is True
-    assert (
-        config.should_retry_llm_api_inside_llm_translation_on_http_error(e, {}) is True
-    )
+    assert config.should_retry_llm_api_inside_llm_translation_on_http_error(e, {}) is True
 
     request_data = {
         "model": "FW-Kimi-K2.6",
@@ -354,9 +419,7 @@ def test_azure_ai_stripping_does_not_mutate_caller_messages():
         {
             "role": "assistant",
             "content": "I can help.",
-            "thinking_blocks": [
-                {"type": "thinking", "thinking": "Reading the file.", "signature": "sig"}
-            ],
+            "thinking_blocks": [{"type": "thinking", "thinking": "Reading the file.", "signature": "sig"}],
             "provider_specific_fields": {"thought_signature": "sig-top"},
             "tool_calls": [
                 {
diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py
index 716fba370df..562046b2f81 100644
--- a/tests/test_litellm/proxy/test_common_request_processing.py
+++ b/tests/test_litellm/proxy/test_common_request_processing.py
@@ -126,16 +126,12 @@ class TestProxyBaseLLMRequestProcessing:
         assert json.loads(result.body) == guardrailed_body
 
     @pytest.mark.asyncio
-    async def test_handle_non_streaming_allm_passthrough_route_forwards_upstream_headers(
-        self, monkeypatch
-    ):
+    async def test_handle_non_streaming_allm_passthrough_route_forwards_upstream_headers(self, monkeypatch):
         """The guardrail JSON path must forward upstream response headers (e.g.
         x-amzn-requestid) alongside the x-litellm-* headers, matching the
         non-guardrail passthrough path, while dropping length headers that no
         longer match the rewritten body."""
-        processing_obj = ProxyBaseLLMRequestProcessing(
-            data={"custom_llm_provider": "bedrock"}
-        )
+        processing_obj = ProxyBaseLLMRequestProcessing(data={"custom_llm_provider": "bedrock"})
         monkeypatch.setattr(
             processing_obj,
             "_has_post_call_guardrails_for_passthrough",
@@ -175,14 +171,10 @@ class TestProxyBaseLLMRequestProcessing:
         assert result.headers["content-length"] == str(len(result.body))
 
     @pytest.mark.asyncio
-    async def test_handle_event_stream_allm_passthrough_route_forwards_upstream_headers(
-        self, monkeypatch
-    ):
+    async def test_handle_event_stream_allm_passthrough_route_forwards_upstream_headers(self, monkeypatch):
         """The guardrail event-stream branch must also forward upstream response
         headers alongside the x-litellm-* headers."""
-        processing_obj = ProxyBaseLLMRequestProcessing(
-            data={"custom_llm_provider": "bedrock"}
-        )
+        processing_obj = ProxyBaseLLMRequestProcessing(data={"custom_llm_provider": "bedrock"})
         monkeypatch.setattr(
             processing_obj,
             "_has_post_call_guardrails_for_passthrough",
@@ -224,15 +216,11 @@ class TestProxyBaseLLMRequestProcessing:
         assert result.headers["x-litellm-call-id"] == "test-call-id"
 
     @pytest.mark.asyncio
-    async def test_handle_non_streaming_allm_passthrough_route_applies_response_headers_hook(
-        self, monkeypatch
-    ):
+    async def test_handle_non_streaming_allm_passthrough_route_applies_response_headers_hook(self, monkeypatch):
         """Guardrailed non-streaming passthrough responses must include headers
         injected by post_call_response_headers_hook, matching the headers a
         non-guardrailed passthrough response would carry."""
-        processing_obj = ProxyBaseLLMRequestProcessing(
-            data={"custom_llm_provider": "bedrock"}
-        )
+        processing_obj = ProxyBaseLLMRequestProcessing(data={"custom_llm_provider": "bedrock"})
         monkeypatch.setattr(
             processing_obj,
             "_has_post_call_guardrails_for_passthrough",
@@ -251,9 +239,7 @@ class TestProxyBaseLLMRequestProcessing:
             return kwargs["response"]
 
         proxy_logging_obj.post_call_success_hook = fake_post_call_success_hook
-        proxy_logging_obj.post_call_response_headers_hook = AsyncMock(
-            return_value={"x-litellm-custom": "from-hook"}
-        )
+        proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value={"x-litellm-custom": "from-hook"})
 
         result = await processing_obj._handle_non_streaming_allm_passthrough_route(
             response=upstream,
@@ -2221,6 +2207,52 @@ class TestOverrideOpenAIResponseModel:
         assert response_obj.model == actual_model_used
         assert response_obj.model != requested_model
 
+    def test_override_model_preserves_model_router_model_for_alias_without_router_in_name(self):
+        """
+        The client sends a model group alias, which carries no model_router/ prefix, so the
+        name check alone only fires when the operator happened to put "model-router" in the
+        alias. With the stamp on the response the actual model survives whatever it is named.
+        """
+        from litellm.llms.azure_ai.common_utils import (
+            AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY,
+        )
+
+        requested_model = "smart-pick"
+        actual_model_used = "azure_ai/grok-4-1-fast-reasoning"
+
+        response_obj = MagicMock()
+        response_obj.model = actual_model_used
+        response_obj._hidden_params = {
+            "additional_headers": {},
+            AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY: actual_model_used,
+        }
+
+        _override_openai_response_model(
+            response_obj=response_obj,
+            requested_model=requested_model,
+            log_context="test_context",
+        )
+        assert response_obj.model == actual_model_used
+
+    def test_override_model_still_restamps_non_router_alias_without_stamp(self):
+        """
+        Control for the test above: absent the stamp, an ordinary deployment keeps being
+        restamped to the requested model, so the stamp is doing the work rather than the
+        preserve branch having gone unconditional.
+        """
+        requested_model = "smart-pick"
+
+        response_obj = MagicMock()
+        response_obj.model = "azure_ai/grok-4-1-fast-reasoning"
+        response_obj._hidden_params = {"additional_headers": {}}
+
+        _override_openai_response_model(
+            response_obj=response_obj,
+            requested_model=requested_model,
+            log_context="test_context",
+        )
+        assert response_obj.model == requested_model
+
     def test_override_model_uses_winning_model_for_fastest_response(self):
         """
         Test that when fastest_response batch completion is used with a
@@ -2793,9 +2825,7 @@ class TestStreamCloseOnDisconnect:
             finally:
                 closed.set()
 
-        response = _UpstreamClosingStreamingResponse(
-            body(), media_type="text/event-stream"
-        )
+        response = _UpstreamClosingStreamingResponse(body(), media_type="text/event-stream")
 
         async def receive():
             await asyncio.Event().wait()
@@ -2826,9 +2856,7 @@ class TestStreamCloseOnDisconnect:
             finally:
                 closed.set()
 
-        response = _UpstreamClosingStreamingResponse(
-            body(), media_type="text/event-stream"
-        )
+        response = _UpstreamClosingStreamingResponse(body(), media_type="text/event-stream")
 
         async def receive():
             await disconnected.wait()
@@ -2899,9 +2927,7 @@ class TestStreamCloseOnDisconnect:
             finally:
                 inner_closed.set()
 
-        response = await create_response(
-            generator=wrapped(), media_type="text/event-stream", headers={}
-        )
+        response = await create_response(generator=wrapped(), media_type="text/event-stream", headers={})
 
         async def receive():
             await asyncio.Event().wait()
@@ -3097,9 +3123,7 @@ class TestStreamCloseOnDisconnect:
 
         with pytest.raises(_ClientDisconnectedBeforeFirstChunk):
             await asyncio.wait_for(
-                _buffer_first_chunk_honoring_disconnect(
-                    AcloseRaises(), request=self._request_that_disconnects()
-                ),
+                _buffer_first_chunk_honoring_disconnect(AcloseRaises(), request=self._request_that_disconnects()),
                 timeout=5,
             )
 
@@ -3115,9 +3139,7 @@ class TestStreamCloseOnDisconnect:
 
         with pytest.raises(_ClientDisconnectedBeforeFirstChunk):
             await asyncio.wait_for(
-                _buffer_first_chunk_honoring_disconnect(
-                    blocking_gen(), request=self._request_that_disconnects()
-                ),
+                _buffer_first_chunk_honoring_disconnect(blocking_gen(), request=self._request_that_disconnects()),
                 timeout=5,
             )
         assert closed.is_set()
@@ -3133,9 +3155,7 @@ class TestHandleLLMApiExceptionRetryAfter:
         user_api_key_dict = UserAPIKeyAuth(api_key="sk-test")
         proxy_logging_obj = MagicMock()
         proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None)
-        proxy_logging_obj.post_call_response_headers_hook = AsyncMock(
-            return_value=callback_headers or {}
-        )
+        proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value=callback_headers or {})
 
         try:
             await processor._handle_llm_api_exception(
@@ -3187,9 +3207,7 @@ class TestHandleLLMApiExceptionRetryAfter:
             enable_pre_call_checks=False,
             cooldown_list=[],
         )
-        proxy_exc = await self._invoke(
-            exc, callback_headers={"retry-after": "", "x-custom": "1"}
-        )
+        proxy_exc = await self._invoke(exc, callback_headers={"retry-after": "", "x-custom": "1"})
         assert proxy_exc.headers["retry-after"] == "43"
         assert proxy_exc.headers["x-custom"] == "1"
 
@@ -3385,9 +3403,7 @@ class TestDisconnectGatherCleanup:
         return Request(scope={"type": "http", "headers": []}, receive=receive)
 
     @pytest.mark.asyncio
-    async def test_base_process_llm_request_raises_499_on_client_disconnect(
-        self, monkeypatch
-    ):
+    async def test_base_process_llm_request_raises_499_on_client_disconnect(self, monkeypatch):
         """With cancel_on_disconnect enabled, base_process_llm_request returns 499."""
         import asyncio
 
@@ -3416,9 +3432,7 @@ class TestDisconnectGatherCleanup:
             "common_processing_pre_call_logic",
             AsyncMock(return_value=({"model": "gemini-2.0-flash"}, mock_logging_obj)),
         )
-        monkeypatch.setattr(
-            processing_obj, "_has_post_call_guardrails", MagicMock(return_value=False)
-        )
+        monkeypatch.setattr(processing_obj, "_has_post_call_guardrails", MagicMock(return_value=False))
 
         with pytest.raises(HTTPException) as exc_info:
             await processing_obj.base_process_llm_request(
@@ -3436,9 +3450,7 @@ class TestDisconnectGatherCleanup:
         assert "disconnected" in exc_info.value.detail.lower()
 
     @pytest.mark.asyncio
-    async def test_base_process_llm_request_reraises_cancelled_error_without_client_disconnect(
-        self, monkeypatch
-    ):
+    async def test_base_process_llm_request_reraises_cancelled_error_without_client_disconnect(self, monkeypatch):
         import asyncio
 
         import litellm.proxy.common_request_processing as cpr
@@ -3463,9 +3475,7 @@ class TestDisconnectGatherCleanup:
             "common_processing_pre_call_logic",
             AsyncMock(return_value=({"model": "gemini-2.0-flash"}, mock_logging_obj)),
         )
-        monkeypatch.setattr(
-            processing_obj, "_has_post_call_guardrails", MagicMock(return_value=False)
-        )
+        monkeypatch.setattr(processing_obj, "_has_post_call_guardrails", MagicMock(return_value=False))
         monkeypatch.setattr(
             cpr,
             "route_request",
@@ -3526,9 +3536,7 @@ class TestDisconnectGatherCleanup:
             "common_processing_pre_call_logic",
             AsyncMock(return_value=({"model": "gemini-2.0-flash"}, mock_logging_obj)),
         )
-        monkeypatch.setattr(
-            processing_obj, "_has_post_call_guardrails", MagicMock(return_value=False)
-        )
+        monkeypatch.setattr(processing_obj, "_has_post_call_guardrails", MagicMock(return_value=False))
 
         with pytest.raises(HTTPException):
             await processing_obj.base_process_llm_request(
@@ -3579,9 +3587,7 @@ class TestDisconnectGatherCleanup:
         assert task.done()
 
     @pytest.mark.asyncio
-    async def test_base_process_llm_request_preserves_llm_error_after_gather(
-        self, monkeypatch
-    ):
+    async def test_base_process_llm_request_preserves_llm_error_after_gather(self, monkeypatch):
         import litellm.proxy.common_request_processing as cpr
         from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
 
@@ -3610,9 +3616,7 @@ class TestDisconnectGatherCleanup:
             "common_processing_pre_call_logic",
             AsyncMock(return_value=({"model": "gemini-2.0-flash"}, mock_logging_obj)),
         )
-        monkeypatch.setattr(
-            processing_obj, "_has_post_call_guardrails", MagicMock(return_value=False)
-        )
+        monkeypatch.setattr(processing_obj, "_has_post_call_guardrails", MagicMock(return_value=False))
 
         mock_request = MagicMock(spec=Request)
         mock_request.is_disconnected = AsyncMock(return_value=False)
@@ -3649,19 +3653,13 @@ class TestStreamingClientDisconnectLogging:
             "litellm_params": {"metadata": {}},
         }
 
-        recorded = await _record_streaming_client_disconnect_if_needed(
-            mock_request, request_data
-        )
+        recorded = await _record_streaming_client_disconnect_if_needed(mock_request, request_data)
 
         assert recorded is True
         assert request_data["metadata"]["client_disconnected"] is True
+        assert request_data["metadata"]["error_information"]["error_code"] == "499"
         assert (
-            request_data["metadata"]["error_information"]["error_code"] == "499"
-        )
-        assert (
-            mock_logging_obj.model_call_details["litellm_params"]["metadata"][
-                "error_information"
-            ]["error_code"]
+            mock_logging_obj.model_call_details["litellm_params"]["metadata"]["error_information"]["error_code"]
             == "499"
         )
 
@@ -3675,9 +3673,7 @@ class TestStreamingClientDisconnectLogging:
         mock_request.is_disconnected = AsyncMock(return_value=False)
         request_data = {"metadata": {}}
 
-        recorded = await _record_streaming_client_disconnect_if_needed(
-            mock_request, request_data
-        )
+        recorded = await _record_streaming_client_disconnect_if_needed(mock_request, request_data)
 
         assert recorded is False
         assert "client_disconnected" not in request_data["metadata"]
@@ -3702,22 +3698,12 @@ class TestStreamingClientDisconnectLogging:
             "litellm_params": {"metadata": {}},
         }
 
-        recorded = await _record_streaming_client_disconnect_if_needed(
-            mock_request, request_data
-        )
+        recorded = await _record_streaming_client_disconnect_if_needed(mock_request, request_data)
 
         assert recorded is True
         assert request_data["metadata"]["client_disconnected"] is True
-        assert (
-            mock_logging_obj.model_call_details["litellm_params"]["metadata"][
-                "client_disconnected"
-            ]
-            is True
-        )
-        assert (
-            mock_logging_obj.model_call_details["metadata"]["client_disconnected"]
-            is True
-        )
+        assert mock_logging_obj.model_call_details["litellm_params"]["metadata"]["client_disconnected"] is True
+        assert mock_logging_obj.model_call_details["metadata"]["client_disconnected"] is True
 
     @pytest.mark.asyncio
     async def test_record_streaming_client_disconnect_handles_none_request_data_metadata(self):
@@ -3733,15 +3719,11 @@ class TestStreamingClientDisconnectLogging:
             "litellm_params": {"metadata": None},
         }
 
-        recorded = await _record_streaming_client_disconnect_if_needed(
-            mock_request, request_data
-        )
+        recorded = await _record_streaming_client_disconnect_if_needed(mock_request, request_data)
 
         assert recorded is True
         assert request_data["metadata"]["client_disconnected"] is True
-        assert (
-            request_data["litellm_params"]["metadata"]["client_disconnected"] is True
-        )
+        assert request_data["litellm_params"]["metadata"]["client_disconnected"] is True
 
     @pytest.mark.asyncio
     async def test_apply_client_disconnect_metadata_none_returns_early(self):
@@ -3752,9 +3734,7 @@ class TestStreamingClientDisconnectLogging:
         _apply_client_disconnect_metadata(None)
 
     @pytest.mark.asyncio
-    async def test_finalize_streaming_generator_cleanup_fires_deferred_logging(
-        self, monkeypatch
-    ):
+    async def test_finalize_streaming_generator_cleanup_fires_deferred_logging(self, monkeypatch):
         from litellm.proxy.common_request_processing import (
             ProxyBaseLLMRequestProcessing,
         )
@@ -3786,9 +3766,7 @@ class TestStreamingClientDisconnectLogging:
         assert request_data["metadata"]["error_information"]["error_code"] == "499"
 
     @pytest.mark.asyncio
-    async def test_finalize_streaming_generator_cleanup_skips_disconnect_after_completion(
-        self, monkeypatch
-    ):
+    async def test_finalize_streaming_generator_cleanup_skips_disconnect_after_completion(self, monkeypatch):
         from litellm.proxy.common_request_processing import (
             ProxyBaseLLMRequestProcessing,
         )
@@ -3818,9 +3796,7 @@ class TestStreamingClientDisconnectLogging:
         assert "client_disconnected" not in request_data["metadata"]
 
     @pytest.mark.asyncio
-    async def test_async_streaming_data_generator_records_499_on_early_aclose(
-        self, monkeypatch
-    ):
+    async def test_async_streaming_data_generator_records_499_on_early_aclose(self, monkeypatch):
         from litellm.proxy.common_request_processing import (
             ProxyBaseLLMRequestProcessing,
         )
@@ -3835,9 +3811,7 @@ class TestStreamingClientDisconnectLogging:
             yield {"choices": [{"delta": {"content": " there"}}]}
 
         mock_proxy_logging = MagicMock(spec=ProxyLogging)
-        mock_proxy_logging.async_post_call_streaming_iterator_hook = (
-            mock_streaming_iterator
-        )
+        mock_proxy_logging.async_post_call_streaming_iterator_hook = mock_streaming_iterator
         ProxyLogging._callback_capabilities_cache.clear()
 
         mock_request = MagicMock(spec=Request)
@@ -3848,9 +3822,7 @@ class TestStreamingClientDisconnectLogging:
             "model": "gemini-2.0-flash",
             "metadata": {},
             "litellm_params": {"metadata": {}},
-            "litellm_logging_obj": MagicMock(
-                model_call_details={"metadata": {}, "litellm_params": {}}
-            ),
+            "litellm_logging_obj": MagicMock(model_call_details={"metadata": {}, "litellm_params": {}}),
         }
 
         gen = ProxyBaseLLMRequestProcessing.async_streaming_data_generator(
@@ -3869,6 +3841,8 @@ class TestStreamingClientDisconnectLogging:
         assert request_data["metadata"]["error_information"]["error_code"] == "499"
 
         ProxyLogging._callback_capabilities_cache.clear()
+
+
 class TestCancelOnDisconnect:
     """
     Coverage for the opt-in `general_settings.cancel_on_disconnect` flag:
@@ -3895,23 +3869,17 @@ class TestCancelOnDisconnect:
         llm_call = asyncio.get_running_loop().create_future()
         disconnect_event = asyncio.Event()
 
-        await _cancel_llm_call_on_client_disconnect(
-            request, llm_call, disconnect_event
-        )
+        await _cancel_llm_call_on_client_disconnect(request, llm_call, disconnect_event)
 
         assert llm_call.cancelled()
         assert disconnect_event.is_set()
 
     async def test_monitor_is_noop_while_client_stays_connected(self):
-        request = self._request(
-            [{"type": "http.request", "body": b"", "more_body": False}]
-        )
+        request = self._request([{"type": "http.request", "body": b"", "more_body": False}])
         llm_call = asyncio.get_running_loop().create_future()
         disconnect_event = asyncio.Event()
 
-        monitor = asyncio.create_task(
-            _cancel_llm_call_on_client_disconnect(request, llm_call, disconnect_event)
-        )
+        monitor = asyncio.create_task(_cancel_llm_call_on_client_disconnect(request, llm_call, disconnect_event))
         await asyncio.sleep(0.01)
 
         assert not monitor.done()
@@ -3930,9 +3898,7 @@ class TestCancelOnDisconnect:
         llm_call = asyncio.get_running_loop().create_future()
         disconnect_event = asyncio.Event()
 
-        await _cancel_llm_call_on_client_disconnect(
-            request, llm_call, disconnect_event
-        )
+        await _cancel_llm_call_on_client_disconnect(request, llm_call, disconnect_event)
 
         assert not llm_call.cancelled()
         assert not disconnect_event.is_set()
@@ -3947,9 +3913,7 @@ class TestCancelOnDisconnect:
         with pytest.raises(asyncio.CancelledError):
             await _await_llm_call_cancelling_on_disconnect(request, llm_call)
 
-    async def _drive_base_process_llm_request(
-        self, monkeypatch, general_settings: dict, llm_call, request: Request
-    ):
+    async def _drive_base_process_llm_request(self, monkeypatch, general_settings: dict, llm_call, request: Request):
         from litellm.proxy._types import UserAPIKeyAuth
 
         logging_obj = MagicMock()
@@ -3958,9 +3922,7 @@ class TestCancelOnDisconnect:
         logging_obj._on_deferred_stream_complete = None
         logging_obj.cost_breakdown = None
 
-        processor = ProxyBaseLLMRequestProcessing(
-            data={"model": "fake-model", "litellm_logging_obj": logging_obj}
-        )
+        processor = ProxyBaseLLMRequestProcessing(data={"model": "fake-model", "litellm_logging_obj": logging_obj})
 
         proxy_logging_obj = MagicMock(spec=ProxyLogging)
         proxy_logging_obj.during_call_hook = AsyncMock(return_value=None)
@@ -3968,9 +3930,7 @@ class TestCancelOnDisconnect:
         proxy_logging_obj.post_call_success_hook = AsyncMock(
             side_effect=lambda data, user_api_key_dict, response: response
         )
-        proxy_logging_obj.post_call_response_headers_hook = AsyncMock(
-            return_value=None
-        )
+        proxy_logging_obj.post_call_response_headers_hook = AsyncMock(return_value=None)
 
         async def fake_route_request(**kwargs):
             return llm_call()
@@ -4049,9 +4009,7 @@ class TestCancelOnDisconnect:
 
         with pytest.raises(ProxyException) as exc_info:
             await processor._handle_llm_api_exception(
-                e=HTTPException(
-                    status_code=499, detail="Client disconnected the request"
-                ),
+                e=HTTPException(status_code=499, detail="Client disconnected the request"),
                 user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
                 proxy_logging_obj=proxy_logging_obj,
             )
@@ -4117,7 +4075,9 @@ class TestAllmPassthroughRoutePostCallGuardrails:
         proxy_logging_obj = ProxyLogging(user_api_key_cache=MagicMock())
         monkeypatch.setattr(proxy_logging_obj, "post_call_success_hook", capture_hook)
 
-        with patch.object(ProxyBaseLLMRequestProcessing, "_has_post_call_guardrails_for_passthrough", return_value=True):
+        with patch.object(
+            ProxyBaseLLMRequestProcessing, "_has_post_call_guardrails_for_passthrough", return_value=True
+        ):
             processing_obj = ProxyBaseLLMRequestProcessing(data={})
             result = await processing_obj._handle_non_streaming_allm_passthrough_route(
                 response=httpx_response,
@@ -4167,7 +4127,9 @@ class TestAllmPassthroughRoutePostCallGuardrails:
         proxy_logging_obj = ProxyLogging(user_api_key_cache=MagicMock())
         monkeypatch.setattr(proxy_logging_obj, "post_call_success_hook", non_dict_hook)
 
-        with patch.object(ProxyBaseLLMRequestProcessing, "_has_post_call_guardrails_for_passthrough", return_value=True):
+        with patch.object(
+            ProxyBaseLLMRequestProcessing, "_has_post_call_guardrails_for_passthrough", return_value=True
+        ):
             processing_obj = ProxyBaseLLMRequestProcessing(data={})
             result = await processing_obj._handle_non_streaming_allm_passthrough_route(
                 response=httpx_response,
@@ -4205,7 +4167,9 @@ class TestAllmPassthroughRoutePostCallGuardrails:
         hook_spy = AsyncMock()
         monkeypatch.setattr(proxy_logging_obj, "post_call_success_hook", hook_spy)
 
-        with patch.object(ProxyBaseLLMRequestProcessing, "_has_post_call_guardrails_for_passthrough", return_value=True):
+        with patch.object(
+            ProxyBaseLLMRequestProcessing, "_has_post_call_guardrails_for_passthrough", return_value=True
+        ):
             processing_obj = ProxyBaseLLMRequestProcessing(data={})
             result = await processing_obj._handle_non_streaming_allm_passthrough_route(
                 response=httpx_response,
@@ -4246,7 +4210,9 @@ class TestAllmPassthroughRoutePostCallGuardrails:
         hook_spy = AsyncMock()
         monkeypatch.setattr(proxy_logging_obj, "post_call_success_hook", hook_spy)
 
-        with patch.object(ProxyBaseLLMRequestProcessing, "_has_post_call_guardrails_for_passthrough", return_value=False):
+        with patch.object(
+            ProxyBaseLLMRequestProcessing, "_has_post_call_guardrails_for_passthrough", return_value=False
+        ):
             processing_obj = ProxyBaseLLMRequestProcessing(data={})
             result = await processing_obj._handle_non_streaming_allm_passthrough_route(
                 response=httpx_response,
@@ -4358,7 +4324,9 @@ class TestEventStreamAllmPassthroughRoute:
             "content-length": "99",
         }
 
-        with patch.object(ProxyBaseLLMRequestProcessing, "_has_post_call_guardrails_for_passthrough", return_value=True):
+        with patch.object(
+            ProxyBaseLLMRequestProcessing, "_has_post_call_guardrails_for_passthrough", return_value=True
+        ):
             processing_obj = ProxyBaseLLMRequestProcessing(data={})
             result = await processing_obj._handle_non_streaming_allm_passthrough_route(
                 response=mock_response,
@@ -4389,9 +4357,7 @@ class TestAllmPassthroughStreamingProviderGate:
     de-anonymized.
     """
 
-    def _build_processing_obj(
-        self, custom_llm_provider: str, endpoint: str = ""
-    ) -> ProxyBaseLLMRequestProcessing:
+    def _build_processing_obj(self, custom_llm_provider: str, endpoint: str = "") -> ProxyBaseLLMRequestProcessing:
         logging_obj = MagicMock()
         logging_obj.litellm_call_id = "call-123"
         logging_obj.cost_breakdown = None
@@ -4442,14 +4408,17 @@ class TestAllmPassthroughStreamingProviderGate:
         processing_obj = self._build_processing_obj("anthropic")
         chunks = [b"chunk-1", b"chunk-2"]
 
-        with patch.object(
-            ProxyBaseLLMRequestProcessing,
-            "_has_post_call_guardrails",
-            return_value=False,
-        ), patch.object(
-            ProxyBaseLLMRequestProcessing,
-            "_has_post_call_guardrails_for_passthrough",
-            return_value=True,
+        with (
+            patch.object(
+                ProxyBaseLLMRequestProcessing,
+                "_has_post_call_guardrails",
+                return_value=False,
+            ),
+            patch.object(
+                ProxyBaseLLMRequestProcessing,
+                "_has_post_call_guardrails_for_passthrough",
+                return_value=True,
+            ),
         ):
             result = await self._run(processing_obj, monkeypatch, chunks)
 
@@ -4458,27 +4427,27 @@ class TestAllmPassthroughStreamingProviderGate:
         assert streamed == chunks
 
     @pytest.mark.asyncio
-    async def test_bedrock_converse_stream_is_buffered_through_handler(
-        self, monkeypatch
-    ):
-        processing_obj = self._build_processing_obj(
-            "bedrock", "model/us.amazon.nova-lite-v1:0/converse-stream"
-        )
+    async def test_bedrock_converse_stream_is_buffered_through_handler(self, monkeypatch):
+        processing_obj = self._build_processing_obj("bedrock", "model/us.amazon.nova-lite-v1:0/converse-stream")
         chunks = [b"raw-1", b"raw-2"]
 
-        with patch.object(
-            ProxyBaseLLMRequestProcessing,
-            "_has_post_call_guardrails",
-            return_value=False,
-        ), patch.object(
-            ProxyBaseLLMRequestProcessing,
-            "_has_post_call_guardrails_for_passthrough",
-            return_value=True,
-        ), patch(
-            "litellm.llms.bedrock.passthrough.guardrail_translation.handler."
-            "BedrockPassthroughGuardrailHandler.de_anonymize_event_stream",
-            new=AsyncMock(return_value=b"modified-body"),
-        ) as mock_handler:
+        with (
+            patch.object(
+                ProxyBaseLLMRequestProcessing,
+                "_has_post_call_guardrails",
+                return_value=False,
+            ),
+            patch.object(
+                ProxyBaseLLMRequestProcessing,
+                "_has_post_call_guardrails_for_passthrough",
+                return_value=True,
+            ),
+            patch(
+                "litellm.llms.bedrock.passthrough.guardrail_translation.handler."
+                "BedrockPassthroughGuardrailHandler.de_anonymize_event_stream",
+                new=AsyncMock(return_value=b"modified-body"),
+            ) as mock_handler,
+        ):
             result = await self._run(processing_obj, monkeypatch, chunks)
 
         assert isinstance(result, Response)
@@ -4494,19 +4463,23 @@ class TestAllmPassthroughStreamingProviderGate:
         )
         chunks = [b"raw-1", b"raw-2"]
 
-        with patch.object(
-            ProxyBaseLLMRequestProcessing,
-            "_has_post_call_guardrails",
-            return_value=False,
-        ), patch.object(
-            ProxyBaseLLMRequestProcessing,
-            "_has_post_call_guardrails_for_passthrough",
-            return_value=True,
-        ), patch(
-            "litellm.llms.bedrock.passthrough.guardrail_translation.handler."
-            "BedrockPassthroughGuardrailHandler.de_anonymize_event_stream",
-            new=AsyncMock(return_value=b"modified-body"),
-        ) as mock_handler:
+        with (
+            patch.object(
+                ProxyBaseLLMRequestProcessing,
+                "_has_post_call_guardrails",
+                return_value=False,
+            ),
+            patch.object(
+                ProxyBaseLLMRequestProcessing,
+                "_has_post_call_guardrails_for_passthrough",
+                return_value=True,
+            ),
+            patch(
+                "litellm.llms.bedrock.passthrough.guardrail_translation.handler."
+                "BedrockPassthroughGuardrailHandler.de_anonymize_event_stream",
+                new=AsyncMock(return_value=b"modified-body"),
+            ) as mock_handler,
+        ):
             result = await self._run(processing_obj, monkeypatch, chunks)
 
         assert isinstance(result, StreamingResponse)
@@ -4528,14 +4501,17 @@ class TestAllmPassthroughStreamingProviderGate:
         )
         chunks = [b"raw-1", b"raw-2"]
 
-        with patch.object(
-            ProxyBaseLLMRequestProcessing,
-            "_has_post_call_guardrails",
-            return_value=False,
-        ), patch.object(
-            ProxyBaseLLMRequestProcessing,
-            "_has_post_call_guardrails_for_passthrough",
-            return_value=False,
+        with (
+            patch.object(
+                ProxyBaseLLMRequestProcessing,
+                "_has_post_call_guardrails",
+                return_value=False,
+            ),
+            patch.object(
+                ProxyBaseLLMRequestProcessing,
+                "_has_post_call_guardrails_for_passthrough",
+                return_value=False,
+            ),
         ):
             result = await self._run(processing_obj, monkeypatch, chunks)
 
@@ -4554,14 +4530,17 @@ class TestAllmPassthroughStreamingProviderGate:
         processing_obj = self._build_processing_obj("anthropic")
         chunks = [b"chunk-1", b"chunk-2"]
 
-        with patch.object(
-            ProxyBaseLLMRequestProcessing,
-            "_has_post_call_guardrails",
-            return_value=False,
-        ), patch.object(
-            ProxyBaseLLMRequestProcessing,
-            "_has_post_call_guardrails_for_passthrough",
-            return_value=False,
+        with (
+            patch.object(
+                ProxyBaseLLMRequestProcessing,
+                "_has_post_call_guardrails",
+                return_value=False,
+            ),
+            patch.object(
+                ProxyBaseLLMRequestProcessing,
+                "_has_post_call_guardrails_for_passthrough",
+                return_value=False,
+            ),
         ):
             result = await self._run(processing_obj, monkeypatch, chunks)
 
@@ -4902,7 +4881,6 @@ class TestResponseCostHeaderForTypedDictResponses:
 
 
 class TestPreCallWithFallbacksOnLocalRateLimit:
-
     @pytest.mark.asyncio
     async def test_fallback_triggered_on_local_rate_limit(self):
         from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
@@ -5054,9 +5032,7 @@ class TestPreCallWithFallbacksOnLocalRateLimit:
         mock_router.fallbacks = [{"gpt-4": ["gpt-3.5-turbo"]}]
 
         user_api_key_dict = MagicMock()
-        user_api_key_dict.router_settings = {
-            "fallbacks": [{"gpt-4": ["claude-3-haiku"]}]
-        }
+        user_api_key_dict.router_settings = {"fallbacks": [{"gpt-4": ["claude-3-haiku"]}]}
 
         with patch.object(
             processor,
@@ -5087,9 +5063,7 @@ class TestPreCallWithFallbacksOnLocalRateLimit:
         from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
         from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
 
-        processor = ProxyBaseLLMRequestProcessing(
-            data={"model": "gpt-4", "disable_fallbacks": True}
-        )
+        processor = ProxyBaseLLMRequestProcessing(data={"model": "gpt-4", "disable_fallbacks": True})
 
         async def mock_pre_call_logic(**kwargs):
             raise ProxyRateLimitError(
@@ -5215,9 +5189,7 @@ class TestPreCallWithFallbacksOnLocalRateLimit:
 
         # Real per-key per-model TPM limiter + a key carrying the customer's
         # `model_tpm_limit` metadata (only the primary is capped).
-        limiter = _PROXY_MaxParallelRequestsHandler(
-            internal_usage_cache=InternalUsageCache(DualCache())
-        )
+        limiter = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(DualCache()))
         user_api_key_dict = UserAPIKeyAuth(
             api_key="sk-lit3890",
             metadata={"model_tpm_limit": {primary_model: 100}},
@@ -5225,10 +5197,7 @@ class TestPreCallWithFallbacksOnLocalRateLimit:
 
         # Pre-seed the primary's per-model token counter at the cap so the very
         # next request trips it. The counter key uses the *hashed* api_key.
-        counter_key = (
-            f"{user_api_key_dict.api_key}::{primary_model}"
-            f"::{precise_minute}::request_count"
-        )
+        counter_key = f"{user_api_key_dict.api_key}::{primary_model}::{precise_minute}::request_count"
         await limiter.internal_usage_cache.async_set_cache(
             key=counter_key,
             value={"current_requests": 0, "current_tpm": 100, "current_rpm": 0},
@@ -5259,9 +5228,7 @@ class TestPreCallWithFallbacksOnLocalRateLimit:
         mock_router = MagicMock()
         mock_router.fallbacks = [{primary_model: [fallback_model]}]
 
-        with patch(
-            "litellm.proxy.hooks.parallel_request_limiter.datetime", _FrozenClock
-        ):
+        with patch("litellm.proxy.hooks.parallel_request_limiter.datetime", _FrozenClock):
             with patch.object(
                 processor,
                 "common_processing_pre_call_logic",
@@ -5291,9 +5258,7 @@ class TestPreCallWithFallbacksOnLocalRateLimit:
 
         # Sanity-check the premise: the limiter genuinely raises a
         # ProxyRateLimitError for the capped primary under the frozen clock.
-        with patch(
-            "litellm.proxy.hooks.parallel_request_limiter.datetime", _FrozenClock
-        ):
+        with patch("litellm.proxy.hooks.parallel_request_limiter.datetime", _FrozenClock):
             with pytest.raises(ProxyRateLimitError):
                 await limiter.async_pre_call_hook(
                     user_api_key_dict=user_api_key_dict,
@@ -5654,16 +5619,12 @@ class TestStreamingClientDisconnectBilling:
                         prompt_tokens=1000,
                         completion_tokens=10,
                         total_tokens=1010,
-                        prompt_tokens_details=PromptTokensDetailsWrapper(
-                            cached_tokens=500
-                        ),
+                        prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=500),
                     ),
                 )
             )
 
-        event = await self._bill_and_collect_success_event(
-            append_openai_style_cached_usage_chunk
-        )
+        event = await self._bill_and_collect_success_event(append_openai_style_cached_usage_chunk)
 
         usage = event["response_obj"].usage
         assert getattr(usage, "cache_read_input_tokens", None) == 500
@@ -6433,9 +6394,7 @@ class TestInjectCostIntoUsageDict:
         logging_obj.model_call_details["custom_llm_provider"] = "anthropic"
         assert logging_obj.cost_breakdown is None
 
-        model_response = ModelResponse(
-            usage=Usage(prompt_tokens=3216, completion_tokens=8, total_tokens=3224)
-        )
+        model_response = ModelResponse(usage=Usage(prompt_tokens=3216, completion_tokens=8, total_tokens=3224))
         cost = ProxyBaseLLMRequestProcessing._logging_obj_cost_or_none(model_response, logging_obj)
 
         assert cost is not None and cost > 0
@@ -6464,9 +6423,7 @@ class TestInjectCostIntoUsageDict:
         )
         existing = logging_obj.cost_breakdown
 
-        model_response = ModelResponse(
-            usage=Usage(prompt_tokens=3216, completion_tokens=8, total_tokens=3224)
-        )
+        model_response = ModelResponse(usage=Usage(prompt_tokens=3216, completion_tokens=8, total_tokens=3224))
         ProxyBaseLLMRequestProcessing._logging_obj_cost_or_none(model_response, logging_obj)
 
         assert logging_obj.cost_breakdown is existing
@@ -6761,9 +6718,7 @@ def test_ttft_keepalive_interval_only_arms_for_a_streaming_request(request_data,
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize("stream_requested, expect_ping", [(True, True), (False, False)])
-async def test_base_process_llm_request_pings_while_the_upstream_call_is_still_running(
-    stream_requested, expect_ping
-):
+async def test_base_process_llm_request_pings_while_the_upstream_call_is_still_running(stream_requested, expect_ping):
     """The wiring, not the helper: every route funnels through this method, and the
     whole time-to-first-token is spent inside the call it wraps."""
 
@@ -6909,9 +6864,7 @@ async def test_a_late_failure_is_reported_to_the_failure_hook():
     async def record(exc):
         audited.append(exc)
 
-    response = await open_sse_before_first_byte(
-        slow_failure(), ping_interval_seconds=0.05, on_late_failure=record
-    )
+    response = await open_sse_before_first_byte(slow_failure(), ping_interval_seconds=0.05, on_late_failure=record)
     collected = await _drain(response)
 
     assert [type(exc).__name__ for exc in audited] == ["HTTPException"]
@@ -6928,9 +6881,7 @@ async def test_a_failing_audit_hook_never_costs_the_client_its_error_frame():
     async def broken_hook(exc):
         raise RuntimeError("the audit backend is down")
 
-    response = await open_sse_before_first_byte(
-        slow_failure(), ping_interval_seconds=0.05, on_late_failure=broken_hook
-    )
+    response = await open_sse_before_first_byte(slow_failure(), ping_interval_seconds=0.05, on_late_failure=broken_hook)
     collected = await _drain(response)
 
     error_frame = json.loads(collected[-2].decode().removeprefix("data: ").strip())
@@ -6984,9 +6935,7 @@ async def test_base_process_llm_request_audits_a_failure_that_lands_after_its_ke
     [(0, False), (None, True)],
     ids=["operator-hard-disabled-this-deployment", "deployment-says-nothing"],
 )
-async def test_base_process_llm_request_honours_a_deployment_hard_disable(
-    deployment_keepalive, expect_ping
-):
+async def test_base_process_llm_request_honours_a_deployment_hard_disable(deployment_keepalive, expect_ping):
     """`keepalive_seconds: 0` is documented as a disable a request cannot lift. The
     funnel has to hand its router to the gate for that to hold before the upstream
     has answered, since no deployment has served the request yet."""
@@ -7032,9 +6981,7 @@ async def test_a_hook_returning_a_replacement_decides_what_the_client_sees():
     async def sanitize(exc):
         return HTTPException(status_code=502, detail="upstream unavailable")
 
-    response = await open_sse_before_first_byte(
-        slow_failure(), ping_interval_seconds=0.05, on_late_failure=sanitize
-    )
+    response = await open_sse_before_first_byte(slow_failure(), ping_interval_seconds=0.05, on_late_failure=sanitize)
     collected = await _drain(response)
 
     error_frame = json.loads(collected[-2].decode().removeprefix("data: ").strip())
@@ -7073,9 +7020,7 @@ async def test_a_hook_that_returns_nothing_leaves_the_real_error_intact():
     async def audit_only(exc):
         return None
 
-    response = await open_sse_before_first_byte(
-        slow_failure(), ping_interval_seconds=0.05, on_late_failure=audit_only
-    )
+    response = await open_sse_before_first_byte(slow_failure(), ping_interval_seconds=0.05, on_late_failure=audit_only)
     collected = await _drain(response)
 
     error_frame = json.loads(collected[-2].decode().removeprefix("data: ").strip())
@@ -7092,9 +7037,7 @@ async def test_a_broken_hook_does_not_replace_the_real_error_with_its_own_bug():
     async def broken_hook(exc):
         raise RuntimeError("the audit backend is down")
 
-    response = await open_sse_before_first_byte(
-        slow_failure(), ping_interval_seconds=0.05, on_late_failure=broken_hook
-    )
+    response = await open_sse_before_first_byte(slow_failure(), ping_interval_seconds=0.05, on_late_failure=broken_hook)
     collected = await _drain(response)
 
     error_frame = json.loads(collected[-2].decode().removeprefix("data: ").strip())

From 90f9a8bfda7aff99e1c969f573931cc86eee2103 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 11:05:38 -0700
Subject: [PATCH 035/105] test(e2e): retry timeout-shaped Mantle
 test_connection probes

The endpoint answers a probe that exceeds HEALTH_CHECK_TIMEOUT_SECONDS with
HTTP 200 and an in-body "Timeout exceeded", which the harness's status-code
rerun policy cannot see. The suite's parallel Bedrock load can push a Mantle
probe past that cap transiently, so only that exact error is retried, three
bounded attempts with visible prints; any other error verdict still fails
immediately.
---
 .../test_model_test_connection_e2e.py         | 51 ++++++++++++++-----
 1 file changed, 38 insertions(+), 13 deletions(-)

diff --git a/tests/e2e/management/test_model_test_connection_e2e.py b/tests/e2e/management/test_model_test_connection_e2e.py
index a1f714df4c8..25b0b4f24e6 100644
--- a/tests/e2e/management/test_model_test_connection_e2e.py
+++ b/tests/e2e/management/test_model_test_connection_e2e.py
@@ -8,35 +8,60 @@ verdict from the live provider rather than just a 200 envelope. The region is a
 literal because the endpoint rejects request-supplied os.environ/ references;
 credentials fall through to the proxy's own environment (bearer token locally,
 pod identity in CI).
+
+The endpoint caps every probe at HEALTH_CHECK_TIMEOUT_SECONDS and answers a
+timed-out probe with HTTP 200 and an in-body "Timeout exceeded", which the
+harness's status-code retry policy cannot see. A Mantle probe can hit that cap
+transiently while the rest of the suite saturates the same AWS account, so only
+that exact error is retried here; any other error verdict fails immediately.
 """
 
 from __future__ import annotations
 
+import time
+
 import pytest
 
 from e2e_http import unwrap
 from management_client import ManagementClient
-from models import ConnectionTestBody, LiteLLMParamsBody
+from models import ConnectionTestBody, ConnectionTestResponse, LiteLLMParamsBody
 
 pytestmark = pytest.mark.e2e
 
 MANTLE_RESPONSES_BACKEND = "bedrock_mantle/openai.gpt-5.6-luna"
 MANTLE_REGION = "us-east-1"
+PROBE_TIMEOUT_ERROR = "Timeout exceeded"
+PROBE_ATTEMPTS = 3
+PROBE_RETRY_SLEEP_SECONDS = 30
+
+
+def _probe_mantle(client: ManagementClient) -> ConnectionTestResponse:
+    return unwrap(
+        client.connection_test(
+            ConnectionTestBody(
+                litellm_params=LiteLLMParamsBody(
+                    model=MANTLE_RESPONSES_BACKEND, aws_region_name=MANTLE_REGION
+                ),
+                mode="responses",
+            )
+        )
+    )
 
 
 class TestModelTestConnection:
     @pytest.mark.covers("mgmt.model.test_connection.happy_path")
     def test_bedrock_mantle_responses_connection_succeeds(self, client: ManagementClient) -> None:
-        response = unwrap(
-            client.connection_test(
-                ConnectionTestBody(
-                    litellm_params=LiteLLMParamsBody(
-                        model=MANTLE_RESPONSES_BACKEND, aws_region_name=MANTLE_REGION
-                    ),
-                    mode="responses",
+        for attempt in range(1, PROBE_ATTEMPTS + 1):
+            response = _probe_mantle(client)
+            if response.status == "success":
+                return
+            error = response.result.error if response.result else None
+            assert error == PROBE_TIMEOUT_ERROR, f"test_connection reported an error: {error}"
+            if attempt < PROBE_ATTEMPTS:
+                print(
+                    f"test_connection probe timed out; retry {attempt}/{PROBE_ATTEMPTS - 1}"
+                    f" in {PROBE_RETRY_SLEEP_SECONDS}s",
+                    flush=True,
                 )
-            )
-        )
-
-        error = response.result.error if response.result else None
-        assert response.status == "success", f"test_connection reported an error: {error}"
+                time.sleep(PROBE_RETRY_SLEEP_SECONDS)
+        pytest.fail(f"test_connection timed out on all {PROBE_ATTEMPTS} attempts")

From 559588a4732e94971781983eaf433cb11da9fbb9 Mon Sep 17 00:00:00 2001
From: milan <milan@berri.ai>
Date: Tue, 25 Aug 2026 18:23:09 +0000
Subject: [PATCH 036/105] fix(azure_ai): remove new LIT002 violations to
 satisfy type-discipline budget

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/llms/azure_ai/azure_model_router/transformation.py | 2 +-
 litellm/llms/azure_ai/common_utils.py                      | 6 +++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/litellm/llms/azure_ai/azure_model_router/transformation.py b/litellm/llms/azure_ai/azure_model_router/transformation.py
index d33564c0f9a..61cbc213b11 100644
--- a/litellm/llms/azure_ai/azure_model_router/transformation.py
+++ b/litellm/llms/azure_ai/azure_model_router/transformation.py
@@ -99,7 +99,7 @@ class AzureModelRouterConfig(AzureAIStudioConfig):
         if selected_model:
             # Rebuilt rather than mutated in place: ModelResponseBase declares _hidden_params as a
             # class-level dict, so an in-place write can bleed into unrelated responses.
-            transformed_response._hidden_params = {  # pyright: ignore[reportPrivateUsage]  # ModelResponse exposes no public hidden-params setter
+            transformed_response._hidden_params = {  # pyright: ignore[reportPrivateUsage]  # ModelResponse exposes no public hidden-params setter  # mutable-ok: ModelResponse requires _hidden_params to be a plain dict
                 **get_hidden_params_dict(transformed_response),
                 AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY: selected_model,
             }
diff --git a/litellm/llms/azure_ai/common_utils.py b/litellm/llms/azure_ai/common_utils.py
index e8431f29f07..55a51176666 100644
--- a/litellm/llms/azure_ai/common_utils.py
+++ b/litellm/llms/azure_ai/common_utils.py
@@ -112,7 +112,11 @@ class AzureFoundryModelInfo(BaseLLMModelInfo):
         """
         if AzureFoundryModelInfo.get_model_router_selected_model(hidden_params) is not None:
             return True
-        deployment_model: Final = (hidden_params or {}).get("litellm_model_name") or (hidden_params or {}).get("model")
+        deployment_model: Final = (
+            hidden_params.get("litellm_model_name") or hidden_params.get("model")
+            if hidden_params is not None
+            else None
+        )
         return any(
             isinstance(candidate, str) and AzureFoundryModelInfo.get_azure_ai_route(candidate) == "model_router"
             for candidate in (deployment_model, model)

From 77f22be2075858dae62de1f44ecec85e6131ab86 Mon Sep 17 00:00:00 2001
From: milan <milan@berri.ai>
Date: Tue, 25 Aug 2026 18:27:19 +0000
Subject: [PATCH 037/105] style: apply ruff format to common_utils

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/llms/azure_ai/common_utils.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/litellm/llms/azure_ai/common_utils.py b/litellm/llms/azure_ai/common_utils.py
index 55a51176666..26a90157455 100644
--- a/litellm/llms/azure_ai/common_utils.py
+++ b/litellm/llms/azure_ai/common_utils.py
@@ -113,9 +113,7 @@ class AzureFoundryModelInfo(BaseLLMModelInfo):
         if AzureFoundryModelInfo.get_model_router_selected_model(hidden_params) is not None:
             return True
         deployment_model: Final = (
-            hidden_params.get("litellm_model_name") or hidden_params.get("model")
-            if hidden_params is not None
-            else None
+            hidden_params.get("litellm_model_name") or hidden_params.get("model") if hidden_params is not None else None
         )
         return any(
             isinstance(candidate, str) and AzureFoundryModelInfo.get_azure_ai_route(candidate) == "model_router"

From 6cd1fcdcf05734cff34d0dfed7e098da73a0a913 Mon Sep 17 00:00:00 2001
From: milan <milan@berri.ai>
Date: Tue, 25 Aug 2026 18:37:26 +0000
Subject: [PATCH 038/105] test: drop unneeded proxy_server patches and ratchet
 lint budgets

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 ruff-strict-budget.json                               |  2 +-
 .../proxy/spend_tracking/test_spend_tracking_utils.py | 11 ++---------
 type-discipline-budget.json                           |  2 +-
 3 files changed, 4 insertions(+), 11 deletions(-)

diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json
index 03318718fb5..9815ccfa23e 100644
--- a/ruff-strict-budget.json
+++ b/ruff-strict-budget.json
@@ -168,7 +168,7 @@
     "limit": 3
   },
   "RET504": {
-    "limit": 176
+    "limit": 175
   },
   "RUF012": {
     "limit": 240
diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
index dab9415de77..843e1d1296f 100644
--- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
+++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
@@ -3,13 +3,10 @@ import datetime
 import json
 from datetime import timezone
 from typing import Any, Final, cast
-
-from typing_extensions import ReadOnly, TypedDict
+from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
-
-
-from unittest.mock import AsyncMock, MagicMock, patch
+from typing_extensions import ReadOnly, TypedDict
 
 import litellm
 from litellm.constants import (
@@ -3526,8 +3523,6 @@ def _model_router_spend_log_kwargs(slp_model: str | None) -> _ModelRouterSpendLo
     }
 
 
-@patch("litellm.proxy.proxy_server.master_key", None)
-@patch("litellm.proxy.proxy_server.general_settings", {})
 def test_get_logging_payload_uses_standard_logging_payload_model():
     payload = get_logging_payload(
         kwargs=_model_router_spend_log_kwargs(slp_model="azure_ai/gpt-5-mini"),
@@ -3538,8 +3533,6 @@ def test_get_logging_payload_uses_standard_logging_payload_model():
     assert payload["model"] == "azure_ai/gpt-5-mini"
 
 
-@patch("litellm.proxy.proxy_server.master_key", None)
-@patch("litellm.proxy.proxy_server.general_settings", {})
 def test_get_logging_payload_falls_back_to_kwargs_model_when_slp_model_missing():
     payload = get_logging_payload(
         kwargs=_model_router_spend_log_kwargs(slp_model=None),
diff --git a/type-discipline-budget.json b/type-discipline-budget.json
index 05098546325..542762f1cea 100644
--- a/type-discipline-budget.json
+++ b/type-discipline-budget.json
@@ -30,7 +30,7 @@
     "limit": 16673
   },
   "LIT011": {
-    "limit": 5588
+    "limit": 5587
   },
   "LIT012": {
     "limit": 4510

From 62ec3b61167d780b93839a7b03be30f2305f7e86 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 11:44:06 -0700
Subject: [PATCH 039/105] fix(together_ai): route chat completions through a
 dedicated TogetherAIChatConfig

---
 litellm/__init__.py                           |   3 +
 litellm/_lazy_imports_registry.py             |   5 +
 .../get_supported_openai_params.py            |   2 +-
 litellm/llms/together_ai/chat.py              |  58 -----
 litellm/llms/together_ai/chat/__init__.py     |   3 +
 .../llms/together_ai/chat/transformation.py   |  49 ++++
 litellm/main.py                               |  62 ++++-
 litellm/utils.py                              |   4 +-
 .../test_together_ai_chat_transformation.py   | 232 ++++++++++++++++++
 9 files changed, 348 insertions(+), 70 deletions(-)
 delete mode 100644 litellm/llms/together_ai/chat.py
 create mode 100644 litellm/llms/together_ai/chat/__init__.py
 create mode 100644 litellm/llms/together_ai/chat/transformation.py
 create mode 100644 tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py

diff --git a/litellm/__init__.py b/litellm/__init__.py
index ee2c551481c..39556d1f04d 100644
--- a/litellm/__init__.py
+++ b/litellm/__init__.py
@@ -1628,6 +1628,9 @@ if TYPE_CHECKING:
         AmazonMantleMessagesConfig as AmazonMantleMessagesConfig,
     )
     from .llms.together_ai.chat import TogetherAIConfig as TogetherAIConfig
+    from .llms.together_ai.chat.transformation import (
+        TogetherAIChatConfig as TogetherAIChatConfig,
+    )
     from .llms.nlp_cloud.chat.handler import NLPCloudConfig as NLPCloudConfig
     from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
         VertexGeminiConfig as VertexGeminiConfig,
diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py
index c34c9eefe85..1c833256598 100644
--- a/litellm/_lazy_imports_registry.py
+++ b/litellm/_lazy_imports_registry.py
@@ -177,6 +177,7 @@ LLM_CONFIG_NAMES: Final = (
     "AmazonAnthropicClaudeMessagesConfig",
     "AmazonMantleMessagesConfig",
     "TogetherAIConfig",
+    "TogetherAIChatConfig",
     "NLPCloudConfig",
     "VertexGeminiConfig",
     "GoogleAIStudioGeminiConfig",
@@ -741,6 +742,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
         "AmazonMantleMessagesConfig",
     ),
     "TogetherAIConfig": (".llms.together_ai.chat", "TogetherAIConfig"),
+    "TogetherAIChatConfig": (
+        ".llms.together_ai.chat.transformation",
+        "TogetherAIChatConfig",
+    ),
     "NLPCloudConfig": (".llms.nlp_cloud.chat.handler", "NLPCloudConfig"),
     "VertexGeminiConfig": (
         ".llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini",
diff --git a/litellm/litellm_core_utils/get_supported_openai_params.py b/litellm/litellm_core_utils/get_supported_openai_params.py
index 72f36661f4c..7a16ffe4d85 100644
--- a/litellm/litellm_core_utils/get_supported_openai_params.py
+++ b/litellm/litellm_core_utils/get_supported_openai_params.py
@@ -172,7 +172,7 @@ def get_supported_openai_params(
         if request_type == "embeddings":
             return litellm.JinaAIEmbeddingConfig().get_supported_openai_params(model=model)
     elif custom_llm_provider == "together_ai":
-        return litellm.TogetherAIConfig().get_supported_openai_params(model=model)
+        return litellm.TogetherAIChatConfig().get_supported_openai_params(model=model)
     elif custom_llm_provider == "databricks":
         if request_type == "chat_completion":
             return litellm.DatabricksConfig().get_supported_openai_params(model=model)
diff --git a/litellm/llms/together_ai/chat.py b/litellm/llms/together_ai/chat.py
deleted file mode 100644
index 58d47e45faa..00000000000
--- a/litellm/llms/together_ai/chat.py
+++ /dev/null
@@ -1,58 +0,0 @@
-"""
-Support for OpenAI's `/v1/chat/completions` endpoint.
-
-Calls done in OpenAI/openai.py as TogetherAI is openai-compatible.
-
-Docs: https://docs.together.ai/reference/completions-1
-"""
-
-from typing import Final
-
-from litellm._logging import verbose_logger
-from litellm.utils import supports_function_calling
-
-from ..openai.chat.gpt_transformation import OpenAIGPTConfig
-
-
-class TogetherAIConfig(OpenAIGPTConfig):
-    def get_supported_openai_params(self, model: str) -> list:
-        """
-        Only some together models support response_format / tool calling
-
-        Docs: https://docs.together.ai/docs/json-mode
-        """
-        # Use supports_function_calling() — which reads _get_model_info_helper
-        # directly — instead of get_model_info(). get_model_info() calls
-        # get_supported_openai_params() as its first step, which routes back
-        # into this method for together_ai models, creating a recursion that
-        # only terminates when Python's recursion limit or the "not mapped"
-        # exception in _get_model_info_helper is hit (~332 deep calls).
-        supports_fc: bool | None = None
-        try:
-            supports_fc = supports_function_calling(model, custom_llm_provider="together_ai")
-        except Exception as e:
-            verbose_logger.debug("Error getting supported openai params: %s", e)
-
-        optional_params: Final = super().get_supported_openai_params(model)
-        if supports_fc is not True:
-            verbose_logger.debug(
-                "Only some together models support function calling/response_format. Docs - https://docs.together.ai/docs/function-calling"
-            )
-            optional_params.remove("tools")
-            optional_params.remove("tool_choice")
-            optional_params.remove("function_call")
-            optional_params.remove("response_format")
-        return optional_params
-
-    def map_openai_params(
-        self,
-        non_default_params: dict,
-        optional_params: dict,
-        model: str,
-        drop_params: bool,
-    ) -> dict:
-        mapped_openai_params: Final = super().map_openai_params(non_default_params, optional_params, model, drop_params)
-
-        if "response_format" in mapped_openai_params and mapped_openai_params["response_format"] == {"type": "text"}:
-            mapped_openai_params.pop("response_format")
-        return mapped_openai_params
diff --git a/litellm/llms/together_ai/chat/__init__.py b/litellm/llms/together_ai/chat/__init__.py
new file mode 100644
index 00000000000..f260d9126d7
--- /dev/null
+++ b/litellm/llms/together_ai/chat/__init__.py
@@ -0,0 +1,3 @@
+from .transformation import TogetherAIChatConfig as TogetherAIChatConfig
+
+TogetherAIConfig = TogetherAIChatConfig
diff --git a/litellm/llms/together_ai/chat/transformation.py b/litellm/llms/together_ai/chat/transformation.py
new file mode 100644
index 00000000000..eb0954bceef
--- /dev/null
+++ b/litellm/llms/together_ai/chat/transformation.py
@@ -0,0 +1,49 @@
+"""
+Translates from OpenAI's `/v1/chat/completions` to Together AI's `/v1/chat/completions`.
+
+Docs: https://docs.together.ai/docs/chat-overview
+"""
+
+from types import MappingProxyType
+from typing import Final
+
+from litellm._logging import verbose_logger
+from litellm.utils import supports_function_calling
+
+from ...openai.chat.gpt_transformation import OpenAIGPTConfig
+
+FUNCTION_CALLING_ONLY_PARAMS: Final = ("tools", "tool_choice", "function_call", "response_format")
+PLAIN_TEXT_RESPONSE_FORMAT: Final = MappingProxyType({"type": "text"})
+
+
+class TogetherAIChatConfig(OpenAIGPTConfig):
+    def get_supported_openai_params(self, model: str) -> list:
+        supports_fc: bool | None = None
+        try:
+            supports_fc = supports_function_calling(model, custom_llm_provider="together_ai")
+        except Exception as e:
+            verbose_logger.debug("Error getting supported openai params: %s", e)
+
+        supported_params: Final = super().get_supported_openai_params(model)
+        if supports_fc is True:
+            return supported_params
+        verbose_logger.debug(
+            "Only some together models support function calling/response_format. Docs - https://docs.together.ai/docs/function-calling"
+        )
+        for param in FUNCTION_CALLING_ONLY_PARAMS:
+            if param in supported_params:
+                supported_params.remove(param)
+        return supported_params
+
+    def map_openai_params(
+        self,
+        non_default_params: dict,
+        optional_params: dict,
+        model: str,
+        drop_params: bool,
+    ) -> dict:
+        mapped_openai_params: Final = super().map_openai_params(non_default_params, optional_params, model, drop_params)
+
+        if mapped_openai_params.get("response_format") == PLAIN_TEXT_RESPONSE_FORMAT:
+            mapped_openai_params.pop("response_format")
+        return mapped_openai_params
diff --git a/litellm/main.py b/litellm/main.py
index d3967473f99..8ddcef2b9fa 100644
--- a/litellm/main.py
+++ b/litellm/main.py
@@ -24,6 +24,7 @@ from concurrent import futures
 from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
 from copy import deepcopy
 from functools import partial
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args
 
 from litellm._logging import _redact_string
@@ -1811,6 +1812,56 @@ def _complete_fireworks_ai(
     return response
 
 
+def _complete_together_ai(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
+    acompletion: Final = ctx.acompletion
+    api_base: Final = ctx.api_base
+    api_key: Final = ctx.api_key
+    client: Final = _dispatch_client_http(ctx)
+    custom_llm_provider: Final = ctx.custom_llm_provider
+    headers: Final = ctx.headers
+    litellm_params: Final = ctx.litellm_params
+    logging: Final = ctx.logging
+    messages: Final = ctx.messages
+    model: Final = ctx.model
+    model_response: Final = ctx.model_response
+    optional_params: Final = ctx.optional_params
+    provider_config: Final = ctx.provider_config
+    shared_session: Final = ctx.shared_session
+    stream: Final = ctx.stream
+    timeout: Final = ctx.timeout
+
+    try:
+        response: Final = base_llm_http_handler.completion(
+            model=model,
+            messages=messages,
+            headers=headers,
+            model_response=model_response,
+            api_key=api_key,
+            api_base=api_base,
+            acompletion=acompletion,
+            logging_obj=logging,
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            shared_session=shared_session,
+            timeout=timeout,
+            client=client,
+            custom_llm_provider=custom_llm_provider,
+            encoding=_get_encoding(),
+            stream=stream,
+            provider_config=provider_config,
+        )
+    except Exception as e:
+        logging.post_call(
+            input=messages,
+            api_key=api_key,
+            original_response=str(e),
+            additional_args=MappingProxyType({"headers": headers}),
+        )
+        raise
+
+    return response
+
+
 def _complete_heroku(ctx: _CompletionDispatchContext) -> _CompletionDispatchResult:
     acompletion: Final = ctx.acompletion
     api_base: Final = ctx.api_base
@@ -5600,6 +5651,8 @@ def completion(
         elif custom_llm_provider == "fireworks_ai":
             ## COMPLETION CALL
             response = _complete_fireworks_ai(_dispatch_ctx)
+        elif custom_llm_provider == "together_ai":
+            response = _complete_together_ai(_dispatch_ctx)
         elif custom_llm_provider == "heroku":
             response = _complete_heroku(_dispatch_ctx)
 
@@ -5649,7 +5702,6 @@ def completion(
             or custom_llm_provider == "volcengine"
             or custom_llm_provider == "anyscale"
             or custom_llm_provider == "openai"
-            or custom_llm_provider == "together_ai"
             or custom_llm_provider == "nebius"
             or custom_llm_provider == "wandb"
             or custom_llm_provider == "clarifai"
@@ -5699,14 +5751,6 @@ def completion(
             response = _complete_openrouter(_dispatch_ctx)
         elif custom_llm_provider == "vercel_ai_gateway":
             response = _complete_vercel_ai_gateway(_dispatch_ctx)
-        elif (
-            custom_llm_provider == "together_ai"
-            or ("togethercomputer" in model)
-            or (model in litellm.together_ai_models)
-        ):
-            """
-            Deprecated. We now do together ai calls via the openai client - https://docs.together.ai/docs/openai-api-compatibility
-            """
         elif custom_llm_provider == "palm":
             raise ValueError(
                 "Palm was decommisioned on October 2024. Please use the `gemini/` route for Gemini Google AI Studio Models. Announcement: https://ai.google.dev/palm_docs/palm?hl=en"
diff --git a/litellm/utils.py b/litellm/utils.py
index 012e8785321..43f146d52d5 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -4130,7 +4130,7 @@ def get_optional_params(
             drop_params=(drop_params if drop_params is not None and isinstance(drop_params, bool) else False),
         )
     elif custom_llm_provider == "together_ai":
-        optional_params = litellm.TogetherAIConfig().map_openai_params(
+        optional_params = litellm.TogetherAIChatConfig().map_openai_params(
             non_default_params=non_default_params,
             optional_params=optional_params,
             model=model,
@@ -7898,7 +7898,7 @@ class ProviderConfigManager:
             LlmProviders.GALADRIEL: (lambda: litellm.GaladrielChatConfig(), False),
             LlmProviders.REPLICATE: (lambda: litellm.ReplicateConfig(), False),
             LlmProviders.HUGGINGFACE: (lambda: litellm.HuggingFaceChatConfig(), False),
-            LlmProviders.TOGETHER_AI: (lambda: litellm.TogetherAIConfig(), False),
+            LlmProviders.TOGETHER_AI: (lambda: litellm.TogetherAIChatConfig(), False),
             LlmProviders.OPENROUTER: (lambda: litellm.OpenrouterConfig(), False),
             LlmProviders.VERCEL_AI_GATEWAY: (
                 lambda: litellm.VercelAIGatewayConfig(),
diff --git a/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py b/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py
new file mode 100644
index 00000000000..6216d3bf225
--- /dev/null
+++ b/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py
@@ -0,0 +1,232 @@
+import json
+from unittest.mock import MagicMock
+
+import httpx
+import pytest
+
+import litellm
+from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
+from litellm.llms.openai.chat.gpt_transformation import (
+    OpenAIChatCompletionStreamingHandler,
+)
+from litellm.llms.together_ai.chat.transformation import TogetherAIChatConfig
+from litellm.types.utils import LlmProviders, ModelResponse
+
+TOOL_CALLING_MODEL = "openai/gpt-oss-20b"
+REASONING_MODEL = "deepseek-ai/DeepSeek-V3.1"
+PLAIN_MODEL = "Qwen/Qwen3-235B-A22B-fp8-tput"
+UNMAPPED_MODEL = "MiniMaxAI/MiniMax-M3"
+
+FUNCTION_CALLING_PARAMS = ("tools", "tool_choice", "function_call", "response_format")
+
+
+@pytest.fixture(autouse=True)
+def force_local_model_cost(monkeypatch):
+    monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
+    from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map
+
+    monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url=litellm.model_cost_map_url))
+
+
+def test_supported_params_tool_calling_model():
+    supported = TogetherAIChatConfig().get_supported_openai_params(model=TOOL_CALLING_MODEL)
+
+    for param in FUNCTION_CALLING_PARAMS:
+        assert param in supported
+
+
+def test_supported_params_plain_model():
+    supported = TogetherAIChatConfig().get_supported_openai_params(model=PLAIN_MODEL)
+
+    for param in FUNCTION_CALLING_PARAMS:
+        assert param not in supported
+    assert "temperature" in supported
+    assert "max_tokens" in supported
+
+
+def test_supported_params_unmapped_model_treated_as_plain():
+    supported = TogetherAIChatConfig().get_supported_openai_params(model=UNMAPPED_MODEL)
+
+    for param in FUNCTION_CALLING_PARAMS:
+        assert param not in supported
+    assert "stream" in supported
+
+
+def test_map_openai_params_tool_calling_model_passes_tools():
+    tools = [{"type": "function", "function": {"name": "get_weather", "parameters": {}}}]
+
+    mapped = TogetherAIChatConfig().map_openai_params(
+        non_default_params={"tools": tools, "tool_choice": "auto"},
+        optional_params={},
+        model=TOOL_CALLING_MODEL,
+        drop_params=False,
+    )
+
+    assert mapped["tools"] == tools
+    assert mapped["tool_choice"] == "auto"
+
+
+def test_map_openai_params_reasoning_model_passes_sampling_params():
+    mapped = TogetherAIChatConfig().map_openai_params(
+        non_default_params={"temperature": 0.2, "max_tokens": 512},
+        optional_params={},
+        model=REASONING_MODEL,
+        drop_params=False,
+    )
+
+    assert mapped["temperature"] == 0.2
+    assert mapped["max_tokens"] == 512
+
+
+def test_map_openai_params_drops_text_response_format():
+    mapped = TogetherAIChatConfig().map_openai_params(
+        non_default_params={"response_format": {"type": "text"}, "temperature": 0.5},
+        optional_params={},
+        model=REASONING_MODEL,
+        drop_params=False,
+    )
+
+    assert "response_format" not in mapped
+    assert mapped["temperature"] == 0.5
+
+
+def test_map_openai_params_keeps_json_response_format():
+    response_format = {"type": "json_object"}
+
+    mapped = TogetherAIChatConfig().map_openai_params(
+        non_default_params={"response_format": response_format},
+        optional_params={},
+        model=TOOL_CALLING_MODEL,
+        drop_params=False,
+    )
+
+    assert mapped["response_format"] == response_format
+
+
+def _transform_response(message: dict) -> ModelResponse:
+    raw_response_json = {
+        "id": "chatcmpl-test",
+        "object": "chat.completion",
+        "created": 1234567890,
+        "model": REASONING_MODEL,
+        "choices": [{"index": 0, "message": message, "finish_reason": "stop"}],
+        "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
+    }
+    mock_response = MagicMock(spec=httpx.Response)
+    mock_response.json.return_value = raw_response_json
+    mock_response.text = json.dumps(raw_response_json)
+    mock_response.headers = {}
+    logging_obj = MagicMock(spec=LiteLLMLoggingObj)
+    logging_obj.post_call = MagicMock()
+    logging_obj.model_call_details = {}
+
+    return TogetherAIChatConfig().transform_response(
+        model=REASONING_MODEL,
+        raw_response=mock_response,
+        model_response=ModelResponse(),
+        logging_obj=logging_obj,
+        request_data={},
+        messages=[{"role": "user", "content": "What is 2+2?"}],
+        optional_params={},
+        litellm_params={},
+        encoding=None,
+        api_key="test-key",
+        json_mode=False,
+    )
+
+
+def test_transform_response_maps_reasoning_to_reasoning_content():
+    result = _transform_response(
+        {"role": "assistant", "content": "4", "reasoning": "2+2 equals 4"}
+    )
+
+    assert result.choices[0].message.content == "4"
+    assert result.choices[0].message.reasoning_content == "2+2 equals 4"
+
+
+def test_transform_response_preserves_reasoning_content_field():
+    result = _transform_response(
+        {"role": "assistant", "content": "4", "reasoning_content": "adding 2 and 2"}
+    )
+
+    assert result.choices[0].message.reasoning_content == "adding 2 and 2"
+
+
+def test_streaming_chunk_maps_delta_reasoning_to_reasoning_content():
+    iterator = TogetherAIChatConfig().get_model_response_iterator(
+        streaming_response=iter(()), sync_stream=True
+    )
+    assert isinstance(iterator, OpenAIChatCompletionStreamingHandler)
+
+    parsed = iterator.chunk_parser(
+        {
+            "id": "chunk-1",
+            "created": 1234567890,
+            "model": REASONING_MODEL,
+            "choices": [{"index": 0, "delta": {"reasoning": "thinking about 2+2"}}],
+        }
+    )
+
+    assert parsed.choices[0]["delta"]["reasoning_content"] == "thinking about 2+2"
+
+
+def test_together_ai_config_alias_points_at_chat_config():
+    assert litellm.TogetherAIConfig is litellm.TogetherAIChatConfig
+    config = litellm.TogetherAIConfig(max_tokens=10)
+    assert isinstance(config, TogetherAIChatConfig)
+
+
+def test_provider_config_manager_returns_together_chat_config():
+    from litellm.utils import ProviderConfigManager
+
+    config = ProviderConfigManager.get_provider_chat_config(
+        model=REASONING_MODEL, provider=LlmProviders.TOGETHER_AI
+    )
+
+    assert isinstance(config, TogetherAIChatConfig)
+
+
+def test_completion_routes_through_together_chat_config():
+    from litellm.llms.custom_httpx.http_handler import HTTPHandler
+
+    captured_requests = []
+
+    def respond(request: httpx.Request) -> httpx.Response:
+        captured_requests.append(request)
+        return httpx.Response(
+            200,
+            json={
+                "id": "chatcmpl-together",
+                "object": "chat.completion",
+                "created": 1234567890,
+                "model": REASONING_MODEL,
+                "choices": [
+                    {
+                        "index": 0,
+                        "message": {
+                            "role": "assistant",
+                            "content": "4",
+                            "reasoning": "2+2 equals 4",
+                        },
+                        "finish_reason": "stop",
+                    }
+                ],
+                "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
+            },
+        )
+
+    client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond)))
+
+    response = litellm.completion(
+        model=f"together_ai/{REASONING_MODEL}",
+        messages=[{"role": "user", "content": "What is 2+2?"}],
+        api_key="fake-key",
+        client=client,
+    )
+
+    request = captured_requests[0]
+    assert str(request.url) == "https://api.together.ai/v1/chat/completions"
+    assert request.headers["authorization"] == "Bearer fake-key"
+    assert json.loads(request.content)["model"] == REASONING_MODEL
+    assert response.choices[0].message.content == "4"
+    assert response.choices[0].message.reasoning_content == "2+2 equals 4"

From 32ebfba5ed7810ead375c613ee2419e167ba831c Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 12:03:51 -0700
Subject: [PATCH 040/105] refactor(together_ai): build the trimmed
 supported-params list without mutating the inherited list

---
 litellm/llms/together_ai/chat/transformation.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/litellm/llms/together_ai/chat/transformation.py b/litellm/llms/together_ai/chat/transformation.py
index eb0954bceef..88fd79f2366 100644
--- a/litellm/llms/together_ai/chat/transformation.py
+++ b/litellm/llms/together_ai/chat/transformation.py
@@ -30,10 +30,9 @@ class TogetherAIChatConfig(OpenAIGPTConfig):
         verbose_logger.debug(
             "Only some together models support function calling/response_format. Docs - https://docs.together.ai/docs/function-calling"
         )
-        for param in FUNCTION_CALLING_ONLY_PARAMS:
-            if param in supported_params:
-                supported_params.remove(param)
-        return supported_params
+        return [  # mutable-ok: the inherited contract returns a plain list; building fresh avoids mutating the base class's value
+            param for param in supported_params if param not in FUNCTION_CALLING_ONLY_PARAMS
+        ]
 
     def map_openai_params(
         self,

From 17845b4fb01b8ec3d0c90254bece1b802807f77c Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 12:20:51 -0700
Subject: [PATCH 041/105] fix(anthropic): translate tool_result document blocks
 in the /v1/messages bridge

---
 .../adapters/transformation.py                |  6 +-
 ...al_pass_through_adapters_transformation.py | 71 +++++++++++++++++++
 2 files changed, 74 insertions(+), 3 deletions(-)

diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
index 7c89da81fe6..109017bda27 100644
--- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
+++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
@@ -434,7 +434,7 @@ class LiteLLMAnthropicMessagesAdapter:
                                 content_items = list(content.get("content", []))
 
                                 # Single-item text keeps the backward-compatible string format; a single
-                                # image becomes a structured image_url part
+                                # image or document becomes a structured image_url part
                                 if len(content_items) == 1:
                                     c = content_items[0]
                                     if isinstance(c, str):
@@ -454,7 +454,7 @@ class LiteLLMAnthropicMessagesAdapter:
                                             )
                                             self._add_cache_control_if_applicable(content, tool_result, model)
                                             tool_message_list.append(tool_result)
-                                        elif c.get("type") == "image":
+                                        elif c.get("type") in ("image", "document"):
                                             image_part = self._tool_result_image_part(c.get("source"))
                                             tool_result = ChatCompletionToolMessage(
                                                 role="tool",
@@ -482,7 +482,7 @@ class LiteLLMAnthropicMessagesAdapter:
                                                         text=c.get("text", ""),
                                                     )
                                                 )
-                                            elif c.get("type") == "image":
+                                            elif c.get("type") in ("image", "document"):
                                                 image_part = self._tool_result_image_part(c.get("source"))
                                                 if image_part:
                                                     combined_content_parts.append(image_part)
diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py
index d0169963962..a7fbd069e61 100644
--- a/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py
+++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/adapters/test_anthropic_experimental_pass_through_adapters_transformation.py
@@ -1,3 +1,4 @@
+import base64
 from typing import Any, cast
 
 import pytest
@@ -11,6 +12,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
 )
 from litellm.litellm_core_utils.prompt_templates.factory import (
     THOUGHT_SIGNATURE_SEPARATOR,
+    _bedrock_converse_messages_pt,
 )
 from litellm.llms.anthropic.experimental_pass_through.adapters.transformation import (
     OPENAI_MAX_TOOL_NAME_LENGTH,
@@ -3872,6 +3874,75 @@ def test_tool_result_plain_text_unchanged_by_openai_transform():
     assert _image_urls_in_user_messages(result) == []
 
 
+TOOL_RESULT_PDF_B64 = base64.b64encode(b"%PDF-1.4 minimal regression fixture").decode()
+
+
+def _base64_pdf_block():
+    return {
+        "type": "document",
+        "source": {"type": "base64", "media_type": "application/pdf", "data": TOOL_RESULT_PDF_B64},
+    }
+
+
+def test_tool_result_single_document_kept_as_pdf_data_url():
+    adapter = LiteLLMAnthropicMessagesAdapter()
+    translated = adapter.translate_anthropic_messages_to_openai(
+        messages=[
+            _anthropic_tool_use_turn("toolu_01"),
+            _anthropic_tool_result_turn({"toolu_01": [_base64_pdf_block()]}),
+        ]
+    )
+
+    tool_messages = [m for m in translated if m.get("role") == "tool"]
+    assert len(tool_messages) == 1
+    assert tool_messages[0]["content"] == [
+        {
+            "type": "image_url",
+            "image_url": {"url": f"data:application/pdf;base64,{TOOL_RESULT_PDF_B64}"},
+        }
+    ]
+
+
+def test_tool_result_text_and_document_reach_bedrock_converse_tool_result():
+    """Claude Code >= 2.1.245 sends Read-tool PDF output as a document block inside
+    tool_result; dropping it left bedrock converse models blind to the PDF content."""
+    adapter = LiteLLMAnthropicMessagesAdapter()
+    translated = adapter.translate_anthropic_messages_to_openai(
+        messages=[
+            AnthropicMessagesUserMessageParam(role="user", content="Read pong.pdf"),
+            _anthropic_tool_use_turn("toolu_01"),
+            _anthropic_tool_result_turn(
+                {
+                    "toolu_01": [
+                        {"type": "text", "text": "PDF file read: pong.pdf (579 bytes)"},
+                        _base64_pdf_block(),
+                    ]
+                }
+            ),
+        ]
+    )
+
+    converse_messages = _bedrock_converse_messages_pt(
+        messages=translated,
+        model="anthropic.claude-haiku-4-5-20251001-v1:0",
+        llm_provider="bedrock_converse",
+    )
+
+    tool_results = [
+        block["toolResult"]
+        for message in converse_messages
+        for block in message["content"]
+        if "toolResult" in block
+    ]
+    assert len(tool_results) == 1
+    documents = [part["document"] for part in tool_results[0]["content"] if "document" in part]
+    assert len(documents) == 1
+    assert documents[0]["format"] == "pdf"
+    assert documents[0]["source"]["bytes"] == TOOL_RESULT_PDF_B64
+    texts = [part["text"] for part in tool_results[0]["content"] if "text" in part]
+    assert texts == ["PDF file read: pong.pdf (579 bytes)"]
+
+
 def test_translate_anthropic_to_openai_carries_prompt_cache_breakpoint_on_system_and_user_blocks():
     explicit = {"mode": "explicit"}
     openai_request, _ = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(

From 2bd2c1393cb231f6572b4e99f5cff427bcb66562 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 12:31:43 -0700
Subject: [PATCH 042/105] docs(pr-template): split Caveats bullets into
 severity tiers and call for plain engineering language

---
 .github/pull_request_template.md | 16 ++++++++++++++--
 CLAUDE.md                        |  1 +
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index 4e428d8cebf..bcdb228746a 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -1,7 +1,10 @@
+<!-- The whole description's target audience is humans, not AI agents: write it in plain, simple,
+     everyday engineering language, extremely parsable and readable at a glance. This goes double for
+     the TLDR, User Flow, and Caveats sections -->
+
 ## TLDR
 
-<!-- Fill in the bullets below and keep each one short and concrete: one line per bullet, roughly 10 words max
-     This section must be extremely human parsable, comprehensible, and readable: its target audience is humans, not AI agents -->
+<!-- Fill in the bullets below and keep each one short and concrete: one line per bullet, roughly 10 words max -->
 
 Problem this solves:
 
@@ -112,6 +115,15 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
 
 <!-- Short bullet points, just like the TLDR: one line per bullet, roughly 10 words max
      Call out known limitations, follow-up work, or anything a reviewer should watch out for
+     Lead every bullet with its severity: Severe, High, Medium, or Low
+     - Severe: an operator must plan around it before rollout: it can degrade or take down a running
+       deployment (e.g. a slow or table-locking boot migration), rewrite or lose data, break an
+       existing workflow, or change auth behavior
+     - High: unsafe to ship as is: a correctness, security, data-loss, or backward-compatibility hole
+     - Medium: a real gap someone can hit, but with a workaround or a narrow blast radius
+     - Low: anything else worth noting: naming, cleanup, an edge case nobody hits
+     Nest bullets as deep as helps: hierarchy beats one long line when it makes things clearer to a
+     human reader
      Leave this section empty if there are none -->
 
 ## QA runbook
diff --git a/CLAUDE.md b/CLAUDE.md
index b3383b4a895..03053b8392c 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -44,6 +44,7 @@ If you ever make public-facing PR descriptions, comments, issues, commit message
 - don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose
 - don't add a trailing "." at the end of paragraphs (just like this file). That means every paragraph, not just the last one (of the markdown file, PR description, GitHub comment, etc.). Rule of thumb: if you're adding new line(s) before the next sentence, don't add a "."
 - don't use →. Instead, prefer not to use arrows, and if need be, use -> instead
+- do use plain, simple, everyday engineering language: the common phrase engineers actually say over rare compact phrasing, in grammatically complete sentences. When structure genuinely helps the reader, prefer nested bullets (any depth is fine) over one dense line. This applies to all human-facing text: discussion posts, release notes, and docs included
 
 Don't hesitate to use values in .env to get needed API keys and other secrets, as long as you never add them to conversation history, commit them, or include them in GitHub issues / PRs
 

From d749b186de18b1861a996044eca01156e1ae2a4e Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 12:34:31 -0700
Subject: [PATCH 043/105] docs(pr-template): make intent the severe-vs-high
 discriminator

---
 .github/pull_request_template.md | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index bcdb228746a..8a19547cb34 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -116,10 +116,12 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
 <!-- Short bullet points, just like the TLDR: one line per bullet, roughly 10 words max
      Call out known limitations, follow-up work, or anything a reviewer should watch out for
      Lead every bullet with its severity: Severe, High, Medium, or Low
-     - Severe: an operator must plan around it before rollout: it can degrade or take down a running
-       deployment (e.g. a slow or table-locking boot migration), rewrite or lose data, break an
-       existing workflow, or change auth behavior
-     - High: unsafe to ship as is: a correctness, security, data-loss, or backward-compatibility hole
+     - Severe: inherent to what the PR deliberately ships, there even when the code works as intended:
+       it can degrade or take down a running deployment (e.g. a slow or table-locking boot migration),
+       rewrite data by design, break an existing workflow on purpose, or change auth behavior. An
+       operator must plan around it before rollout
+     - High: an unintended hole: a correctness, security, data-loss, or backward-compatibility bug,
+       unsafe to ship as is
      - Medium: a real gap someone can hit, but with a workaround or a narrow blast radius
      - Low: anything else worth noting: naming, cleanup, an edge case nobody hits
      Nest bullets as deep as helps: hierarchy beats one long line when it makes things clearer to a

From e25ce85dbc0dfc9059f11e02387ba0cf7a916a3b Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 25 Aug 2026 12:44:16 -0700
Subject: [PATCH 044/105] fix(proxy): stop expected 4xx responses from
 saturating worker CPU on failure logging (#38102)

* fix(proxy): stop expected 4xx responses from saturating worker CPU on failure logging

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(proxy): keep threaded sync failure handler so CustomLogger sync callbacks still run

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* chore: remove redundant comments per review

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/__init__.py                           |   1 +
 litellm/litellm_core_utils/core_helpers.py    |  20 ++
 litellm/litellm_core_utils/litellm_logging.py |  14 +-
 litellm/proxy/auth/auth_exception_handler.py  |   8 +-
 litellm/proxy/common_request_processing.py    |   9 +-
 litellm/proxy/utils.py                        |  42 ++--
 .../litellm_core_utils/test_core_helpers.py   |  23 ++
 .../test_litellm_logging.py                   |  59 +++++
 .../proxy/auth/test_auth_exception_handler.py |  55 +++++
 .../proxy/test_common_request_processing.py   |  28 +++
 tests/test_litellm/proxy/test_proxy_utils.py  | 218 +++++++++++-------
 11 files changed, 373 insertions(+), 104 deletions(-)

diff --git a/litellm/__init__.py b/litellm/__init__.py
index ee2c551481c..f42b8280761 100644
--- a/litellm/__init__.py
+++ b/litellm/__init__.py
@@ -199,6 +199,7 @@ standard_logging_payload_excluded_fields: Optional[List[str]] = (
     None  # Fields to exclude from StandardLoggingPayload before callbacks receive it
 )
 log_raw_request_response: bool = False
+log_client_error_tracebacks: bool = False
 request_correlation_in_logs: bool = False
 redact_messages_in_exceptions: Optional[bool] = False
 redact_user_api_key_info: Optional[bool] = False
diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py
index de1092bc02f..b71ef4c8e06 100644
--- a/litellm/litellm_core_utils/core_helpers.py
+++ b/litellm/litellm_core_utils/core_helpers.py
@@ -58,6 +58,26 @@ def safe_divide(
     return numerator / denominator
 
 
+def is_expected_client_error(exception: BaseException | None) -> bool:
+    """
+    True when the exception maps to an HTTP 4xx status.
+
+    ProxyException stores the status on .code (as a str), HTTPException and
+    litellm exceptions on .status_code.
+    """
+    if exception is None:
+        return False
+    code: Final[object] = getattr(exception, "code", None)
+    status_code: Final[object] = code if code is not None else getattr(exception, "status_code", None)
+    if status_code is None or isinstance(status_code, bool):
+        return False
+    try:
+        status: Final = int(str(status_code))
+    except ValueError:
+        return False
+    return 400 <= status < 500
+
+
 def coerce_token_limit(value: object) -> int | None:
     """
     Coerce a max_input_tokens / max_output_tokens value to an int, treating a
diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py
index 960ed266e75..626af7530a4 100644
--- a/litellm/litellm_core_utils/litellm_logging.py
+++ b/litellm/litellm_core_utils/litellm_logging.py
@@ -62,7 +62,7 @@ from litellm.integrations.custom_logger import CustomLogger
 from litellm.integrations.deepeval.deepeval import DeepEvalLogger
 from litellm.integrations.mlflow import MlflowLogger
 from litellm.integrations.sqs import SQSLogger
-from litellm.litellm_core_utils.core_helpers import reconstruct_model_name
+from litellm.litellm_core_utils.core_helpers import is_expected_client_error, reconstruct_model_name
 from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
 from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import (
     cost_breakdown_with_guardrail,
@@ -3124,6 +3124,13 @@ class Logging(LiteLLMLoggingBaseClass):
         if not hasattr(self, "model_call_details"):
             self.model_call_details = {}
 
+        if (
+            self.model_call_details.get("log_event_type") == "failed_api_call"
+            and self.model_call_details.get("exception") is exception
+            and self.model_call_details.get("standard_logging_object") is not None
+        ):
+            return start_time, self.model_call_details["end_time"]
+
         self.model_call_details["log_event_type"] = "failed_api_call"
         self.model_call_details["exception"] = exception
         self.model_call_details["traceback_exception"] = (
@@ -5455,9 +5462,10 @@ class StandardLoggingPayloadSetup:
         error_class: Final[str] = str(original_exception.__class__.__name__) if original_exception else ""
         _llm_provider_in_exception: Final = getattr(original_exception, "llm_provider", "")
 
-        # Get traceback information (first 100 lines)
         traceback_info = traceback_str or ""
-        if original_exception:
+        if original_exception and (
+            litellm.log_client_error_tracebacks or not is_expected_client_error(original_exception)
+        ):
             tb: Final[TracebackType | None] = getattr(original_exception, "__traceback__", None)
             if tb:
                 tb_lines: Final = traceback.format_tb(tb)
diff --git a/litellm/proxy/auth/auth_exception_handler.py b/litellm/proxy/auth/auth_exception_handler.py
index 233679126f8..a42187b3a44 100644
--- a/litellm/proxy/auth/auth_exception_handler.py
+++ b/litellm/proxy/auth/auth_exception_handler.py
@@ -11,6 +11,7 @@ import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.constants import EMPTY_MAPPING
 from litellm.integrations.otel.runtime import seed_request_identity
+from litellm.litellm_core_utils.core_helpers import is_expected_client_error
 from litellm.proxy._types import (
     LitellmUserRoles,
     ProxyErrorTypes,
@@ -109,7 +110,12 @@ class UserAPIKeyAuthExceptionHandler:
                 request=request,
                 use_x_forwarded_for=general_settings.get("use_x_forwarded_for") is True,
             )
-            verbose_proxy_logger.exception(
+            log_fn: Final = (
+                verbose_proxy_logger.error
+                if is_expected_client_error(e) and not litellm.log_client_error_tracebacks
+                else verbose_proxy_logger.exception
+            )
+            log_fn(
                 "litellm.proxy.proxy_server.user_api_key_auth(): Exception occured - %s\nRequester IP Address:%s",
                 e,
                 requester_ip,
diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py
index 413fead9bd4..6fd743deb44 100644
--- a/litellm/proxy/common_request_processing.py
+++ b/litellm/proxy/common_request_processing.py
@@ -33,7 +33,7 @@ from litellm.constants import (
     UNSAFE_PROXY_RESPONSE_HEADERS,
 )
 from litellm.integrations.custom_guardrail import CustomGuardrail
-from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket
+from litellm.litellm_core_utils.core_helpers import get_or_create_metadata_bucket, is_expected_client_error
 from litellm.litellm_core_utils.dd_tracing import NullTracer, tracer
 from litellm.litellm_core_utils.get_supported_openai_params import (
     get_supported_openai_params,
@@ -1380,7 +1380,12 @@ def _log_llm_api_exception(e: Exception) -> None:
             "litellm.proxy.proxy_server._handle_llm_api_exception(): client disconnected, upstream LLM request cancelled"
         )
         return
-    verbose_proxy_logger.exception("litellm.proxy.proxy_server._handle_llm_api_exception(): Exception occured - %s", e)
+    log_fn: Final = (
+        verbose_proxy_logger.error
+        if is_expected_client_error(e) and not litellm.log_client_error_tracebacks
+        else verbose_proxy_logger.exception
+    )
+    log_fn("litellm.proxy.proxy_server._handle_llm_api_exception(): Exception occured - %s", e)
 
 
 async def _cancel_llm_call_on_client_disconnect(
diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index 86d954c0913..2bbdad9c487 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -91,7 +91,7 @@ from litellm.integrations.custom_logger import CustomLogger
 from litellm.integrations.prometheus import PrometheusLogger
 from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
 from litellm.integrations.SlackAlerting.utils import _add_langfuse_trace_id_to_alert
-from litellm.litellm_core_utils.core_helpers import coerce_token_limit
+from litellm.litellm_core_utils.core_helpers import coerce_token_limit, is_expected_client_error
 from litellm.litellm_core_utils.litellm_logging import Logging
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
@@ -2575,20 +2575,36 @@ class ProxyLogging:
                 api_key="",
             )
 
-            # log the custom exception
-            await litellm_logging_obj.async_failure_handler(
-                exception=original_exception,
-                traceback_exception=traceback.format_exc(),
+            await self._dispatch_proxy_only_failure_handlers(
+                litellm_logging_obj=litellm_logging_obj,
+                original_exception=original_exception,
             )
 
-            threading.Thread(
-                target=litellm_logging_obj.failure_handler,
-                args=(
-                    original_exception,
-                    traceback.format_exc(),
-                ),
-                daemon=True,
-            ).start()
+    @staticmethod
+    async def _dispatch_proxy_only_failure_handlers(
+        litellm_logging_obj: Logging,
+        original_exception: Exception | None,
+    ) -> None:
+        """Runs the async failure handler plus the threaded sync handler. Expected
+        client (4xx) errors skip traceback formatting unless
+        litellm.log_client_error_tracebacks is set."""
+        include_traceback: Final = litellm.log_client_error_tracebacks or not is_expected_client_error(
+            original_exception
+        )
+        traceback_str: Final = traceback.format_exc() if include_traceback else ""
+        await litellm_logging_obj.async_failure_handler(
+            exception=original_exception,
+            traceback_exception=traceback_str,
+        )
+
+        threading.Thread(
+            target=litellm_logging_obj.failure_handler,
+            args=(
+                original_exception,
+                traceback_str,
+            ),
+            daemon=True,
+        ).start()
 
     async def post_call_success_hook(
         self,
diff --git a/tests/test_litellm/litellm_core_utils/test_core_helpers.py b/tests/test_litellm/litellm_core_utils/test_core_helpers.py
index aa7f2990c13..33a33711698 100644
--- a/tests/test_litellm/litellm_core_utils/test_core_helpers.py
+++ b/tests/test_litellm/litellm_core_utils/test_core_helpers.py
@@ -255,3 +255,26 @@ class TestRedactNestedMatchAndRegexKeys:
     def test_passes_through_none_and_str(self):
         assert redact_nested_match_and_regex_keys(None) is None
         assert redact_nested_match_and_regex_keys("plain") == "plain"
+
+
+class TestIsExpectedClientError:
+    def test_status_ranges(self):
+        from litellm.litellm_core_utils.core_helpers import is_expected_client_error
+
+        class WithStatusCode(Exception):
+            def __init__(self, status_code):
+                self.status_code = status_code
+
+        class WithCode(Exception):
+            def __init__(self, code):
+                self.code = code
+
+        assert is_expected_client_error(WithStatusCode(400)) is True
+        assert is_expected_client_error(WithStatusCode(429)) is True
+        assert is_expected_client_error(WithStatusCode(499)) is True
+        assert is_expected_client_error(WithStatusCode(500)) is False
+        assert is_expected_client_error(WithStatusCode(399)) is False
+        assert is_expected_client_error(WithCode("403")) is True
+        assert is_expected_client_error(WithCode("invalid_request_error")) is False
+        assert is_expected_client_error(Exception("no status")) is False
+        assert is_expected_client_error(None) is False
diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
index cc738d68016..f93daa61570 100644
--- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
+++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
@@ -5678,3 +5678,62 @@ def test_get_custom_logger_compatible_class_finds_v2_newrelic(monkeypatch):
         logging_module._in_memory_loggers.clear()
         monkeypatch.delenv("LITELLM_OTEL_V2", raising=False)
         is_otel_v2_enabled.cache_clear()
+
+
+class _ClientError(Exception):
+    def __init__(self, status_code, message):
+        self.status_code = status_code
+        self.message = message
+        super().__init__(message)
+
+
+def _raise_and_catch(exc):
+    try:
+        raise exc
+    except Exception as caught:
+        return caught
+
+
+def test_get_error_information_skips_traceback_for_expected_4xx(monkeypatch):
+    """Regression for LIT-6043: expected client (4xx) errors must not pay for
+    traceback.format_tb on every rejected request unless
+    litellm.log_client_error_tracebacks is enabled."""
+    from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
+
+    client_exc = _raise_and_catch(_ClientError(status_code=403, message="team does not allow model"))
+    assert client_exc.__traceback__ is not None
+    result = StandardLoggingPayloadSetup.get_error_information(client_exc)
+    assert result["traceback"] == ""
+
+    server_exc = _raise_and_catch(_ClientError(status_code=500, message="boom"))
+    result = StandardLoggingPayloadSetup.get_error_information(server_exc)
+    assert "test_litellm_logging" in result["traceback"]
+
+    monkeypatch.setattr(litellm, "log_client_error_tracebacks", True)
+    result = StandardLoggingPayloadSetup.get_error_information(client_exc)
+    assert "test_litellm_logging" in result["traceback"]
+
+
+def test_failure_handler_helper_fn_builds_payload_once_per_exception():
+    """Regression for LIT-6043: async and sync failure handlers both call
+    _failure_handler_helper_fn for the same failed request; the standardized
+    payload must be built once, not once per handler."""
+    obj = LitellmLogging(
+        model="gpt-4o",
+        messages=[{"role": "user", "content": "Hey"}],
+        stream=False,
+        call_type="acompletion",
+        start_time=time.time(),
+        litellm_call_id="lit-6043-1",
+        function_id="f",
+    )
+    exc = _raise_and_catch(_ClientError(status_code=400, message="invalid model"))
+    obj._failure_handler_helper_fn(exception=exc, traceback_exception="")
+    first_payload = obj.model_call_details["standard_logging_object"]
+    assert first_payload is not None
+    obj._failure_handler_helper_fn(exception=exc, traceback_exception="")
+    assert obj.model_call_details["standard_logging_object"] is first_payload
+
+    other_exc = _raise_and_catch(_ClientError(status_code=429, message="rate limited"))
+    obj._failure_handler_helper_fn(exception=other_exc, traceback_exception="")
+    assert obj.model_call_details["standard_logging_object"] is not first_payload
diff --git a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py
index b0094b81112..90b3b29d919 100644
--- a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py
+++ b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py
@@ -701,3 +701,58 @@ async def test_auth_failure_ip_stamp_does_not_mutate_callers_request_data():
             )
 
     assert request_data == {"model": "gpt-4o"}
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "auth_error,expect_traceback",
+    [
+        pytest.param(
+            ProxyException(
+                message="Authentication Error", type=ProxyErrorTypes.auth_error, param=None, code=401
+            ),
+            False,
+            id="expected_401_no_traceback",
+        ),
+        pytest.param(ValueError("unexpected internal error"), True, id="unexpected_error_keeps_traceback"),
+    ],
+)
+async def test_handle_authentication_error_traceback_only_for_unexpected_errors(auth_error, expect_traceback, caplog):
+    """Regression for LIT-6043: expected 4xx auth rejections must not format a
+    traceback via logger.exception; unexpected errors must keep it."""
+    handler = UserAPIKeyAuthExceptionHandler()
+
+    with (
+        patch(  # test-quality-ok: handler reads proxy_server globals at call time
+            "litellm.proxy.proxy_server.proxy_logging_obj.post_call_failure_hook",
+            new_callable=AsyncMock,
+            return_value=None,
+        ),
+        patch(  # test-quality-ok: handler reads proxy_server globals at call time
+            "litellm.proxy.auth.auth_exception_handler.seed_request_identity",
+        ),
+        patch(  # test-quality-ok: handler reads proxy_server globals at call time
+            "litellm.proxy.proxy_server.general_settings",
+            {"allow_requests_on_db_unavailable": False},
+        ),
+    ):
+        verbose_proxy_logger.propagate = True
+        try:
+            try:
+                raise auth_error
+            except (ProxyException, ValueError) as caught:
+                with caplog.at_level("ERROR", logger="LiteLLM Proxy"), pytest.raises(ProxyException):
+                    await handler._handle_authentication_error(
+                        caught,
+                        MagicMock(),
+                        {},
+                        "/v1/chat/completions",
+                        None,
+                        "sk-bad-key",
+                    )
+        finally:
+            verbose_proxy_logger.propagate = False
+
+    records = [r for r in caplog.records if "user_api_key_auth(): Exception occured" in r.getMessage()]
+    assert len(records) == 1
+    assert (records[0].exc_info is not None) is expect_traceback
diff --git a/tests/test_litellm/proxy/test_common_request_processing.py b/tests/test_litellm/proxy/test_common_request_processing.py
index c9b3e5faf8b..b595c44d2ce 100644
--- a/tests/test_litellm/proxy/test_common_request_processing.py
+++ b/tests/test_litellm/proxy/test_common_request_processing.py
@@ -7343,3 +7343,31 @@ class TestRouterModelNameOnNonStreamingResponse:
         )
 
         assert response[ROUTER_MODEL_NAME_RESPONSE_FIELD] == "deep-model"
+
+
+@pytest.mark.parametrize(
+    "exc,expect_traceback",
+    [
+        pytest.param(HTTPException(status_code=400, detail="Invalid model name passed in"), False, id="expected_400"),
+        pytest.param(ValueError("unexpected internal error"), True, id="unexpected_error"),
+    ],
+)
+def test_log_llm_api_exception_traceback_only_for_unexpected_errors(exc, expect_traceback, caplog):
+    """Regression for LIT-6043: expected 4xx errors log without formatting a
+    traceback; unexpected errors keep logger.exception behavior."""
+    from litellm._logging import verbose_proxy_logger
+    from litellm.proxy.common_request_processing import _log_llm_api_exception
+
+    verbose_proxy_logger.propagate = True
+    try:
+        with caplog.at_level("ERROR", logger="LiteLLM Proxy"):
+            try:
+                raise exc
+            except Exception as raised:
+                _log_llm_api_exception(raised)
+    finally:
+        verbose_proxy_logger.propagate = False
+
+    records = [r for r in caplog.records if "_handle_llm_api_exception(): Exception occured" in r.getMessage()]
+    assert len(records) == 1
+    assert (records[0].exc_info is not None) is expect_traceback
diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py
index fb01216982f..6920cc0dae3 100644
--- a/tests/test_litellm/proxy/test_proxy_utils.py
+++ b/tests/test_litellm/proxy/test_proxy_utils.py
@@ -11,7 +11,6 @@ from litellm.proxy.utils import ProxyLogging
 from litellm.types.guardrails import GuardrailEventHooks
 
 
-
 from unittest.mock import MagicMock, patch
 
 from litellm.proxy.utils import get_custom_url, join_paths
@@ -82,9 +81,7 @@ async def test_proxy_only_error_log_marks_no_upstream_llm_call():
     captured = {}
 
     def fake_pre_call(self, *args, **kwargs):
-        captured["flag"] = self.model_call_details.get(
-            LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL
-        )
+        captured["flag"] = self.model_call_details.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL)
 
     from litellm.litellm_core_utils.litellm_logging import Logging
 
@@ -102,9 +99,7 @@ async def test_proxy_only_error_log_marks_no_upstream_llm_call():
                 "model": "gpt-4o",
                 "messages": [{"role": "user", "content": "hi"}],
             },
-            user_api_key_dict=UserAPIKeyAuth(
-                api_key="sk-bad", request_route="/v1/chat/completions"
-            ),
+            user_api_key_dict=UserAPIKeyAuth(api_key="sk-bad", request_route="/v1/chat/completions"),
             route="/v1/chat/completions",
             original_exception=Exception("bad key"),
         )
@@ -148,13 +143,9 @@ async def test_proxy_only_error_log_keeps_litellm_metadata_in_litellm_params():
             request_data={
                 "model": "gpt-4o",
                 "input": "blocked prompt",
-                "litellm_metadata": {
-                    "standard_logging_guardrail_information": guardrail_info
-                },
+                "litellm_metadata": {"standard_logging_guardrail_information": guardrail_info},
             },
-            user_api_key_dict=UserAPIKeyAuth(
-                api_key="sk-1234", request_route="/v1/responses"
-            ),
+            user_api_key_dict=UserAPIKeyAuth(api_key="sk-1234", request_route="/v1/responses"),
             route="/v1/responses",
             original_exception=HTTPException(status_code=400, detail="blocked"),
         )
@@ -163,12 +154,7 @@ async def test_proxy_only_error_log_keeps_litellm_metadata_in_litellm_params():
         Logging.pre_call = orig_pre_call
         Logging.async_failure_handler = orig_async_failure
 
-    assert (
-        captured["litellm_params"]["litellm_metadata"][
-            "standard_logging_guardrail_information"
-        ]
-        == guardrail_info
-    )
+    assert captured["litellm_params"]["litellm_metadata"]["standard_logging_guardrail_information"] == guardrail_info
     assert "litellm_metadata" not in captured["optional_params"]
 
 
@@ -206,9 +192,7 @@ def test_get_model_group_info_order():
 
 def test_join_paths_no_duplication():
     """Test that join_paths doesn't duplicate route when base_path already ends with it"""
-    result = join_paths(
-        base_path="http://0.0.0.0:4000/my-custom-path/", route="/my-custom-path"
-    )
+    result = join_paths(base_path="http://0.0.0.0:4000/my-custom-path/", route="/my-custom-path")
     assert result == "http://0.0.0.0:4000/my-custom-path"
 
 
@@ -814,9 +798,9 @@ class TestPostCallFailureHookEstimatesDispatchedInputTokens:
 
         estimated = request_data["combined_usage_object"]
         assert isinstance(estimated, Usage)
-        expected = litellm_module.token_counter(model="gpt-3.5-turbo", messages=messages) + litellm_module.token_counter(
-            model="gpt-3.5-turbo", text=system_prompt
-        )
+        expected = litellm_module.token_counter(
+            model="gpt-3.5-turbo", messages=messages
+        ) + litellm_module.token_counter(model="gpt-3.5-turbo", text=system_prompt)
         assert estimated.prompt_tokens == expected
 
     @pytest.mark.asyncio
@@ -834,7 +818,9 @@ class TestPostCallFailureHookEstimatesDispatchedInputTokens:
 
         estimated = request_data["combined_usage_object"]
         assert isinstance(estimated, Usage)
-        expected = litellm_module.token_counter(model="gpt-3.5-turbo", messages=messages) + litellm_module.token_counter(
+        expected = litellm_module.token_counter(
+            model="gpt-3.5-turbo", messages=messages
+        ) + litellm_module.token_counter(
             model="gpt-3.5-turbo", text="part one of the system prompt. part two of the system prompt."
         )
         assert estimated.prompt_tokens == expected
@@ -870,9 +856,9 @@ class TestPostCallFailureHookEstimatesDispatchedInputTokens:
 
         estimated = request_data["combined_usage_object"]
         assert isinstance(estimated, Usage)
-        expected = litellm_module.token_counter(model="gpt-3.5-turbo", messages=messages) + litellm_module.token_counter(
-            model="gpt-3.5-turbo", text=system_prompt
-        )
+        expected = litellm_module.token_counter(
+            model="gpt-3.5-turbo", messages=messages
+        ) + litellm_module.token_counter(model="gpt-3.5-turbo", text=system_prompt)
         assert estimated.prompt_tokens == expected
 
     @pytest.mark.asyncio
@@ -890,9 +876,9 @@ class TestPostCallFailureHookEstimatesDispatchedInputTokens:
 
         estimated = request_data["combined_usage_object"]
         assert isinstance(estimated, Usage)
-        expected = litellm_module.token_counter(model="gpt-3.5-turbo", messages=messages) + litellm_module.token_counter(
-            model="gpt-3.5-turbo", text=dispatched_system
-        )
+        expected = litellm_module.token_counter(
+            model="gpt-3.5-turbo", messages=messages
+        ) + litellm_module.token_counter(model="gpt-3.5-turbo", text=dispatched_system)
         assert estimated.prompt_tokens == expected
 
 
@@ -916,9 +902,7 @@ def test_create_model_info_response_includes_max_tokens_from_lookup():
         model_id="some-model",
         provider="openai",
         llm_router=None,
-        get_model_info=lambda _model: _fake_model_info(
-            max_input_tokens=128000, max_output_tokens=16384
-        ),
+        get_model_info=lambda _model: _fake_model_info(max_input_tokens=128000, max_output_tokens=16384),
     )
 
     assert response["id"] == "some-model"
@@ -935,9 +919,7 @@ def test_create_model_info_response_does_not_call_router_group_info():
         model_id="some-model",
         provider="openai",
         llm_router=router,
-        get_model_info=lambda _model: _fake_model_info(
-            max_input_tokens=128000, max_output_tokens=16384
-        ),
+        get_model_info=lambda _model: _fake_model_info(max_input_tokens=128000, max_output_tokens=16384),
     )
 
     router.get_model_group_info.assert_not_called()
@@ -968,9 +950,7 @@ def test_create_model_info_response_deployment_limits_override_cost_map():
         model_id="gpt-4o",
         provider="openai",
         llm_router=router,
-        get_model_info=lambda _model: _fake_model_info(
-            max_input_tokens=128000, max_output_tokens=16384
-        ),
+        get_model_info=lambda _model: _fake_model_info(max_input_tokens=128000, max_output_tokens=16384),
     )
 
     assert response["max_input_tokens"] == 200000
@@ -1008,9 +988,7 @@ def test_create_model_info_response_survives_malformed_cost_map_limits(bad_value
         model_id="some-model",
         provider="openai",
         llm_router=None,
-        get_model_info=lambda _model: _fake_model_info(
-            max_input_tokens=bad_value, max_output_tokens=bad_value
-        ),
+        get_model_info=lambda _model: _fake_model_info(max_input_tokens=bad_value, max_output_tokens=bad_value),
     )
 
     assert response["id"] == "some-model"
@@ -1023,9 +1001,7 @@ def test_create_model_info_response_keeps_valid_cost_map_limit_beside_malformed_
         model_id="some-model",
         provider="openai",
         llm_router=None,
-        get_model_info=lambda _model: _fake_model_info(
-            max_input_tokens="128,000", max_output_tokens=16384
-        ),
+        get_model_info=lambda _model: _fake_model_info(max_input_tokens="128,000", max_output_tokens=16384),
     )
 
     assert "max_input_tokens" not in response
@@ -1068,9 +1044,7 @@ def test_create_model_info_response_emits_integer_token_counts():
         model_id="some-model",
         provider="openai",
         llm_router=None,
-        get_model_info=lambda _model: _fake_model_info(
-            max_input_tokens=128000, max_output_tokens=16384
-        ),
+        get_model_info=lambda _model: _fake_model_info(max_input_tokens=128000, max_output_tokens=16384),
     )
 
     assert isinstance(response["max_input_tokens"], int)
@@ -1119,9 +1093,7 @@ def test_create_model_info_response_no_router_keeps_base_fields():
 
 
 def test_create_model_info_response_reads_real_cost_map():
-    response = create_model_info_response(
-        model_id="gpt-4o", provider="openai", llm_router=None
-    )
+    response = create_model_info_response(model_id="gpt-4o", provider="openai", llm_router=None)
 
     assert isinstance(response["max_input_tokens"], int)
     assert response["max_input_tokens"] > 0
@@ -1205,10 +1177,7 @@ class TestPostCallFailureHookLLMExceptionAlerting:
 
     @pytest.mark.asyncio
     async def test_http_exception_does_not_alert(self):
-        assert (
-            await self._alerted(HTTPException(status_code=400, detail="blocked"))
-            is False
-        )
+        assert await self._alerted(HTTPException(status_code=400, detail="blocked")) is False
 
     @pytest.mark.asyncio
     async def test_genuine_llm_api_error_still_alerts(self):
@@ -1241,9 +1210,7 @@ class TestPostCallFailureHookProxyExceptionLogging:
             await proxy_logging_obj.post_call_failure_hook(
                 request_data={},
                 original_exception=exc,
-                user_api_key_dict=UserAPIKeyAuth(
-                    api_key="sk-test", request_route=request_route
-                ),
+                user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", request_route=request_route),
             )
         return handle_mock.await_count > 0
 
@@ -1260,20 +1227,12 @@ class TestPostCallFailureHookProxyExceptionLogging:
 
     @pytest.mark.asyncio
     async def test_proxy_exception_on_llm_route_is_logged(self):
-        assert (
-            await self._logged(self._block(), request_route="/v1/chat/completions")
-            is True
-        )
+        assert await self._logged(self._block(), request_route="/v1/chat/completions") is True
 
     @pytest.mark.asyncio
     async def test_generic_exception_on_llm_route_is_not_logged(self):
         # A raw provider/unknown exception is logged by the LLM call path, not here.
-        assert (
-            await self._logged(
-                Exception("upstream 503"), request_route="/v1/chat/completions"
-            )
-            is False
-        )
+        assert await self._logged(Exception("upstream 503"), request_route="/v1/chat/completions") is False
 
 
 class TestShouldUseSmtpSsl:
@@ -1307,9 +1266,7 @@ class TestCreateSmtpConnection:
             patch("smtplib.SMTP_SSL") as mock_smtp_ssl,
             patch("smtplib.SMTP") as mock_smtp,
         ):
-            result = _create_smtp_connection(
-                smtp_host="mail.example.com", smtp_port=465
-            )
+            result = _create_smtp_connection(smtp_host="mail.example.com", smtp_port=465)
 
         mock_smtp.assert_not_called()
         assert result is mock_smtp_ssl.return_value
@@ -1329,9 +1286,7 @@ class TestCreateSmtpConnection:
             patch("smtplib.SMTP_SSL") as mock_smtp_ssl,
             patch("smtplib.SMTP") as mock_smtp,
         ):
-            result = _create_smtp_connection(
-                smtp_host="mail.example.com", smtp_port=587
-            )
+            result = _create_smtp_connection(smtp_host="mail.example.com", smtp_port=587)
 
         mock_smtp_ssl.assert_not_called()
         assert result is mock_smtp.return_value
@@ -1352,9 +1307,7 @@ class TestSendEmailStartTls:
         monkeypatch.delenv("SMTP_USE_SSL", raising=False)
 
         mock_server = MagicMock(spec=smtplib.SMTP)
-        with patch(
-            "litellm.proxy.utils._create_smtp_connection"
-        ) as mock_create_connection:
+        with patch("litellm.proxy.utils._create_smtp_connection") as mock_create_connection:
             mock_create_connection.return_value.__enter__.return_value = mock_server
             await send_email(
                 receiver_email="receiver@example.com",
@@ -1690,9 +1643,7 @@ def test_a_failed_dispatch_is_estimated_as_input_only():
     usage = _estimate_dispatched_failure_usage(FAILURE_USAGE_MODEL, ONE_USER_MESSAGE, None)
 
     assert usage is not None
-    assert usage.prompt_tokens == _count_request_input_tokens(
-        FAILURE_USAGE_MODEL, ONE_USER_MESSAGE, None
-    )
+    assert usage.prompt_tokens == _count_request_input_tokens(FAILURE_USAGE_MODEL, ONE_USER_MESSAGE, None)
     assert usage.completion_tokens == 0
     assert usage.total_tokens == usage.prompt_tokens
 
@@ -1759,9 +1710,7 @@ def test_a_request_that_reached_a_provider_bills_its_input_at_no_cost():
 def test_a_failure_that_cost_the_provider_nothing_lifts_nothing(model_call_details, dispatched):
     from litellm.proxy.utils import _failure_usage_to_lift
 
-    assert _failure_usage_to_lift(
-        model_call_details=model_call_details, request_body={}, dispatched=dispatched
-    ) is None
+    assert _failure_usage_to_lift(model_call_details=model_call_details, request_body={}, dispatched=dispatched) is None
 
 
 def test_the_no_upstream_call_key_the_module_uses_is_the_one_asserted_above():
@@ -1829,3 +1778,102 @@ def test_a_dispatched_failure_lifts_the_four_fields_the_spend_log_needs():
     assert lifted["response_cost"] == 0.0
     assert lifted["combined_usage_object"].prompt_tokens > 0
     assert lifted["standard_logging_object"] == {"id": "log-1"}
+
+
+@pytest.mark.asyncio
+async def test_proxy_only_error_expected_4xx_skips_traceback_for_both_handlers(monkeypatch):
+    """Regression for LIT-6043: an expected 4xx must not format a traceback for
+    either the async or the threaded sync failure handler."""
+    import asyncio
+
+    import litellm
+    from litellm.litellm_core_utils.litellm_logging import Logging
+    from litellm.proxy._types import UserAPIKeyAuth
+
+    monkeypatch.setattr(litellm, "failure_callback", [])
+    proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache())
+    captured = {}
+    sync_ran = asyncio.Event()
+    loop = asyncio.get_running_loop()
+
+    async def fake_async_failure(self, exception, traceback_exception, *args, **kwargs):
+        captured["async_traceback"] = traceback_exception
+
+    def fake_sync_failure(self, exception, traceback_exception, *args, **kwargs):
+        captured["sync_traceback"] = traceback_exception
+        loop.call_soon_threadsafe(sync_ran.set)
+
+    orig_async_failure = Logging.async_failure_handler
+    orig_sync_failure = Logging.failure_handler
+    Logging.async_failure_handler = fake_async_failure
+    Logging.failure_handler = fake_sync_failure
+    try:
+        try:
+            raise HTTPException(status_code=400, detail="Invalid model name passed in")
+        except HTTPException as exc:
+            await proxy_logging_obj._handle_logging_proxy_only_error(
+                request_data={
+                    "model": "does-not-exist",
+                    "messages": [{"role": "user", "content": "hi"}],
+                },
+                user_api_key_dict=UserAPIKeyAuth(api_key="sk-1234", request_route="/v1/chat/completions"),
+                route="/v1/chat/completions",
+                original_exception=exc,
+            )
+        await asyncio.wait_for(sync_ran.wait(), timeout=5)
+    finally:
+        Logging.async_failure_handler = orig_async_failure
+        Logging.failure_handler = orig_sync_failure
+
+    assert captured["async_traceback"] == ""
+    assert captured["sync_traceback"] == ""
+
+
+@pytest.mark.asyncio
+async def test_proxy_only_error_5xx_keeps_traceback_and_runs_sync_callbacks(monkeypatch):
+    """Unexpected (5xx) errors keep the full traceback, and a configured
+    sync-only failure callback still gets its threaded handler."""
+    import asyncio
+
+    import litellm
+    from litellm.litellm_core_utils.litellm_logging import Logging
+    from litellm.proxy._types import UserAPIKeyAuth
+
+    def _custom_sync_callback(kwargs, completion_response, start_time, end_time):
+        pass
+
+    monkeypatch.setattr(litellm, "failure_callback", [_custom_sync_callback])
+    proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache())
+    captured = {}
+    sync_ran = asyncio.Event()
+    loop = asyncio.get_running_loop()
+
+    async def fake_async_failure(self, exception, traceback_exception, *args, **kwargs):
+        captured["async_traceback"] = traceback_exception
+
+    def fake_sync_failure(self, *args, **kwargs):
+        loop.call_soon_threadsafe(sync_ran.set)
+
+    orig_async_failure = Logging.async_failure_handler
+    orig_sync_failure = Logging.failure_handler
+    Logging.async_failure_handler = fake_async_failure
+    Logging.failure_handler = fake_sync_failure
+    try:
+        try:
+            raise HTTPException(status_code=500, detail="internal error")
+        except HTTPException as exc:
+            await proxy_logging_obj._handle_logging_proxy_only_error(
+                request_data={
+                    "model": "gpt-4o",
+                    "messages": [{"role": "user", "content": "hi"}],
+                },
+                user_api_key_dict=UserAPIKeyAuth(api_key="sk-1234", request_route="/v1/chat/completions"),
+                route="/v1/chat/completions",
+                original_exception=exc,
+            )
+        await asyncio.wait_for(sync_ran.wait(), timeout=5)
+    finally:
+        Logging.async_failure_handler = orig_async_failure
+        Logging.failure_handler = orig_sync_failure
+
+    assert "test_proxy_utils" in captured["async_traceback"]

From 896e2598dadcab650e4109320c69e527ede62b18 Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 25 Aug 2026 12:45:23 -0700
Subject: [PATCH 045/105] fix(caching): keep upstream RedisCluster on redis-py
 with per-connection recovery (#38171)

Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../caching/redis_cluster_node_isolation.py   | 34 ++++++++++++++--
 .../test_redis_cluster_node_isolation.py      | 39 ++++++++++++++++++-
 2 files changed, 69 insertions(+), 4 deletions(-)

diff --git a/litellm/caching/redis_cluster_node_isolation.py b/litellm/caching/redis_cluster_node_isolation.py
index 8b0c120e80c..ae8c78709d9 100644
--- a/litellm/caching/redis_cluster_node_isolation.py
+++ b/litellm/caching/redis_cluster_node_isolation.py
@@ -18,6 +18,14 @@ already does when one of its pooled connections errors), leaving every other nod
 connections untouched. Every other branch (MOVED, ASK, CLUSTERDOWN, slot-not-covered,
 retry-exhaustion) is unchanged from upstream, since those already carry real evidence the
 topology changed.
+
+redis-py 8.x fixed this upstream with gentler machinery than this override's
+``node.disconnect()`` (which also kills connections other coroutines are mid-operation
+on, so one timeout cascades into a reconnect storm and, with TLS, a fresh handshake per
+killed connection): it marks in-use connections for reconnect only after their current
+operation completes, disconnects only the idle pooled ones, and defers reinitialization
+to the outer retry loop. When the installed ``ClusterNode`` has that per-connection
+recovery API, the factory returns the base ``RedisCluster`` unmodified.
 """
 
 import asyncio
@@ -72,8 +80,16 @@ class _ClusterAttrs(Protocol):
 _VERIFIED_REDIS_VERSIONS: Final = frozenset({"5.3.1"})
 
 
-def get_litellm_async_redis_cluster_class() -> type["_AsyncRedisClusterType"]:
-    """Builds the ``RedisCluster`` subclass with the per-node isolation fix.
+def get_litellm_async_redis_cluster_class(
+    cluster_node_class: type | None = None,
+) -> type["_AsyncRedisClusterType"]:
+    """Returns the base ``RedisCluster`` when the installed redis-py already recovers a
+    node-level connection error per-connection (8.x+), else builds the ``RedisCluster``
+    subclass with the per-node isolation fix for older versions whose upstream branch
+    tears down the whole cluster client.
+
+    ``cluster_node_class`` exists for dependency injection in tests; production callers
+    leave it unset and the installed ``ClusterNode`` is used.
 
     Imported lazily because this module is reachable from a base ``import litellm`` while
     redis is not a base dependency. Cheap to call repeatedly: the underlying redis
@@ -81,7 +97,10 @@ def get_litellm_async_redis_cluster_class() -> type["_AsyncRedisClusterType"]:
     """
     import redis
     from redis.asyncio.cluster import (
-        RedisCluster as _BaseAsyncRedisCluster,  # pyright: ignore[reportUnknownVariableType]  # redis-py ships no resolvable stub for this class under the repo's current (stale) types-redis pin
+        ClusterNode as _AsyncClusterNode,  # pyright: ignore[reportUnknownVariableType]  # redis-py ships no resolvable stub for this class under the repo's current (stale) types-redis pin
+    )
+    from redis.asyncio.cluster import (
+        RedisCluster as _BaseAsyncRedisCluster,  # pyright: ignore[reportUnknownVariableType]  # same stale-stub gap as the import above
     )
     from redis.cluster import get_node_name
     from redis.commands import READ_COMMANDS
@@ -98,6 +117,15 @@ def get_litellm_async_redis_cluster_class() -> type["_AsyncRedisClusterType"]:
     from redis.exceptions import ConnectionError as _RedisConnectionError
     from redis.exceptions import TimeoutError as _RedisTimeoutError
 
+    node_class: Final = cluster_node_class if cluster_node_class is not None else _AsyncClusterNode
+    if hasattr(node_class, "update_active_connections_for_reconnect"):
+        verbose_logger.debug(
+            "redis-py %s recovers a node-level connection error per-connection upstream; "
+            "using the base RedisCluster without litellm's node-isolation override.",
+            redis.__version__,
+        )
+        return _BaseAsyncRedisCluster
+
     if redis.__version__ not in _VERIFIED_REDIS_VERSIONS:
         verbose_logger.warning(
             "redis-py %s is not in the set this cluster-teardown-storm fix was verified "
diff --git a/tests/test_litellm/caching/test_redis_cluster_node_isolation.py b/tests/test_litellm/caching/test_redis_cluster_node_isolation.py
index f4cd3ab20ef..c16ceec8c31 100644
--- a/tests/test_litellm/caching/test_redis_cluster_node_isolation.py
+++ b/tests/test_litellm/caching/test_redis_cluster_node_isolation.py
@@ -28,6 +28,14 @@ if TYPE_CHECKING:
     from redis.asyncio.cluster import RedisCluster as _AsyncRedisClusterType
 
 
+class _NodeClassWithPerConnectionRecovery:
+    def update_active_connections_for_reconnect(self) -> None: ...
+
+
+class _NodeClassWithoutPerConnectionRecovery:
+    pass
+
+
 class _FakeClusterNode:
     def __init__(self, name: str, raises: Exception | None = None, response: object = None) -> None:
         self.name = name
@@ -47,7 +55,9 @@ class _FakeNodesManager:
 
 
 def _build_cluster_instance() -> "_AsyncRedisClusterType":
-    cluster_cls = get_litellm_async_redis_cluster_class()
+    cluster_cls = get_litellm_async_redis_cluster_class(
+        cluster_node_class=_NodeClassWithoutPerConnectionRecovery
+    )
     instance = cluster_cls.__new__(cluster_cls)
     instance.RedisClusterRequestTTL = 1
     instance.reinitialize_counter = 0
@@ -58,6 +68,33 @@ def _build_cluster_instance() -> "_AsyncRedisClusterType":
     return instance
 
 
+def test_per_connection_recovery_redis_py_gets_the_unmodified_upstream_class() -> None:
+    """Regression (redis-py 8.x): when upstream ClusterNode already recovers a node-level
+    connection error per-connection, the factory must NOT install the copied override,
+    whose node.disconnect() also kills connections other coroutines are mid-operation on."""
+    from redis.asyncio.cluster import RedisCluster
+
+    cluster_cls = get_litellm_async_redis_cluster_class(
+        cluster_node_class=_NodeClassWithPerConnectionRecovery
+    )
+
+    assert cluster_cls is RedisCluster
+
+
+def test_pre_recovery_redis_py_still_gets_the_node_isolation_override() -> None:
+    """Old redis-py (5.x) responds to a node-level error with a full-cluster aclose(),
+    so those versions must keep litellm's per-node isolation override."""
+    from redis.asyncio.cluster import RedisCluster
+
+    cluster_cls = get_litellm_async_redis_cluster_class(
+        cluster_node_class=_NodeClassWithoutPerConnectionRecovery
+    )
+
+    assert cluster_cls is not RedisCluster
+    assert issubclass(cluster_cls, RedisCluster)
+    assert "_execute_command" in cluster_cls.__dict__
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("error_cls", [RedisConnectionError, RedisTimeoutError])
 async def test_node_level_error_resets_only_that_node_not_the_whole_client(error_cls: type[Exception]) -> None:

From 0458accaa0b968ca37b98322012512572fadd0c7 Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 25 Aug 2026 12:47:37 -0700
Subject: [PATCH 046/105] perf(auth): drop guaranteed-miss internal-cache Redis
 read from team object lookup (#38073)

Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/proxy/auth/auth_checks.py             | 14 ----
 .../access_group_endpoints.py                 |  2 -
 .../proxy/auth/test_auth_checks.py            | 82 +++++++++++++++++--
 3 files changed, 76 insertions(+), 22 deletions(-)

diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py
index 4af942a357e..5d52af3b364 100644
--- a/litellm/proxy/auth/auth_checks.py
+++ b/litellm/proxy/auth/auth_checks.py
@@ -71,7 +71,6 @@ from litellm.proxy.auth.budget_throttle import (
 )
 from litellm.proxy.auth.route_checks import RouteChecks
 from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import publish_auth_cache_invalidation
-from litellm.proxy.common_utils.cache_pydantic_utils import CacheCodec
 from litellm.proxy.common_utils.http_parsing_utils import (
     _safe_get_request_headers,
     _safe_get_request_query_params,
@@ -2741,20 +2740,9 @@ async def _get_team_object_from_user_api_key_cache(
 
 async def _get_team_object_from_cache(
     key: str,
-    proxy_logging_obj: ProxyLogging | None,
     user_api_key_cache: UserApiKeyCache,
     parent_otel_span: Span | None,
 ) -> LiteLLM_TeamTableCachedObj | None:
-    ## INTERNAL USAGE CACHE (plain DualCache) — checked before UserApiKeyCache stores ##
-    if proxy_logging_obj is not None and proxy_logging_obj.internal_usage_cache.dual_cache:
-        cached_raw: Final = await proxy_logging_obj.internal_usage_cache.dual_cache.async_get_cache(
-            key=key, parent_otel_span=parent_otel_span
-        )
-        if cached_raw is not None:
-            from_internal: Final = CacheCodec.deserialize(cached_raw, LiteLLM_TeamTableCachedObj)
-            if from_internal is not None:
-                return from_internal
-
     decoded: Final = await user_api_key_cache.async_get_cache(
         key=key,
         parent_otel_span=parent_otel_span,
@@ -2790,7 +2778,6 @@ async def get_team_object(
     if not check_db_only:
         cached_team_obj: Final = await _get_team_object_from_cache(
             key=key,
-            proxy_logging_obj=proxy_logging_obj,
             user_api_key_cache=user_api_key_cache,
             parent_otel_span=parent_otel_span,
         )
@@ -2953,7 +2940,6 @@ async def get_team_object_by_alias(
 
     cached_team_obj: Final = await _get_team_object_from_cache(
         key=cache_key,
-        proxy_logging_obj=proxy_logging_obj,
         user_api_key_cache=user_api_key_cache,
         parent_otel_span=parent_otel_span,
     )
diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py
index 2271501d480..c803d87e436 100644
--- a/litellm/proxy/management_endpoints/access_group_endpoints.py
+++ b/litellm/proxy/management_endpoints/access_group_endpoints.py
@@ -210,7 +210,6 @@ async def _patch_team_caches_add_access_group(
     for team_id in team_ids:
         cached_team = await _get_team_object_from_cache(
             key=f"team_id:{team_id}",
-            proxy_logging_obj=proxy_logging_obj,
             user_api_key_cache=user_api_key_cache,
             parent_otel_span=None,
         )
@@ -240,7 +239,6 @@ async def _patch_team_caches_remove_access_group(
     for team_id in team_ids:
         cached_team = await _get_team_object_from_cache(
             key=f"team_id:{team_id}",
-            proxy_logging_obj=proxy_logging_obj,
             user_api_key_cache=user_api_key_cache,
             parent_otel_span=None,
         )
diff --git a/tests/test_litellm/proxy/auth/test_auth_checks.py b/tests/test_litellm/proxy/auth/test_auth_checks.py
index abe73f7d05c..06fd3730f7d 100644
--- a/tests/test_litellm/proxy/auth/test_auth_checks.py
+++ b/tests/test_litellm/proxy/auth/test_auth_checks.py
@@ -4869,10 +4869,9 @@ async def test_cache_team_object_writes_team_id_and_invalidates_team_alias():
       3. When team_alias is None, NO alias-key operation happens (no
          delete of an empty-keyed entry, no spurious write).
       4. DELETES the team_id-keyed entry from the internal usage cache
-         BEFORE the fresh write (LIT-4391). `_get_team_object_from_cache`
-         consults the internal usage cache first, so a leftover copy there
-         (backfilled from a Redis shared with `user_api_key_cache`) would
-         keep serving the pre-update team allowlist.
+         BEFORE the fresh write (LIT-4391). `_get_team_object_from_cache` no
+         longer reads the internal usage cache (LIT-5944), but the delete
+         protects mixed-version rolling deploys where older workers still do.
     """
     from unittest.mock import AsyncMock, MagicMock
 
@@ -4981,8 +4980,10 @@ async def test_team_update_not_shadowed_by_internal_usage_cache_lit_4391():
     Regression test for LIT-4391: keys with models=["all-team-models"] kept
     getting 403 team_model_access_denied for models added via /team/update.
 
-    `_get_team_object_from_cache` consults the internal usage cache BEFORE
-    `user_api_key_cache`. When both share one Redis (enable_redis_auth_cache),
+    `_get_team_object_from_cache` used to consult the internal usage cache
+    BEFORE `user_api_key_cache` (removed in LIT-5944; this test now also
+    guards against reintroducing that read).
+    When both share one Redis (enable_redis_auth_cache),
     any team read backfills the internal cache's in-memory tier with the team
     object. `_cache_team_object` (the /team/update refresh) only wrote
     `user_api_key_cache`, so that backfilled copy kept shadowing the update
@@ -5054,6 +5055,75 @@ async def test_team_update_not_shadowed_by_internal_usage_cache_lit_4391():
     )
 
 
+class _CountingFakeRedis(_SharedFakeRedis):
+    """Counts per-key Redis round-trips so tests can pin the number of
+    network operations a code path issues."""
+
+    def __init__(self):
+        super().__init__()
+        self.get_calls: int = 0
+
+    async def async_get_cache(self, key, **kwargs):
+        self.get_calls += 1
+        return await super().async_get_cache(key, **kwargs)
+
+
+@pytest.mark.asyncio
+async def test_warm_team_object_reads_issue_no_redis_ops_lit_5944():
+    """
+    Regression test for LIT-5944: project/team-scoped virtual-key requests
+    paid ~4 awaited Redis GETs per request just to re-read the team object.
+
+    `_get_team_object_from_cache` used to consult
+    `proxy_logging_obj.internal_usage_cache.dual_cache` (in-memory TTL 1s,
+    Redis-backed) BEFORE `user_api_key_cache`. Nothing writes team objects
+    into that internal cache — `_cache_team_object` only DELETES the key
+    there — so when `user_api_key_cache` has no Redis tier the shared Redis
+    key stays absent forever and every team lookup in the auth hot path
+    (4 call sites per chat-completion request) became a guaranteed-miss
+    Redis round-trip, saturating the event loop at high TPS.
+
+    Pins: once `_cache_team_object` has cached a team, repeated
+    `get_team_object` reads are served from `user_api_key_cache`'s in-memory
+    tier and issue ZERO Redis operations.
+    """
+    from litellm.caching.dual_cache import DualCache
+    from litellm.proxy._types import LiteLLM_TeamTableCachedObj
+    from litellm.proxy.auth.auth_checks import _cache_team_object, get_team_object
+
+    team_id = "team-lit-5944"
+    counting_redis = _CountingFakeRedis()
+    user_api_key_cache = UserApiKeyCache()
+    proxy_logging_obj = MagicMock()
+    proxy_logging_obj.internal_usage_cache.dual_cache = DualCache(
+        redis_cache=counting_redis,
+        default_in_memory_ttl=1,
+    )
+    prisma_client = MagicMock()
+
+    await _cache_team_object(
+        team_id=team_id,
+        team_table=LiteLLM_TeamTableCachedObj(team_id=team_id, models=["model-a"]),
+        user_api_key_cache=user_api_key_cache,
+        proxy_logging_obj=proxy_logging_obj,
+    )
+
+    for _ in range(4):
+        team_obj = await get_team_object(
+            team_id=team_id,
+            prisma_client=prisma_client,
+            user_api_key_cache=user_api_key_cache,
+            proxy_logging_obj=proxy_logging_obj,
+        )
+        assert team_obj is not None and team_obj.models == ["model-a"]
+
+    assert counting_redis.get_calls == 0, (
+        "Warm team-object reads must be served from user_api_key_cache's "
+        "in-memory tier without any Redis round-trips. "
+        f"Got {counting_redis.get_calls} Redis GETs for 4 get_team_object calls."
+    )
+
+
 @pytest.mark.asyncio
 async def test_cache_team_object_tolerates_cache_invalidation_failures():
     """

From 8829b3269f0d72b52a105e6c1ec4f9e1d07ef207 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 12:52:54 -0700
Subject: [PATCH 047/105] docs(pr-template): group caveats under severity
 subheadings and promote Final Attestation

---
 .github/pull_request_template.md | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index 8a19547cb34..e85a397cbd2 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -113,9 +113,10 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
 
 ## Caveats (if any)
 
-<!-- Short bullet points, just like the TLDR: one line per bullet, roughly 10 words max
+<!-- Group caveats under severity subheadings (### Severe, ### High, ### Medium, ### Low), with
+     short bullet points inside each, just like the TLDR: one line per bullet, roughly 10 words max
      Call out known limitations, follow-up work, or anything a reviewer should watch out for
-     Lead every bullet with its severity: Severe, High, Medium, or Low
+     Include only the tiers that have caveats; drop the empty ones
      - Severe: inherent to what the PR deliberately ships, there even when the code works as intended:
        it can degrade or take down a running deployment (e.g. a slow or table-locking boot migration),
        rewrite data by design, break an existing workflow on purpose, or change auth behavior. An
@@ -148,6 +149,6 @@ Example checklists:
   - [ ] Sanity check: this test makes sense to add and is not hand-wavey (e.g., assert actual expected spend instead of just spend > 0) or potentially flaky
 -->
 
-### Final Attestation
+## Final Attestation
 
 - [ ] The tests check the right things, including the edge cases, and regressions in the respective real-world customer use-cases are not possible after this PR

From 27ca05a70759d8c2e78b2e4c0bc08aa526640ed2 Mon Sep 17 00:00:00 2001
From: Yassin Kortam <yassin@berri.ai>
Date: Tue, 25 Aug 2026 13:01:51 -0700
Subject: [PATCH 048/105] fix(ui): read reasoning tokens from Responses API
 output_tokens_details (#37952)

---
 .../src/components/llm_calls/responses_api.test.tsx  | 12 ++++++++++++
 .../src/components/llm_calls/responses_api.tsx       |  6 ++++--
 2 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx
index 065eaaf3632..033813397fc 100644
--- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx
+++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx
@@ -400,4 +400,16 @@ describe("responses_api prompt cache usage", () => {
     expect(usageData).not.toHaveProperty("cacheCreationTokens");
     expect(usageData.promptTokens).toBe(5000);
   });
+
+  it("surfaces reasoning tokens from Responses-shape output_tokens_details", async () => {
+    await expect(captureUsage({ output_tokens_details: { reasoning_tokens: 42 } })).resolves.toMatchObject({
+      reasoningTokens: 42,
+    });
+  });
+
+  it("falls back to completion_tokens_details reasoning tokens when output_tokens_details is absent", async () => {
+    await expect(captureUsage({ completion_tokens_details: { reasoning_tokens: 17 } })).resolves.toMatchObject({
+      reasoningTokens: 17,
+    });
+  });
 });
diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx
index 94e8cb46765..8d71a4e29a8 100644
--- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx
+++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx
@@ -295,8 +295,10 @@ export async function makeOpenAIResponsesRequest(
             };
 
             // Add reasoning tokens if available
-            if (usage.completion_tokens_details?.reasoning_tokens) {
-              usageData.reasoningTokens = usage.completion_tokens_details.reasoning_tokens;
+            const reasoningTokens =
+              usage.output_tokens_details?.reasoning_tokens ?? usage.completion_tokens_details?.reasoning_tokens;
+            if (reasoningTokens) {
+              usageData.reasoningTokens = reasoningTokens;
             }
 
             if (usage.cost !== undefined && usage.cost !== null) {

From 104fe73113cafa167a11edfa7c268b1d17b5ca29 Mon Sep 17 00:00:00 2001
From: Yassin Kortam <yassin@berri.ai>
Date: Tue, 25 Aug 2026 13:07:09 -0700
Subject: [PATCH 049/105] fix(dashboard): don't show a stale provider
 prompt-cache chip on a response-cache hit (#37951)

* fix(dashboard): don't show a stale provider prompt-cache chip on a response-cache hit

The playground's non-streaming chat completion and responses paths replayed a cache hit's original usage payload verbatim, so ResponseMetrics kept rendering the provider's prompt-cache-write/read chips using token counts from the original request. Detect the hit via the x-litellm-cache-key response header and render a Response Cache indicator instead.

* fix(dashboard): expose x-litellm-cache-key through CORS for the playground cache-hit indicator
---
 litellm/constants.py                          |   1 +
 tests/test_litellm/proxy/test_proxy_server.py |  10 +
 .../chat_ui/ResponseMetrics.test.tsx          |  18 ++
 .../components/chat_ui/ResponseMetrics.tsx    |  20 ++
 .../llm_calls/chat_completion.test.tsx        | 210 +++++++++++++++---
 .../components/llm_calls/chat_completion.tsx  |  10 +-
 .../llm_calls/responses_api.test.tsx          | 188 +++++++++++++---
 .../components/llm_calls/responses_api.tsx    |  12 +-
 8 files changed, 408 insertions(+), 61 deletions(-)

diff --git a/litellm/constants.py b/litellm/constants.py
index 78aba30f9c0..765bbfe1e54 100644
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -147,6 +147,7 @@ LITELLM_UI_ALLOW_HEADERS: Final = [
     "x-litellm-adaptive-router-model",
     "x-litellm-applied-guardrails",
     "x-litellm-guardrail-scan-id",
+    "x-litellm-cache-key",
 ]
 
 # Gemini model-specific minimal thinking budget constants
diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py
index 3383527e932..b9a31acca96 100644
--- a/tests/test_litellm/proxy/test_proxy_server.py
+++ b/tests/test_litellm/proxy/test_proxy_server.py
@@ -78,6 +78,16 @@ def client_no_auth():
     return TestClient(app)
 
 
+def test_cors_exposes_cache_key_header_to_browser_js():
+    from fastapi.middleware.cors import CORSMiddleware
+
+    from litellm.constants import LITELLM_UI_ALLOW_HEADERS
+
+    cors_middleware = next(m for m in app.user_middleware if m.cls is CORSMiddleware)
+    assert cors_middleware.kwargs["expose_headers"] is LITELLM_UI_ALLOW_HEADERS
+    assert "x-litellm-cache-key" in cors_middleware.kwargs["expose_headers"]
+
+
 def test_login_v2_returns_redirect_url_and_sets_cookie(monkeypatch):
     mock_login_result = {"user_id": "test-user"}
     mock_prisma_client = MagicMock()
diff --git a/ui/litellm-dashboard/src/components/chat_ui/ResponseMetrics.test.tsx b/ui/litellm-dashboard/src/components/chat_ui/ResponseMetrics.test.tsx
index 5afc94eb043..f31e1839739 100644
--- a/ui/litellm-dashboard/src/components/chat_ui/ResponseMetrics.test.tsx
+++ b/ui/litellm-dashboard/src/components/chat_ui/ResponseMetrics.test.tsx
@@ -33,4 +33,22 @@ describe("ResponseMetrics prompt cache chips", () => {
     expect(screen.queryByText(/Cache Read/)).not.toBeInTheDocument();
     expect(screen.queryByText(/Cache Write/)).not.toBeInTheDocument();
   });
+
+  it("shows the response cache indicator instead of the provider cache chips on a response-cache hit", () => {
+    render(
+      <ResponseMetrics
+        usage={{ ...baseUsage, cacheReadTokens: 4695, cacheCreationTokens: 1234, servedFromResponseCache: true }}
+      />,
+    );
+
+    expect(screen.getByText("Response Cache: Hit")).toBeInTheDocument();
+    expect(screen.queryByText(/Cache Read/)).not.toBeInTheDocument();
+    expect(screen.queryByText(/Cache Write/)).not.toBeInTheDocument();
+  });
+
+  it("does not show the response cache indicator when the flag is absent", () => {
+    render(<ResponseMetrics usage={baseUsage} />);
+
+    expect(screen.queryByText(/Response Cache/)).not.toBeInTheDocument();
+  });
 });
diff --git a/ui/litellm-dashboard/src/components/chat_ui/ResponseMetrics.tsx b/ui/litellm-dashboard/src/components/chat_ui/ResponseMetrics.tsx
index 3e7f2884b23..ec62d0618d7 100644
--- a/ui/litellm-dashboard/src/components/chat_ui/ResponseMetrics.tsx
+++ b/ui/litellm-dashboard/src/components/chat_ui/ResponseMetrics.tsx
@@ -7,12 +7,16 @@ import {
   DatabaseBackup,
   DollarSign,
   Hash,
+  History,
   Lightbulb,
   Wrench,
 } from "lucide-react";
 import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
 import { PROMPT_CACHE_CREATION_TOOLTIP, PROMPT_CACHE_READ_TOOLTIP } from "@/utils/promptCacheUsage";
 
+const RESPONSE_CACHE_TOOLTIP =
+  "This response was replayed from LiteLLM's response cache. The request never reached the provider, so it did not read from or write to the provider's own prompt cache.";
+
 export interface TokenUsage {
   completionTokens?: number;
   promptTokens?: number;
@@ -21,6 +25,7 @@ export interface TokenUsage {
   cacheReadTokens?: number;
   cacheCreationTokens?: number;
   cost?: number;
+  servedFromResponseCache?: boolean;
 }
 
 interface ResponseMetricsProps {
@@ -51,7 +56,22 @@ function MetricItem({ label, tooltip, icon, value }: MetricItemProps) {
   );
 }
 
+function ResponseCacheIndicator() {
+  return (
+    <MetricItem
+      label="Response Cache"
+      tooltip={RESPONSE_CACHE_TOOLTIP}
+      icon={<History className="size-3" aria-hidden="true" />}
+      value="Hit"
+    />
+  );
+}
+
 function PromptCacheChips({ usage }: { usage?: TokenUsage }) {
+  if (usage?.servedFromResponseCache) {
+    return <ResponseCacheIndicator />;
+  }
+
   const readTokens = usage?.cacheReadTokens ?? 0;
   const creationTokens = usage?.cacheCreationTokens ?? 0;
 
diff --git a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx
index c91de3f6b20..bc4b4e3a351 100644
--- a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx
+++ b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.test.tsx
@@ -24,6 +24,10 @@ vi.mock("openai", () => ({
   },
 }));
 
+const nonStreamingResponse = (data: unknown, headers: Record<string, string> = {}) => ({
+  withResponse: async () => ({ data, response: { headers: new Headers(headers) } }),
+});
+
 describe("chat_completion", () => {
   const mockUpdateUI = vi.fn();
   const mockChatHistory = [{ role: "user", content: "Hello" }];
@@ -226,25 +230,27 @@ describe("chat_completion", () => {
   });
 
   it("should send a non-streaming request and render the whole message at once when streaming is disabled", async () => {
-    mockCreate.mockResolvedValueOnce({
-      id: "chatcmpl-1",
-      object: "chat.completion",
-      created: 1,
-      model: "gpt-4",
-      choices: [
-        {
-          index: 0,
-          finish_reason: "stop",
-          message: { role: "assistant", content: "Hello there" },
+    mockCreate.mockReturnValueOnce(
+      nonStreamingResponse({
+        id: "chatcmpl-1",
+        object: "chat.completion",
+        created: 1,
+        model: "gpt-4",
+        choices: [
+          {
+            index: 0,
+            finish_reason: "stop",
+            message: { role: "assistant", content: "Hello there" },
+          },
+        ],
+        usage: {
+          completion_tokens: 2,
+          prompt_tokens: 5,
+          total_tokens: 7,
+          cost: 0.25,
         },
-      ],
-      usage: {
-        completion_tokens: 2,
-        prompt_tokens: 5,
-        total_tokens: 7,
-        cost: 0.25,
-      },
-    });
+      }),
+    );
 
     const onTimingData = vi.fn();
     const onUsageData = vi.fn();
@@ -298,24 +304,26 @@ describe("chat_completion", () => {
   });
 
   it("should surface reasoning content and MCP metadata from a non-streaming response", async () => {
-    mockCreate.mockResolvedValueOnce({
-      model: "gpt-4",
-      choices: [
-        {
-          index: 0,
-          finish_reason: "stop",
-          message: {
-            role: "assistant",
-            content: "done",
-            reasoning_content: "thinking",
-            provider_specific_fields: {
-              mcp_tool_calls: [{ id: "call_1", function: { name: "search_docs", arguments: "{}" } }],
-              mcp_call_results: [{ tool_call_id: "call_1", result: "found it" }],
+    mockCreate.mockReturnValueOnce(
+      nonStreamingResponse({
+        model: "gpt-4",
+        choices: [
+          {
+            index: 0,
+            finish_reason: "stop",
+            message: {
+              role: "assistant",
+              content: "done",
+              reasoning_content: "thinking",
+              provider_specific_fields: {
+                mcp_tool_calls: [{ id: "call_1", function: { name: "search_docs", arguments: "{}" } }],
+                mcp_call_results: [{ tool_call_id: "call_1", result: "found it" }],
+              },
             },
           },
-        },
-      ],
-    });
+        ],
+      }),
+    );
 
     const onReasoningContent = vi.fn();
     const onMCPEvent = vi.fn();
@@ -459,3 +467,137 @@ describe("chat_completion prompt cache usage", () => {
     expect(usageData).not.toHaveProperty("cacheCreationTokens");
   });
 });
+
+describe("chat_completion response cache", () => {
+  const mockUpdateUI = vi.fn();
+  const mockChatHistory = [{ role: "user", content: "Hello" }];
+
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("flags a non-streaming response-cache hit even though it replays provider prompt-cache usage", async () => {
+    mockCreate.mockReturnValueOnce(
+      nonStreamingResponse(
+        {
+          id: "chatcmpl-replayed",
+          model: "gpt-4",
+          choices: [{ index: 0, finish_reason: "stop", message: { role: "assistant", content: "Hello there" } }],
+          usage: {
+            completion_tokens: 2,
+            prompt_tokens: 5000,
+            total_tokens: 5002,
+            prompt_tokens_details: { cached_tokens: 4695 },
+          },
+        },
+        { "x-litellm-cache-key": "cache-key-abc" },
+      ),
+    );
+
+    const onUsageData = vi.fn();
+
+    await makeOpenAIChatCompletionRequest(
+      mockChatHistory,
+      mockUpdateUI,
+      "gpt-4",
+      "test-token",
+      undefined, // tags
+      undefined, // signal
+      undefined, // onReasoningContent
+      undefined, // onTimingData
+      onUsageData,
+      undefined, // traceId
+      undefined, // vector_store_ids
+      undefined, // guardrails
+      undefined, // policies
+      undefined, // selectedMCPServers
+      undefined, // onImageGenerated
+      undefined, // onSearchResults
+      undefined, // temperature
+      undefined, // max_tokens
+      undefined, // onTotalLatency
+      undefined, // customBaseUrl
+      undefined, // mcpServers
+      undefined, // mcpServerToolRestrictions
+      undefined, // onMCPEvent
+      undefined, // mockTestFallbacks
+      undefined, // mcpToolsets
+      false, // streamingEnabled
+    );
+
+    expect(onUsageData).toHaveBeenCalledWith(
+      expect.objectContaining({ cacheReadTokens: 4695, servedFromResponseCache: true }),
+    );
+  });
+
+  it("does not flag a non-streaming response that missed the response cache", async () => {
+    mockCreate.mockReturnValueOnce(
+      nonStreamingResponse({
+        id: "chatcmpl-fresh",
+        model: "gpt-4",
+        choices: [{ index: 0, finish_reason: "stop", message: { role: "assistant", content: "Hello there" } }],
+        usage: { completion_tokens: 2, prompt_tokens: 5, total_tokens: 7 },
+      }),
+    );
+
+    const onUsageData = vi.fn();
+
+    await makeOpenAIChatCompletionRequest(
+      mockChatHistory,
+      mockUpdateUI,
+      "gpt-4",
+      "test-token",
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      onUsageData,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      false, // streamingEnabled
+    );
+
+    expect(onUsageData).toHaveBeenCalledWith(expect.not.objectContaining({ servedFromResponseCache: true }));
+  });
+
+  it("never flags a streaming response, even when the proxy reports a cache key", async () => {
+    async function* mockStream() {
+      yield {
+        choices: [{ delta: {}, index: 0 }],
+        model: "gpt-4",
+        usage: { completion_tokens: 2, prompt_tokens: 5, total_tokens: 7 },
+      };
+    }
+    mockCreate.mockResolvedValueOnce(mockStream());
+
+    const onUsageData = vi.fn();
+
+    await makeOpenAIChatCompletionRequest(
+      mockChatHistory,
+      mockUpdateUI,
+      "gpt-4",
+      "test-token",
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      onUsageData,
+    );
+
+    expect(onUsageData).toHaveBeenCalledWith(expect.not.objectContaining({ servedFromResponseCache: true }));
+  });
+});
diff --git a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx
index ea66006349b..cd0852bc06e 100644
--- a/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx
+++ b/ui/litellm-dashboard/src/components/llm_calls/chat_completion.tsx
@@ -73,6 +73,7 @@ export async function makeOpenAIChatCompletionRequest(
     const startTime = Date.now();
     let firstTokenReceived = false;
     let timeToFirstToken: number | undefined = undefined;
+    let servedFromResponseCache = false;
 
     // Track MCP metadata cumulatively across chunks
     let mcpMetadata: {
@@ -143,7 +144,13 @@ export async function makeOpenAIChatCompletionRequest(
           { ...requestBody, stream: true, stream_options: { include_usage: true } },
           { signal },
         )
-      : [completionAsSingleChunk(await client.chat.completions.create({ ...requestBody, stream: false }, { signal }))];
+      : await (async () => {
+          const nonStreamingResponse = await client.chat.completions
+            .create({ ...requestBody, stream: false }, { signal })
+            .withResponse();
+          servedFromResponseCache = nonStreamingResponse.response.headers.get("x-litellm-cache-key") !== null;
+          return [completionAsSingleChunk(nonStreamingResponse.data)];
+        })();
 
     for await (const chunk of response) {
       // Process content and measure time to first token
@@ -228,6 +235,7 @@ export async function makeOpenAIChatCompletionRequest(
           promptTokens: chunkWithUsage.usage.prompt_tokens,
           totalTokens: chunkWithUsage.usage.total_tokens,
           ...extractPromptCacheTokens(chunkWithUsage.usage),
+          ...(servedFromResponseCache ? { servedFromResponseCache: true } : {}),
         };
 
         // Check for reasoning tokens
diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx
index 033813397fc..0b94c093acd 100644
--- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx
+++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.test.tsx
@@ -20,6 +20,10 @@ vi.mock("openai", () => ({
   },
 }));
 
+const nonStreamingResponse = (data: unknown, headers: Record<string, string> = {}) => ({
+  withResponse: async () => ({ data, response: { headers: new Headers(headers) } }),
+});
+
 describe("responses_api", () => {
   const mockUpdateTextUI = vi.fn();
   const messages: MessageType[] = [{ role: "user", content: "Hello" }];
@@ -71,19 +75,21 @@ describe("responses_api", () => {
   });
 
   it("should send a non-streaming request and render the whole output at once when streaming is disabled", async () => {
-    mockResponsesCreate.mockResolvedValueOnce({
-      id: "resp_456",
-      output: [
-        {
-          type: "message",
-          content: [
-            { type: "output_text", text: "Full " },
-            { type: "output_text", text: "answer" },
-          ],
-        },
-      ],
-      usage: { output_tokens: 3, input_tokens: 4, total_tokens: 7 },
-    });
+    mockResponsesCreate.mockReturnValueOnce(
+      nonStreamingResponse({
+        id: "resp_456",
+        output: [
+          {
+            type: "message",
+            content: [
+              { type: "output_text", text: "Full " },
+              { type: "output_text", text: "answer" },
+            ],
+          },
+        ],
+        usage: { output_tokens: 3, input_tokens: 4, total_tokens: 7 },
+      }),
+    );
 
     const onTimingData = vi.fn();
     const onUsageData = vi.fn();
@@ -162,10 +168,12 @@ describe("responses_api", () => {
     expect(onTotalLatency).toHaveBeenCalledTimes(1);
     expect(onTotalLatency).toHaveBeenLastCalledWith(expect.any(Number));
 
-    mockResponsesCreate.mockResolvedValueOnce({
-      id: "resp_latency",
-      output: [{ type: "message", content: [{ type: "output_text", text: "Answer" }] }],
-    });
+    mockResponsesCreate.mockReturnValueOnce(
+      nonStreamingResponse({
+        id: "resp_latency",
+        output: [{ type: "message", content: [{ type: "output_text", text: "Answer" }] }],
+      }),
+    );
 
     await callWithStreaming(false);
     expect(onTotalLatency).toHaveBeenCalledTimes(2);
@@ -224,14 +232,16 @@ describe("responses_api", () => {
   });
 
   it("should replay MCP output items as events for a non-streaming response", async () => {
-    mockResponsesCreate.mockResolvedValueOnce({
-      id: "resp_789",
-      output: [
-        { type: "mcp_call", id: "mcp_1", name: "search_docs", arguments: "{}", output: "found it" },
-        { type: "message", content: [{ type: "output_text", text: "Answer" }] },
-      ],
-      usage: { output_tokens: 1, input_tokens: 1, total_tokens: 2 },
-    });
+    mockResponsesCreate.mockReturnValueOnce(
+      nonStreamingResponse({
+        id: "resp_789",
+        output: [
+          { type: "mcp_call", id: "mcp_1", name: "search_docs", arguments: "{}", output: "found it" },
+          { type: "message", content: [{ type: "output_text", text: "Answer" }] },
+        ],
+        usage: { output_tokens: 1, input_tokens: 1, total_tokens: 2 },
+      }),
+    );
 
     const onMCPEvent = vi.fn();
     const onUsageData = vi.fn();
@@ -413,3 +423,131 @@ describe("responses_api prompt cache usage", () => {
     });
   });
 });
+
+describe("responses_api response cache", () => {
+  const mockUpdateTextUI = vi.fn();
+  const messages: MessageType[] = [{ role: "user", content: "Hello" }];
+
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("flags a non-streaming response-cache hit even though it replays provider prompt-cache usage", async () => {
+    mockResponsesCreate.mockReturnValueOnce(
+      nonStreamingResponse(
+        {
+          id: "resp_replayed",
+          output: [{ type: "message", content: [{ type: "output_text", text: "Full answer" }] }],
+          usage: {
+            output_tokens: 2,
+            input_tokens: 5000,
+            total_tokens: 5002,
+            input_tokens_details: { cached_tokens: 4695 },
+          },
+        },
+        { "x-litellm-cache-key": "cache-key-abc" },
+      ),
+    );
+
+    const onUsageData = vi.fn();
+
+    await makeOpenAIResponsesRequest(
+      messages,
+      mockUpdateTextUI,
+      "gpt-4",
+      "test-token",
+      undefined, // tags
+      undefined, // signal
+      undefined, // onReasoningContent
+      undefined, // onTimingData
+      onUsageData,
+      undefined, // traceId
+      undefined, // vector_store_ids
+      undefined, // guardrails
+      undefined, // policies
+      undefined, // selectedMCPServers
+      undefined, // previousResponseId
+      undefined, // onResponseId
+      undefined, // onMCPEvent
+      undefined, // codeInterpreterEnabled
+      undefined, // onCodeInterpreterResult
+      undefined, // customBaseUrl
+      undefined, // mcpServers
+      undefined, // mcpServerToolRestrictions
+      undefined, // mcpToolsets
+      false, // streamingEnabled
+    );
+
+    expect(onUsageData).toHaveBeenCalledWith(
+      expect.objectContaining({ cacheReadTokens: 4695, servedFromResponseCache: true }),
+      "",
+    );
+  });
+
+  it("does not flag a non-streaming response that missed the response cache", async () => {
+    mockResponsesCreate.mockReturnValueOnce(
+      nonStreamingResponse({
+        id: "resp_fresh",
+        output: [{ type: "message", content: [{ type: "output_text", text: "Full answer" }] }],
+        usage: { output_tokens: 2, input_tokens: 5, total_tokens: 7 },
+      }),
+    );
+
+    const onUsageData = vi.fn();
+
+    await makeOpenAIResponsesRequest(
+      messages,
+      mockUpdateTextUI,
+      "gpt-4",
+      "test-token",
+      undefined, // tags
+      undefined, // signal
+      undefined, // onReasoningContent
+      undefined, // onTimingData
+      onUsageData,
+      undefined, // traceId
+      undefined, // vector_store_ids
+      undefined, // guardrails
+      undefined, // policies
+      undefined, // selectedMCPServers
+      undefined, // previousResponseId
+      undefined, // onResponseId
+      undefined, // onMCPEvent
+      undefined, // codeInterpreterEnabled
+      undefined, // onCodeInterpreterResult
+      undefined, // customBaseUrl
+      undefined, // mcpServers
+      undefined, // mcpServerToolRestrictions
+      undefined, // mcpToolsets
+      false, // streamingEnabled
+    );
+
+    expect(onUsageData).toHaveBeenCalledWith(expect.not.objectContaining({ servedFromResponseCache: true }), "");
+  });
+
+  it("never flags a streaming response, even when the proxy reports a cache key", async () => {
+    async function* mockStream() {
+      yield {
+        type: "response.completed",
+        response: { id: "resp_stream", usage: { output_tokens: 2, input_tokens: 5, total_tokens: 7 } },
+      };
+    }
+    mockResponsesCreate.mockResolvedValueOnce(mockStream());
+
+    const onUsageData = vi.fn();
+
+    await makeOpenAIResponsesRequest(
+      messages,
+      mockUpdateTextUI,
+      "gpt-4",
+      "test-token",
+      undefined,
+      undefined,
+      undefined,
+      undefined,
+      onUsageData,
+    );
+
+    expect(onUsageData).toHaveBeenCalledWith(expect.not.objectContaining({ servedFromResponseCache: true }), "");
+  });
+});
diff --git a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx
index 8d71a4e29a8..7ab76488504 100644
--- a/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx
+++ b/ui/litellm-dashboard/src/components/llm_calls/responses_api.tsx
@@ -116,6 +116,7 @@ export async function makeOpenAIResponsesRequest(
   try {
     const startTime = Date.now();
     let firstTokenReceived = false;
+    let servedFromResponseCache = false;
 
     // Format messages for the API
     const formattedInput = messages.map((message) => {
@@ -202,7 +203,15 @@ export async function makeOpenAIResponsesRequest(
 
     // Create request to OpenAI responses API
     // Use 'any' type to avoid TypeScript issues with the experimental API
-    const response = await (client as any).responses.create({ ...requestBody, stream: streamingEnabled }, { signal });
+    const response = streamingEnabled
+      ? await (client as any).responses.create({ ...requestBody, stream: true }, { signal })
+      : await (async () => {
+          const nonStreamingResponse = await (client as any).responses
+            .create({ ...requestBody, stream: false }, { signal })
+            .withResponse();
+          servedFromResponseCache = nonStreamingResponse.response.headers.get("x-litellm-cache-key") !== null;
+          return nonStreamingResponse.data;
+        })();
     const events = streamingEnabled ? response : responseAsEvents(response);
 
     let mcpToolUsed = "";
@@ -292,6 +301,7 @@ export async function makeOpenAIResponsesRequest(
               promptTokens: usage.input_tokens,
               totalTokens: usage.total_tokens,
               ...extractPromptCacheTokens(usage),
+              ...(servedFromResponseCache ? { servedFromResponseCache: true } : {}),
             };
 
             // Add reasoning tokens if available

From 5998c0d1d2ecba4b1405420b6dcbcccbfce4845c Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 13:32:26 -0700
Subject: [PATCH 050/105] fix(anthropic): carry tool_result document blocks
 through the /v1/messages responses bridge

---
 .../responses_adapters/transformation.py      |  54 +++++++-
 .../test_responses_adapters_transformation.py | 118 +++++++++++++++++-
 2 files changed, 170 insertions(+), 2 deletions(-)

diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py
index 6d47d0de19f..43c3e504964 100644
--- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py
+++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py
@@ -87,6 +87,51 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
             return source.get("url")
         return None
 
+    @staticmethod
+    def _translate_anthropic_document_block_to_file_part(
+        block: Mapping[str, object],
+    ) -> dict[str, str] | None:  # mutable-ok: API message payload
+        """Convert an Anthropic document block to a Responses input_file part."""
+        raw_source: Final = block.get("source")
+        if not isinstance(raw_source, Mapping):
+            return None
+        source: Final = cast(Mapping[str, object], raw_source)  # cast-ok: untrusted client payload
+        source_type: Final = source.get("type")
+        if source_type == "base64":
+            data: Final = source.get("data")
+            if not isinstance(data, str) or not data:
+                return None
+            raw_media_type: Final = source.get("media_type")
+            media_type: Final = (
+                raw_media_type if isinstance(raw_media_type, str) and raw_media_type else "application/pdf"
+            )
+            raw_title: Final = block.get("title")
+            filename: Final = raw_title if isinstance(raw_title, str) and raw_title else "document.pdf"
+            return {  # mutable-ok: API message payload
+                "type": "input_file",
+                "filename": filename,
+                "file_data": f"data:{media_type};base64,{data}",
+            }
+        if source_type == "url":
+            url: Final = source.get("url")
+            if not isinstance(url, str) or not url:
+                return None
+            return {"type": "input_file", "file_url": url}  # mutable-ok: API message payload
+        return None
+
+    @staticmethod
+    def _tool_result_output_value(
+        output_text: str,
+        file_parts: tuple[dict[str, str], ...],  # mutable-ok: json content parts
+    ) -> str | list[dict[str, str]]:  # mutable-ok: API message payload
+        """Plain string output, or a part list when document file parts are present."""
+        if not file_parts:
+            return output_text
+        text_parts: Final = (
+            [{"type": "input_text", "text": output_text}] if output_text else []  # mutable-ok: API message payload
+        )
+        return [*text_parts, *file_parts]  # mutable-ok: API message payload
+
     @staticmethod
     def _translate_midturn_system_content_to_responses(
         content: str | Iterable[AnthropicSystemMessageContent],
@@ -226,6 +271,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
                         elif btype == "tool_result":
                             tool_use_id = block.get("tool_use_id", "")
                             inner = block.get("content")
+                            tool_file_parts: tuple[dict[str, str], ...] = ()  # mutable-ok: json content parts
                             if inner is None:
                                 output_text = ""
                             elif isinstance(inner, str):
@@ -251,6 +297,12 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
                                         {"type": "input_image", "image_url": url}  # mutable-ok: json content part
                                         for url in image_urls
                                     )
+                                document_candidates = tuple(
+                                    self._translate_anthropic_document_block_to_file_part(c)
+                                    for c in inner
+                                    if isinstance(c, dict) and c.get("type") == "document"
+                                )
+                                tool_file_parts = tuple(part for part in document_candidates if part is not None)
                             else:
                                 output_text = str(inner)
                             # tool_result is a top-level item, not inside the message
@@ -258,7 +310,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
                                 {
                                     "type": "function_call_output",
                                     "call_id": tool_use_id,
-                                    "output": output_text,
+                                    "output": self._tool_result_output_value(output_text, tool_file_parts),
                                 }
                             )
                     if tool_image_parts:
diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py
index 8225e7cff39..f0cdaa4e8e3 100644
--- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py
+++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py
@@ -16,7 +16,10 @@ from litellm.constants import (
     DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET,
     DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
 )
-from litellm.litellm_core_utils.prompt_templates.common_utils import TOOL_RESULT_IMAGE_BOUNDARY
+from litellm.litellm_core_utils.prompt_templates.common_utils import (
+    TOOL_RESULT_IMAGE_BOUNDARY,
+    TOOL_RESULT_IMAGE_PLACEHOLDER,
+)
 from litellm.llms.anthropic.experimental_pass_through.responses_adapters.transformation import (
     LiteLLMAnthropicToResponsesAPIAdapter,
 )
@@ -1555,6 +1558,119 @@ class TestToolResultImages:
         assert self._input_images(items) == []
 
 
+class TestToolResultDocuments:
+    """Documents inside tool_result blocks must survive translation (LIT-6135):
+    the function_call_output output becomes a list of parts carrying the joined
+    text as input_text and each document as an input_file. Without documents the
+    output stays the plain string it always was."""
+
+    PDF_B64 = "JVBERi0xLjQKJSBQT05H"
+    PDF_DATA_URI = "data:application/pdf;base64,JVBERi0xLjQKJSBQT05H"
+    PDF_URL = "https://example.com/report.pdf"
+    PNG_B64 = "iVBORw0KGgoAAAANSUhEUg=="
+
+    def _messages(self, tool_result_content):
+        return [
+            {"role": "user", "content": "read the pdf"},
+            {
+                "role": "assistant",
+                "content": [{"type": "tool_use", "id": "toolu_01", "name": "read", "input": {}}],
+            },
+            {
+                "role": "user",
+                "content": [
+                    {"type": "tool_result", "tool_use_id": "toolu_01", "content": tool_result_content}
+                ],
+            },
+        ]
+
+    def _translate(self, tool_result_content):
+        return _ADAPTER.translate_messages_to_responses_input(self._messages(tool_result_content))
+
+    @staticmethod
+    def _tool_output(items):
+        return next(item for item in items if item.get("type") == "function_call_output")["output"]
+
+    def _base64_document(self, **extra):
+        return {
+            "type": "document",
+            "source": {"type": "base64", "media_type": "application/pdf", "data": self.PDF_B64},
+            **extra,
+        }
+
+    def test_text_and_base64_document_produce_part_list(self):
+        output = self._tool_output(
+            self._translate([{"type": "text", "text": "PDF file read: mystery.pdf"}, self._base64_document()])
+        )
+        assert output == [
+            {"type": "input_text", "text": "PDF file read: mystery.pdf"},
+            {"type": "input_file", "filename": "document.pdf", "file_data": self.PDF_DATA_URI},
+        ]
+
+    def test_document_only_produces_single_file_part(self):
+        output = self._tool_output(self._translate([self._base64_document()]))
+        assert output == [{"type": "input_file", "filename": "document.pdf", "file_data": self.PDF_DATA_URI}]
+
+    def test_document_title_becomes_filename(self):
+        output = self._tool_output(self._translate([self._base64_document(title="quarterly-report.pdf")]))
+        assert output == [
+            {"type": "input_file", "filename": "quarterly-report.pdf", "file_data": self.PDF_DATA_URI}
+        ]
+
+    def test_url_document_becomes_file_url_part(self):
+        output = self._tool_output(
+            self._translate([{"type": "document", "source": {"type": "url", "url": self.PDF_URL}}])
+        )
+        assert output == [{"type": "input_file", "file_url": self.PDF_URL}]
+
+    def test_document_with_empty_data_falls_back_to_string_output(self):
+        output = self._tool_output(
+            self._translate(
+                [
+                    {"type": "text", "text": "PDF file read"},
+                    {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": ""}},
+                ]
+            )
+        )
+        assert output == "PDF file read"
+
+    def test_document_without_source_dict_keeps_string_output(self):
+        output = self._tool_output(
+            self._translate([{"type": "text", "text": "stub"}, {"type": "document", "source": self.PDF_URL}])
+        )
+        assert output == "stub"
+
+    def test_text_only_tool_result_keeps_plain_string_output(self):
+        output = self._tool_output(self._translate([{"type": "text", "text": "plain result"}]))
+        assert output == "plain result"
+
+    def test_text_image_and_document_mix(self):
+        items = self._translate(
+            [
+                {"type": "text", "text": "captured"},
+                {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": self.PNG_B64}},
+                self._base64_document(),
+            ]
+        )
+
+        output = self._tool_output(items)
+        assert output == [
+            {"type": "input_text", "text": f"captured\n{TOOL_RESULT_IMAGE_PLACEHOLDER}"},
+            {"type": "input_file", "filename": "document.pdf", "file_data": self.PDF_DATA_URI},
+        ]
+
+        image_message = next(
+            item
+            for item in items
+            if item.get("type") == "message"
+            and any(part.get("type") == "input_image" for part in item.get("content", []))
+        )
+        assert image_message["content"] == [
+            {"type": "input_text", "text": TOOL_RESULT_IMAGE_BOUNDARY},
+            {"type": "input_image", "image_url": f"data:image/png;base64,{self.PNG_B64}"},
+        ]
+
+
 def _contains_key(value, key) -> bool:
     if isinstance(value, dict):
         return key in value or any(_contains_key(v, key) for v in value.values())

From 6d0cc1423ecb0e5ecc66da4553ab0e1c2a1c02f6 Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 25 Aug 2026 13:37:45 -0700
Subject: [PATCH 051/105] fix(scim): return user_id as Group members[].value on
 transformed group responses (#38161)

* fix(scim): return user_id as Group members[].value on transformed group responses

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* style: format scim transformation tests per ruff

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test: drop redundant assertion comment

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* chore: retrigger ci

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../scim/scim_transformations.py              |  11 +-
 .../scim/test_scim_transformations.py         | 117 +++++-------------
 2 files changed, 32 insertions(+), 96 deletions(-)

diff --git a/litellm/proxy/management_endpoints/scim/scim_transformations.py b/litellm/proxy/management_endpoints/scim/scim_transformations.py
index 2d95d0bea29..496be05b4b0 100644
--- a/litellm/proxy/management_endpoints/scim/scim_transformations.py
+++ b/litellm/proxy/management_endpoints/scim/scim_transformations.py
@@ -198,15 +198,8 @@ class ScimTransformations:
 
     @staticmethod
     def _get_scim_member_value(member: Member) -> str:
-        """
-        Get the SCIM member value. Use user_email if available, otherwise use user_id.
-        SCIM member value should be the unique identifier for the user.
-        """
-        if hasattr(member, "user_email") and member.user_email:
-            return member.user_email
-        elif hasattr(member, "user_id"):
-            return member.user_id or ScimTransformations.DEFAULT_SCIM_MEMBER_VALUE
-        return ScimTransformations.DEFAULT_SCIM_MEMBER_VALUE
+        """The member's SCIM resource id, which LiteLLM serves as user_id (RFC 7643 §8.7.1)."""
+        return member.user_id or ScimTransformations.DEFAULT_SCIM_MEMBER_VALUE
 
     @staticmethod
     def _get_scim_member_display(member: Member) -> str:
diff --git a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_transformations.py b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_transformations.py
index 9853ce7e1cf..135175dd29d 100644
--- a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_transformations.py
+++ b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_transformations.py
@@ -2,7 +2,6 @@ from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
 
-
 from litellm.proxy._types import LiteLLM_TeamTable, LiteLLM_UserTable, Member
 from litellm.proxy.management_endpoints.scim.scim_transformations import (
     ScimTransformations,
@@ -95,25 +94,17 @@ def mock_prisma_client():
 
 class TestScimTransformations:
     @pytest.mark.asyncio
-    async def test_transform_litellm_user_to_scim_user(
-        self, mock_user, mock_prisma_client
-    ):
+    async def test_transform_litellm_user_to_scim_user(self, mock_user, mock_prisma_client):
         mock_client, mock_find_unique = mock_prisma_client
 
         # Mock the team lookup
-        team1 = LiteLLM_TeamTable(
-            team_id="team-1", team_alias="Team One", members_with_roles=[]
-        )
-        team2 = LiteLLM_TeamTable(
-            team_id="team-2", team_alias="Team Two", members_with_roles=[]
-        )
+        team1 = LiteLLM_TeamTable(team_id="team-1", team_alias="Team One", members_with_roles=[])
+        team2 = LiteLLM_TeamTable(team_id="team-2", team_alias="Team Two", members_with_roles=[])
 
         mock_find_unique.side_effect = [team1, team2]
 
         with patch("litellm.proxy.proxy_server.prisma_client", mock_client):
-            scim_user = await ScimTransformations.transform_litellm_user_to_scim_user(
-                mock_user
-            )
+            scim_user = await ScimTransformations.transform_litellm_user_to_scim_user(mock_user)
 
             assert scim_user.id == mock_user.user_id
             assert scim_user.userName == mock_user.user_email
@@ -129,21 +120,15 @@ class TestScimTransformations:
             assert scim_user.groups[1].display == "Team Two"
 
     @pytest.mark.asyncio
-    async def test_transform_user_with_scim_metadata(
-        self, mock_user_with_scim_metadata, mock_prisma_client
-    ):
+    async def test_transform_user_with_scim_metadata(self, mock_user_with_scim_metadata, mock_prisma_client):
         mock_client, mock_find_unique = mock_prisma_client
 
         # Mock the team lookup
-        team1 = LiteLLM_TeamTable(
-            team_id="team-1", team_alias="Team One", members_with_roles=[]
-        )
+        team1 = LiteLLM_TeamTable(team_id="team-1", team_alias="Team One", members_with_roles=[])
         mock_find_unique.return_value = team1
 
         with patch("litellm.proxy.proxy_server.prisma_client", mock_client):
-            scim_user = await ScimTransformations.transform_litellm_user_to_scim_user(
-                mock_user_with_scim_metadata
-            )
+            scim_user = await ScimTransformations.transform_litellm_user_to_scim_user(mock_user_with_scim_metadata)
 
             assert scim_user.name.givenName == "Test"
             assert scim_user.name.familyName == "User"
@@ -160,15 +145,11 @@ class TestScimTransformations:
             teams=[],
             created_at=None,
             updated_at=None,
-            metadata={
-                "scim_enterprise": {"costCenter": "CC-42", "department": "Platform"}
-            },
+            metadata={"scim_enterprise": {"costCenter": "CC-42", "department": "Platform"}},
         )
 
         with patch("litellm.proxy.proxy_server.prisma_client", mock_client):
-            scim_user = await ScimTransformations.transform_litellm_user_to_scim_user(
-                user
-            )
+            scim_user = await ScimTransformations.transform_litellm_user_to_scim_user(user)
 
             assert scim_user.enterprise_user is not None
             assert scim_user.enterprise_user.costCenter == "CC-42"
@@ -176,9 +157,7 @@ class TestScimTransformations:
             assert SCIM_ENTERPRISE_USER_SCHEMA in scim_user.schemas
 
     @pytest.mark.asyncio
-    async def test_transform_user_with_entitlements_and_roles_metadata(
-        self, mock_prisma_client
-    ):
+    async def test_transform_user_with_entitlements_and_roles_metadata(self, mock_prisma_client):
         mock_client, mock_find_unique = mock_prisma_client
         mock_find_unique.return_value = None
 
@@ -190,17 +169,13 @@ class TestScimTransformations:
             created_at=None,
             updated_at=None,
             metadata={
-                "scim_entitlements": [
-                    {"value": "jira-software", "display": "Jira Software"}
-                ],
+                "scim_entitlements": [{"value": "jira-software", "display": "Jira Software"}],
                 "scim_roles": [{"value": "engineering-admin", "primary": True}],
             },
         )
 
         with patch("litellm.proxy.proxy_server.prisma_client", mock_client):
-            scim_user = await ScimTransformations.transform_litellm_user_to_scim_user(
-                user
-            )
+            scim_user = await ScimTransformations.transform_litellm_user_to_scim_user(user)
 
             assert scim_user.entitlements is not None
             assert scim_user.entitlements[0].value == "jira-software"
@@ -210,9 +185,7 @@ class TestScimTransformations:
             assert scim_user.roles[0].primary is True
 
     @pytest.mark.asyncio
-    async def test_transform_user_with_malformed_directory_metadata_fails_soft(
-        self, mock_prisma_client
-    ):
+    async def test_transform_user_with_malformed_directory_metadata_fails_soft(self, mock_prisma_client):
         """Metadata is writable outside the SCIM surface; a corrupted value on one
         user must omit the attribute, not fail the whole directory response"""
         mock_client, mock_find_unique = mock_prisma_client
@@ -233,9 +206,7 @@ class TestScimTransformations:
         )
 
         with patch("litellm.proxy.proxy_server.prisma_client", mock_client):
-            scim_user = await ScimTransformations.transform_litellm_user_to_scim_user(
-                user
-            )
+            scim_user = await ScimTransformations.transform_litellm_user_to_scim_user(user)
 
             assert scim_user.id == "user-corrupt"
             assert scim_user.entitlements is None
@@ -244,22 +215,14 @@ class TestScimTransformations:
             assert SCIM_ENTERPRISE_USER_SCHEMA not in scim_user.schemas
 
     @pytest.mark.asyncio
-    async def test_transform_user_without_enterprise_metadata_omits_schema(
-        self, mock_user, mock_prisma_client
-    ):
+    async def test_transform_user_without_enterprise_metadata_omits_schema(self, mock_user, mock_prisma_client):
         mock_client, mock_find_unique = mock_prisma_client
-        team1 = LiteLLM_TeamTable(
-            team_id="team-1", team_alias="Team One", members_with_roles=[]
-        )
-        team2 = LiteLLM_TeamTable(
-            team_id="team-2", team_alias="Team Two", members_with_roles=[]
-        )
+        team1 = LiteLLM_TeamTable(team_id="team-1", team_alias="Team One", members_with_roles=[])
+        team2 = LiteLLM_TeamTable(team_id="team-2", team_alias="Team Two", members_with_roles=[])
         mock_find_unique.side_effect = [team1, team2]
 
         with patch("litellm.proxy.proxy_server.prisma_client", mock_client):
-            scim_user = await ScimTransformations.transform_litellm_user_to_scim_user(
-                mock_user
-            )
+            scim_user = await ScimTransformations.transform_litellm_user_to_scim_user(mock_user)
 
             assert scim_user.enterprise_user is None
             assert SCIM_ENTERPRISE_USER_SCHEMA not in scim_user.schemas
@@ -309,36 +272,28 @@ class TestScimTransformations:
         assert dumped_attrs["roles"][0]["value"] == "engineering-admin"
 
     @pytest.mark.asyncio
-    async def test_transform_litellm_team_to_scim_group(
-        self, mock_team, mock_prisma_client
-    ):
+    async def test_transform_litellm_team_to_scim_group(self, mock_team, mock_prisma_client):
         mock_client, _ = mock_prisma_client
 
         with patch("litellm.proxy.proxy_server.prisma_client", mock_client):
-            scim_group = await ScimTransformations.transform_litellm_team_to_scim_group(
-                mock_team
-            )
+            scim_group = await ScimTransformations.transform_litellm_team_to_scim_group(mock_team)
 
             assert scim_group.id == mock_team.team_id
             assert scim_group.displayName == mock_team.team_alias
             assert len(scim_group.members) == 2
-            assert scim_group.members[0].value == "test@example.com"
+            assert scim_group.members[0].value == "user-123"
             assert scim_group.members[0].display == "test@example.com"
-            assert scim_group.members[1].value == "test2@example.com"
+            assert scim_group.members[1].value == "user-456"
             assert scim_group.members[1].display == "test2@example.com"
 
     @pytest.mark.asyncio
-    async def test_transform_team_marks_members_as_users(
-        self, mock_team, mock_prisma_client
-    ):
+    async def test_transform_team_marks_members_as_users(self, mock_team, mock_prisma_client):
         """A LiteLLM team only holds users, and stating the member type keeps the
         response from emitting a null ``type`` now that SCIMMember carries one."""
         mock_client, _ = mock_prisma_client
 
         with patch("litellm.proxy.proxy_server.prisma_client", mock_client):
-            scim_group = await ScimTransformations.transform_litellm_team_to_scim_group(
-                mock_team
-            )
+            scim_group = await ScimTransformations.transform_litellm_team_to_scim_group(mock_team)
 
         assert [member.type for member in scim_group.members] == ["User", "User"]
 
@@ -351,9 +306,7 @@ class TestScimTransformations:
         result = ScimTransformations._get_scim_user_name(mock_user_minimal)
         assert result == ScimTransformations.DEFAULT_SCIM_DISPLAY_NAME
 
-    def test_get_scim_family_name(
-        self, mock_user, mock_user_with_scim_metadata, mock_user_minimal
-    ):
+    def test_get_scim_family_name(self, mock_user, mock_user_with_scim_metadata, mock_user_minimal):
         # User with alias
         result = ScimTransformations._get_scim_family_name(mock_user)
         assert result == mock_user.user_alias
@@ -366,9 +319,7 @@ class TestScimTransformations:
         result = ScimTransformations._get_scim_family_name(mock_user_minimal)
         assert result == ScimTransformations.DEFAULT_SCIM_FAMILY_NAME
 
-    def test_get_scim_given_name(
-        self, mock_user, mock_user_with_scim_metadata, mock_user_minimal
-    ):
+    def test_get_scim_given_name(self, mock_user, mock_user_with_scim_metadata, mock_user_minimal):
         # User with alias
         result = ScimTransformations._get_scim_given_name(mock_user)
         assert result == mock_user.user_alias
@@ -382,14 +333,10 @@ class TestScimTransformations:
         assert result == ScimTransformations.DEFAULT_SCIM_NAME
 
     def test_get_scim_member_value(self):
-        # Member with email
-        member_with_email = Member(
-            user_id="user-123", user_email="test@example.com", role="admin"
-        )
+        member_with_email = Member(user_id="user-123", user_email="test@example.com", role="admin")
         result = ScimTransformations._get_scim_member_value(member_with_email)
-        assert result == member_with_email.user_email
+        assert result == member_with_email.user_id
 
-        # Member without email should fall back to user_id
         member_without_email = Member(user_id="user-456", user_email=None, role="user")
         result = ScimTransformations._get_scim_member_value(member_without_email)
         assert result == member_without_email.user_id
@@ -415,9 +362,7 @@ class TestScimTransformations:
         mock_find_unique.return_value = None
 
         with patch("litellm.proxy.proxy_server.prisma_client", mock_client):
-            scim_user = await ScimTransformations.transform_litellm_user_to_scim_user(
-                user_with_uuid_email
-            )
+            scim_user = await ScimTransformations.transform_litellm_user_to_scim_user(user_with_uuid_email)
 
             assert scim_user.id == user_with_uuid_email.user_id
             assert scim_user.emails is None or len(scim_user.emails) == 0
@@ -443,9 +388,7 @@ class TestScimTransformations:
         mock_find_unique.return_value = None
 
         with patch("litellm.proxy.proxy_server.prisma_client", mock_client):
-            scim_user = await ScimTransformations.transform_litellm_user_to_scim_user(
-                user_with_none_email
-            )
+            scim_user = await ScimTransformations.transform_litellm_user_to_scim_user(user_with_none_email)
 
             assert scim_user.id == user_with_none_email.user_id
             assert scim_user.emails is None or len(scim_user.emails) == 0

From c96245d7bfc6c9446fa04e1ab31e824843f4259b Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 25 Aug 2026 13:39:34 -0700
Subject: [PATCH 052/105] fix(scim): preserve existing team memberships when
 POST /Users adoption carries no groups (#38166)

Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../management_endpoints/scim/scim_v2.py      |  7 +-
 .../scim/test_scim_v2_endpoints.py            | 73 ++++++++++++++++++-
 2 files changed, 76 insertions(+), 4 deletions(-)

diff --git a/litellm/proxy/management_endpoints/scim/scim_v2.py b/litellm/proxy/management_endpoints/scim/scim_v2.py
index 7183e6cb402..6658963d024 100644
--- a/litellm/proxy/management_endpoints/scim/scim_v2.py
+++ b/litellm/proxy/management_endpoints/scim/scim_v2.py
@@ -176,6 +176,10 @@ class UserProvisionerHelpers:
         is persisted too, so re-upserting an existing email demotes a user who is no
         longer in the admin group instead of leaving the stale role.
 
+        IdPs like Entra manage membership exclusively through /Groups and never send
+        ``groups`` on POST /Users, so a request without teams means "unspecified",
+        not "remove from every team": existing memberships are preserved then.
+
         Args:
             prisma_client: Database client
             new_user_request: New user request data
@@ -194,7 +198,8 @@ class UserProvisionerHelpers:
         if not existing_user:
             return None
 
-        new_teams: Final = list(dict.fromkeys(new_user_request.teams or []))
+        requested_teams: Final = list(dict.fromkeys(new_user_request.teams or []))
+        new_teams: Final = requested_teams if requested_teams else list(existing_user.teams or [])
 
         if new_user_request.user_id != existing_user.user_id:
             verbose_proxy_logger.info(
diff --git a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py
index 5f6c1a2375b..d51e3abde83 100644
--- a/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/scim/test_scim_v2_endpoints.py
@@ -752,6 +752,63 @@ async def test_handle_existing_user_by_email_syncs_roster_and_dedups_teams(mocke
     assert update_calls[0].kwargs["data"]["teams"] == ["team-a", "team-b"]
 
 
+@pytest.mark.asyncio
+async def test_handle_existing_user_by_email_without_teams_preserves_memberships(mocker):
+    """Adoption via POST /Users without ``groups`` must keep the user's existing teams.
+
+    Regression: Entra manages membership exclusively through /Groups and never sends
+    ``groups`` on POST /Users, so the empty team list was treated as the desired
+    state and the adopted user was removed from every team roster and had ``teams``
+    overwritten with [].
+    """
+    existing_user = mocker.MagicMock()
+    existing_user.user_id = "adopted-id"
+    existing_user.user_email = "member@example.com"
+    existing_user.user_alias = "Member"
+    existing_user.teams = ["team-a", "team-b"]
+    existing_user.metadata = {}
+
+    mock_prisma_client = mocker.MagicMock()
+    mock_prisma_client.db = mocker.MagicMock()
+    mock_prisma_client.db.litellm_usertable = mocker.MagicMock()
+    mock_prisma_client.db.litellm_usertable.find_first = AsyncMock(return_value=existing_user)
+    mock_prisma_client.db.litellm_usertable.update = AsyncMock(return_value={})
+
+    mocker.patch(  # test-quality-ok: roster helpers are module-level, not injectable into the helper
+        "litellm.proxy.management_endpoints.scim.scim_v2.ScimTransformations.transform_litellm_user_to_scim_user",
+        AsyncMock(return_value=None),
+    )
+    mock_team_member_add = mocker.patch(  # test-quality-ok: roster helpers are module-level, not injectable into the helper
+        "litellm.proxy.management_endpoints.scim.scim_v2.team_member_add",
+        AsyncMock(),
+    )
+    mock_team_member_delete = mocker.patch(  # test-quality-ok: roster helpers are module-level, not injectable into the helper
+        "litellm.proxy.management_endpoints.scim.scim_v2.team_member_delete",
+        AsyncMock(),
+    )
+
+    new_user_request = NewUserRequest(
+        user_id="entra-object-id",
+        user_email="member@example.com",
+        user_alias="Member",
+        teams=[],
+        metadata={},
+        auto_create_key=False,
+    )
+
+    await UserProvisionerHelpers.handle_existing_user_by_email(
+        prisma_client=mock_prisma_client, new_user_request=new_user_request
+    )
+
+    mock_team_member_add.assert_not_awaited()
+    mock_team_member_delete.assert_not_awaited()
+
+    update_calls = mock_prisma_client.db.litellm_usertable.update.call_args_list
+    assert len(update_calls) == 1
+    assert update_calls[0].kwargs["where"] == {"user_id": "adopted-id"}
+    assert update_calls[0].kwargs["data"]["teams"] == ["team-a", "team-b"]
+
+
 @pytest.mark.asyncio
 async def test_handle_existing_user_by_email_roster_add_failure_blocks_teams_write(mocker):
     """A genuine roster add failure must propagate and must not persist the teams array.
@@ -872,11 +929,16 @@ async def test_handle_existing_user_by_email_roster_remove_failure_blocks_teams_
         AsyncMock(side_effect=HTTPException(status_code=500, detail={"error": "No db connected"})),
     )
 
+    mocker.patch(  # test-quality-ok: roster helpers are module-level, not injectable into the helper
+        "litellm.proxy.management_endpoints.scim.scim_v2.team_member_add",
+        AsyncMock(),
+    )
+
     new_user_request = NewUserRequest(
         user_id="uid",
         user_email="member@example.com",
         user_alias="Member",
-        teams=[],
+        teams=["replacement-team"],
         metadata={},
         auto_create_key=False,
     )
@@ -917,11 +979,16 @@ async def test_handle_existing_user_by_email_roster_remove_already_absent_is_noo
         AsyncMock(return_value=None),
     )
 
+    mocker.patch(  # test-quality-ok: roster helpers are module-level, not injectable into the helper
+        "litellm.proxy.management_endpoints.scim.scim_v2.team_member_add",
+        AsyncMock(),
+    )
+
     new_user_request = NewUserRequest(
         user_id="uid",
         user_email="member@example.com",
         user_alias="Member",
-        teams=[],
+        teams=["replacement-team"],
         metadata={},
         auto_create_key=False,
     )
@@ -933,7 +1000,7 @@ async def test_handle_existing_user_by_email_roster_remove_already_absent_is_noo
     mock_team_member_delete.assert_awaited_once()
     update_calls = mock_prisma_client.db.litellm_usertable.update.call_args_list
     assert len(update_calls) == 1
-    assert update_calls[0].kwargs["data"]["teams"] == []
+    assert update_calls[0].kwargs["data"]["teams"] == ["replacement-team"]
 
 
 @pytest.mark.asyncio

From c70b911122fd8d50dde4a6af525449afeff87aac Mon Sep 17 00:00:00 2001
From: Yassin Kortam <yassin@berri.ai>
Date: Tue, 25 Aug 2026 13:40:46 -0700
Subject: [PATCH 053/105] fix(router): support mid-stream fallback for
 anthropic_messages route type (#38153)

anthropic_messages goes through _ageneric_api_call_with_fallbacks rather
than _acompletion, so its returned streaming iterator was never wrapped
by the chat-completions fallback handler. A retriable SSE event: error
frame (overloaded_error, internal_server_error) from a native
Anthropic/Bedrock passthrough passed through to the client unchanged,
and a MidStreamFallbackError raised by the completion-bridge path's
CustomStreamWrapper propagated unhandled.

Add _aanthropic_messages_streaming_iterator, mirroring
_acompletion_streaming_iterator: it detects a retriable SSE error event
via the new parse_anthropic_error_event helper, raises
MidStreamFallbackError once real generated content (a content_block_delta
frame) has not yet reached the caller, and re-enters the Router's
fallback chain. A MidStreamFallbackError raised directly by the source
iterator (the completion-bridge path) is gated the same way via its own
is_pre_first_chunk flag. The raised MidStreamFallbackError carries a
status-coded original_exception built from the parsed error type, so
status_code/cooldown logic sees the real 429/500/503/etc. instead of a
hardcoded 503.

Lifecycle/bookkeeping frames (message_start, content_block_start, ping,
...) never disqualify a fallback attempt by themselves, since Anthropic
routinely sends message_start before an overload error - but they are
buffered rather than forwarded immediately, since forwarding one and
then appending a fallback attempt's own message_start would produce two
overlapping message lifecycles on one SSE stream. Buffered frames flush,
in order, once real content arrives or the stream ends without error.
Once real content has streamed, or the error is a non-retriable 4xx, the
chunk (or exception) is forwarded as-is rather than starting a second
lifecycle. Content and error coalesced into a single physical read are
handled the same way: once the client has genuinely received the content
(bundled in that same forwarded chunk), no fallback is attempted. A
`ping` keepalive is dropped outright before any real content arrives
(it recurs indefinitely on a slow-starting connection and carries
nothing worth buffering), and the pre-content lifecycle buffer is capped
at MAX_BUFFERED_PRE_CONTENT_ANTHROPIC_CHUNKS, forcing an early commit to
the primary stream so a hostile or pathological upstream can't grow it
without bound. is_anthropic_ping_chunk only matches a chunk whose every
event: line is event: ping, so a ping coalesced with real content or a
retriable error into one physical transport chunk is never dropped.

The fallback request kwargs also deep-copy nested litellm_metadata/metadata
(matching the Responses API path) so the primary attempt's
deployment-specific fields never leak into the fallback request, and the
fallback deployment's own provider headers are merged onto the wrapper's
_hidden_params so they still reach the client/logging pipeline. A
fallback that resolves to a non-streaming response (e.g. an agentic
tool-use interception loop) is synthesized into a real Anthropic SSE
event sequence via the new anthropic_messages_response_as_sse_events
helper, instead of yielding a raw dict into the byte stream - including
a trailing signature_delta for a thinking block, and a message_start
whose stop_reason/stop_sequence/output_tokens stay null/zero the way a
real stream's does instead of leaking the completed response's final
state.

Resolves #24004
---
 litellm/llms/anthropic/common_utils.py        |   15 +
 .../messages/streaming_iterator.py            |  235 +++-
 litellm/llms/anthropic/files/handler.py       |   14 +-
 litellm/router.py                             |  363 +++++-
 .../messages/test_streaming_iterator.py       |  208 ++++
 tests/test_litellm/test_router.py             | 1101 +++++++++++++++++
 6 files changed, 1911 insertions(+), 25 deletions(-)

diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py
index 9461e40cf2e..53cc3464761 100644
--- a/litellm/llms/anthropic/common_utils.py
+++ b/litellm/llms/anthropic/common_utils.py
@@ -38,6 +38,21 @@ DROP_DISABLED_THINKING_WARNING: Final = (
     "thinking blocks, and those thinking tokens are billed as output tokens."
 )
 
+# Anthropic error `type` (both the JSON error body and SSE `event: error`
+# payloads use this field) mapped to the HTTP status code it corresponds to.
+ANTHROPIC_ERROR_STATUS_CODE_MAP: Final = MappingProxyType(
+    {
+        "invalid_request_error": 400,
+        "authentication_error": 401,
+        "permission_error": 403,
+        "not_found_error": 404,
+        "rate_limit_error": 429,
+        "api_error": 500,
+        "overloaded_error": 503,
+        "timeout_error": 504,
+    }
+)
+
 _BEDROCK_VERSION_SUFFIX_RE: Final = re.compile(r"-v\d+(?::\d+)?$")
 _INFERENCE_PROFILE_MINOR_RE: Final = re.compile(r":\d+$")
 _DATED_RELEASE_SUFFIX_RE: Final = re.compile(r"-\d{8}$")
diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py
index 922769dbbfd..0a12bc3135f 100644
--- a/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py
+++ b/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py
@@ -1,6 +1,6 @@
 import asyncio
 import json
-from collections.abc import AsyncIterator
+from collections.abc import AsyncIterator, Mapping
 from datetime import datetime
 from typing import Any, Final, Protocol, runtime_checkable
 
@@ -11,9 +11,11 @@ from typing_extensions import TypedDict
 from litellm.litellm_core_utils.core_helpers import process_response_headers
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
+from litellm.llms.anthropic.common_utils import ANTHROPIC_ERROR_STATUS_CODE_MAP
 from litellm.proxy.pass_through_endpoints.success_handler import (
     PassThroughEndpointLogging,
 )
+from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse
 from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType
 from litellm.types.utils import GenericStreamingChunk, ModelResponseStream
 
@@ -33,26 +35,239 @@ def _is_message_stop_chunk(chunk: object) -> bool:
     return False
 
 
-def _is_provider_error_chunk(chunk: object) -> bool:
+def is_anthropic_ping_chunk(chunk: object) -> bool:
+    """
+    Whether a chunk is a pure ``ping`` keepalive frame. It carries no content
+    and can recur indefinitely on a slow-starting or idle connection, so a
+    mid-stream fallback wrapper drops it outright while still deciding
+    whether to commit to the primary stream, rather than buffering it.
+
+    A physical transport chunk that coalesces a ping with any other SSE
+    event (``message_start``, ``content_block_delta``, ``event: error``, ...)
+    is NOT a pure ping - dropping it whole would discard those events - so
+    only a chunk whose every ``event:`` line is ``event: ping`` qualifies.
+    """
     if isinstance(chunk, dict):
-        return chunk.get("type") == "error"
+        return chunk.get("type") == "ping"
     if isinstance(chunk, (bytes, bytearray)):
-        return any(line == b"event: error" for line in chunk.splitlines())
+        event_lines: Final = tuple(line for line in chunk.splitlines() if line.startswith(b"event:"))
+        return bool(event_lines) and all(line == b"event: ping" for line in event_lines)
     return False
 
 
+def is_anthropic_content_delta_chunk(chunk: object) -> bool:
+    """
+    Whether a chunk carries actual assistant-generated output (a
+    ``content_block_delta`` frame), as opposed to a lifecycle/bookkeeping
+    frame (``message_start``, ``content_block_start``/``stop``,
+    ``message_delta``, ``message_stop``, ``ping``) that carries nothing
+    worth preserving before an invisible mid-stream fallback retry.
+    """
+    if isinstance(chunk, dict):
+        return chunk.get("type") == "content_block_delta"
+    if isinstance(chunk, (bytes, bytearray)):
+        return any(line == b"event: content_block_delta" for line in chunk.splitlines())
+    return False
+
+
+def _decoded_sse_data_line(line: bytes) -> object | None:
+    if not line.startswith(b"data:"):
+        return None
+    try:
+        return json.loads(line[len(b"data:") :].strip())
+    except (ValueError, TypeError):
+        return None
+
+
+def _anthropic_error_event_payload(chunk: object) -> Mapping[str, object] | None:
+    if isinstance(chunk, dict):
+        return chunk if chunk.get("type") == "error" else None
+    if isinstance(chunk, (bytes, bytearray)):
+        decoded_lines: Final = (_decoded_sse_data_line(line) for line in chunk.splitlines())
+        return next(
+            (
+                candidate
+                for candidate in decoded_lines
+                if isinstance(candidate, dict) and candidate.get("type") == "error"
+            ),
+            None,
+        )
+    return None
+
+
+def _anthropic_error_body(chunk: object) -> Mapping[str, object] | None:
+    """Return the ``error`` object of an Anthropic SSE ``event: error`` chunk, or None."""
+    payload: Final = _anthropic_error_event_payload(chunk)
+    error_body: Final = payload.get("error") if payload is not None else None
+    return error_body if isinstance(error_body, dict) else None
+
+
+def _is_provider_error_chunk(chunk: object) -> bool:
+    return _anthropic_error_body(chunk) is not None
+
+
+def parse_anthropic_error_event(chunk: object) -> tuple[str, str, int] | None:
+    """
+    Extract ``(error_type, message, http_status_code)`` from an Anthropic SSE
+    ``event: error`` chunk (raw bytes or an already-decoded dict), or None if
+    ``chunk`` is not an error event.
+
+    The status code is looked up via ANTHROPIC_ERROR_STATUS_CODE_MAP,
+    defaulting to 500 for an error ``type`` Anthropic hasn't documented yet.
+    """
+    error_body: Final = _anthropic_error_body(chunk)
+    if error_body is None:
+        return None
+    error_type: Final = error_body.get("type")
+    if not isinstance(error_type, str):
+        return None
+    message: Final = error_body.get("message")
+    return (
+        error_type,
+        message if isinstance(message, str) else error_type,
+        ANTHROPIC_ERROR_STATUS_CODE_MAP.get(error_type, 500),
+    )
+
+
 def _is_terminal_stream_chunk(chunk: object) -> bool:
     return _is_message_stop_chunk(chunk) or _is_provider_error_chunk(chunk)
 
 
+def _sse_event(event_type: str, payload: Mapping[str, object]) -> bytes:
+    return f"event: {event_type}\ndata: {json.dumps(payload)}\n\n".encode()
+
+
 def _incomplete_stream_error_sse_event() -> bytes:
-    payload: Final = json.dumps(
-        {
-            "type": "error",
-            "error": {"type": "api_error", "message": INCOMPLETE_STREAM_ERROR_MESSAGE},
-        }
+    return _sse_event(  # mutable-ok: one-shot JSON payload, never mutated after construction
+        "error",
+        {"type": "error", "error": {"type": "api_error", "message": INCOMPLETE_STREAM_ERROR_MESSAGE}},
+    )
+
+
+def _anthropic_content_block_start_and_deltas(
+    block: Mapping[str, object],
+) -> tuple[Mapping[str, object], tuple[Mapping[str, object], ...]]:
+    """
+    ``(content_block_start.content_block, content_block_delta.delta events)``
+    for one Anthropic response content block. A thinking block emits both a
+    thinking_delta and a trailing signature_delta - a real Anthropic stream
+    does the same, and dropping the signature makes any replay of that
+    assistant message (a follow-up turn, a tool-use continuation) fail
+    Anthropic's thinking-signature verification. redacted_thinking has no
+    delta at all - it is sent complete in content_block_start.
+    """
+    match block.get("type"):
+        case "tool_use":
+            return (
+                {  # mutable-ok: one-shot payload
+                    "id": block.get("id"),
+                    "name": block.get("name"),
+                    "input": {},  # mutable-ok: one-shot payload
+                    "type": "tool_use",
+                },
+                (
+                    {  # mutable-ok: one-shot payload
+                        "partial_json": json.dumps(block.get("input") or {}),  # mutable-ok: one-shot payload
+                        "type": "input_json_delta",
+                    },
+                ),
+            )
+        case "thinking":
+            signature: Final = block.get("signature")
+            signature_deltas: Final = (
+                ({"signature": signature, "type": "signature_delta"},)  # mutable-ok: one-shot payload
+                if isinstance(signature, str) and signature
+                else ()
+            )
+            return (
+                {"thinking": "", "signature": "", "type": "thinking"},  # mutable-ok: one-shot payload
+                (
+                    {"thinking": block.get("thinking") or "", "type": "thinking_delta"},  # mutable-ok: one-shot payload
+                    *signature_deltas,
+                ),
+            )
+        case "redacted_thinking":
+            return ({"type": "redacted_thinking", "data": block.get("data")}, ())  # mutable-ok: one-shot JSON payload
+        case _:
+            return (
+                {"type": "text", "text": ""},  # mutable-ok: one-shot JSON payload
+                ({"type": "text_delta", "text": block.get("text") or ""},),  # mutable-ok: one-shot JSON payload
+            )
+
+
+def anthropic_messages_response_as_sse_events(response: AnthropicMessagesResponse) -> tuple[bytes, ...]:
+    """
+    Render a complete (non-streaming) AnthropicMessagesResponse as the SSE
+    event sequence a real streaming request would have produced.
+
+    A mid-stream fallback can resolve to a non-streaming response even
+    though the client asked to stream (e.g. an agentic tool-use loop that
+    intercepts and returns a complete message) - yielding that dict directly
+    into a `/v1/messages` SSE byte stream would produce a malformed
+    response, so it's synthesized into the message_start/content_block_*/
+    message_delta/message_stop lifecycle a real stream would have sent.
+    """
+    content_blocks: Final = response.get("content") or ()
+    content_events: Final = (
+        event for index, block in enumerate(content_blocks) for event in _anthropic_content_block_events(index, block)
+    )
+    # A real message_start always carries a null stop_reason/stop_sequence and
+    # a zero output_tokens - those are only known once generation finishes, so
+    # copying the completed response's final values here would let a client
+    # treat the message as already finished, or double-count output tokens.
+    message_start_usage: Final = {  # mutable-ok: one-shot JSON payload
+        **(response.get("usage") or {}),
+        "output_tokens": 0,
+    }
+    message_start_payload: Final = {  # mutable-ok: one-shot JSON payload, never mutated after construction
+        "type": "message_start",
+        "message": {  # mutable-ok: one-shot JSON payload
+            **response,
+            "content": [],  # mutable-ok: one-shot JSON payload
+            "stop_reason": None,
+            "stop_sequence": None,
+            "usage": message_start_usage,
+        },
+    }
+    message_delta_payload: Final = {  # mutable-ok: one-shot JSON payload, never mutated after construction
+        "type": "message_delta",
+        "delta": {  # mutable-ok: one-shot JSON payload
+            "stop_reason": response.get("stop_reason"),
+            "stop_sequence": response.get("stop_sequence"),
+        },
+        "usage": response.get("usage") or {},  # mutable-ok: one-shot JSON payload
+    }
+    return (
+        _sse_event("message_start", message_start_payload),
+        *content_events,
+        _sse_event("message_delta", message_delta_payload),
+        _sse_event("message_stop", {"type": "message_stop"}),  # mutable-ok: one-shot JSON payload
+    )
+
+
+def _anthropic_content_block_events(index: int, block: Mapping[str, object]) -> tuple[bytes, ...]:
+    start_block, deltas = _anthropic_content_block_start_and_deltas(block)
+    start_payload: Final = {  # mutable-ok: one-shot payload
+        "type": "content_block_start",
+        "index": index,
+        "content_block": start_block,
+    }
+    stop_payload: Final = {  # mutable-ok: one-shot payload
+        "type": "content_block_stop",
+        "index": index,
+    }
+    delta_events: Final = tuple(
+        _sse_event(
+            "content_block_delta",
+            {"type": "content_block_delta", "index": index, "delta": delta},  # mutable-ok: one-shot payload
+        )
+        for delta in deltas
+    )
+    return (
+        _sse_event("content_block_start", start_payload),
+        *delta_events,
+        _sse_event("content_block_stop", stop_payload),
     )
-    return f"event: error\ndata: {payload}\n\n".encode()
 
 
 class AnthropicMessagesStreamHiddenParams(TypedDict):
diff --git a/litellm/llms/anthropic/files/handler.py b/litellm/llms/anthropic/files/handler.py
index 0c62418708f..5fdf2ceff7f 100644
--- a/litellm/llms/anthropic/files/handler.py
+++ b/litellm/llms/anthropic/files/handler.py
@@ -22,19 +22,7 @@ from litellm.types.llms.openai import (
 from litellm.types.utils import CallTypes, LlmProviders, ModelResponse
 
 from ..chat.transformation import AnthropicConfig
-from ..common_utils import AnthropicModelInfo
-
-# Map Anthropic error types to HTTP status codes
-ANTHROPIC_ERROR_STATUS_CODE_MAP: Final = {
-    "invalid_request_error": 400,
-    "authentication_error": 401,
-    "permission_error": 403,
-    "not_found_error": 404,
-    "rate_limit_error": 429,
-    "api_error": 500,
-    "overloaded_error": 503,
-    "timeout_error": 504,
-}
+from ..common_utils import ANTHROPIC_ERROR_STATUS_CODE_MAP, AnthropicModelInfo
 
 
 class AnthropicFilesHandler:
diff --git a/litellm/router.py b/litellm/router.py
index c658bc441ba..d07effd0d90 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -8,6 +8,7 @@
 #  Thank you ! We ❤️ you! - Krrish & Ishaan
 
 import asyncio
+import contextlib
 import copy
 import enum
 import hashlib
@@ -20,7 +21,7 @@ import time
 import traceback
 import weakref
 from collections import defaultdict
-from collections.abc import AsyncGenerator, Callable, Generator, Mapping, Sequence
+from collections.abc import AsyncGenerator, AsyncIterator, Callable, Generator, Mapping, Sequence
 from functools import lru_cache
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypeAlias, TypeVar, Union, cast
@@ -248,6 +249,7 @@ from .router_utils.pattern_match_deployments import PatternMatchRouter
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
 
+    from litellm.exceptions import MidStreamFallbackError
     from litellm.responses.streaming_iterator import (
         BaseResponsesAPIStreamingIterator,
     )
@@ -264,6 +266,9 @@ if TYPE_CHECKING:
     from litellm.router_strategy.quality_router.quality_router import (
         QualityRouter,
     )
+    from litellm.types.llms.anthropic_messages.anthropic_response import (
+        AnthropicMessagesResponse,
+    )
     from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject
     from litellm.types.llms.openai import (
         ResponseAPIUsage,
@@ -361,6 +366,101 @@ def _stream_chunks_have_generated_content(chunks: Sequence[ModelResponseStream])
     return False
 
 
+# Router._aanthropic_messages_streaming_iterator buffers lifecycle chunks
+# until real content commits the primary stream; a hostile or slow-starting
+# upstream that never emits content or an error could otherwise grow that
+# buffer without bound, so hitting this cap forces an early commit instead.
+MAX_BUFFERED_PRE_CONTENT_ANTHROPIC_CHUNKS: Final = 200
+
+
+def _anthropic_stream_should_drop_pre_content_ping(chunk: object, has_generated_content: bool) -> bool:
+    """A `ping` keepalive seen before any real content is dropped outright - it recurs indefinitely on a
+    slow-starting connection and carries nothing worth buffering toward a possible fallback."""
+    from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import is_anthropic_ping_chunk
+
+    if has_generated_content:
+        return False
+    return is_anthropic_ping_chunk(chunk)
+
+
+def _is_retriable_anthropic_status(status_code: int) -> bool:
+    return status_code == 429 or status_code >= 500
+
+
+def _anthropic_stream_should_decline_fallback(has_generated_content: bool, error: "MidStreamFallbackError") -> bool:
+    """
+    A MidStreamFallbackError raised directly by the source iterator (the
+    completion-bridge path's CustomStreamWrapper, e.g. on a transport drop)
+    carries its own pre_first_chunk bookkeeping - gated the same way a
+    detected SSE error event is, so a fallback is never appended after real
+    content already reached the client on either path.
+    """
+    return has_generated_content or not error.is_pre_first_chunk
+
+
+def _anthropic_stream_commits_now(chunk: object, has_generated_content: bool, buffered_chunk_count: int) -> bool:
+    """
+    Whether `chunk` should make Router._aanthropic_messages_streaming_iterator
+    commit to the primary Anthropic stream (real content arrived, or the
+    pre-content buffer cap was hit) rather than keep buffering lifecycle
+    frames toward a possible fallback.
+    """
+    from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
+        is_anthropic_content_delta_chunk,
+    )
+
+    if has_generated_content:
+        return False
+    return is_anthropic_content_delta_chunk(chunk) or buffered_chunk_count >= MAX_BUFFERED_PRE_CONTENT_ANTHROPIC_CHUNKS
+
+
+class FallbackAwareAnthropicMessagesStream:
+    """
+    Bare async generators can't carry the `_hidden_params` attribute the
+    proxy reads response headers off of (see
+    router_utils.add_retry_fallback_headers.get_hidden_params_dict), so this
+    thin wrapper carries it through from the source iterator - mirrors
+    AnthropicMessagesStreamingResponse. Used by
+    Router._aanthropic_messages_streaming_iterator.
+    """
+
+    def __init__(self, async_generator: AsyncGenerator[bytes, None], source_iterator: object) -> None:
+        self._async_generator = async_generator
+        self._hidden_params = dict(  # mutable-ok: mutated in place by merge_fallback_hidden_params
+            getattr(source_iterator, "_hidden_params", None) or {}
+        )
+
+    def __aiter__(self) -> "FallbackAwareAnthropicMessagesStream":
+        return self
+
+    async def __anext__(self) -> bytes:
+        return await self._async_generator.__anext__()
+
+    async def aclose(self) -> None:
+        await self._async_generator.aclose()
+
+    def merge_fallback_hidden_params(
+        self,
+        fallback_hidden_params: Mapping[str, object],
+        fallback_headers: Mapping[str, object],
+    ) -> None:
+        """
+        Raw bytes can't carry their own _hidden_params the way a
+        ModelResponseStream/ResponsesAPI event can, so a mid-stream
+        fallback's provider headers (e.g. Bedrock's x-amzn-requestid) are
+        merged onto the wrapper itself instead - mirrors
+        Router._apply_fallback_hidden_params_to_item's merge shape.
+        """
+        existing_headers: Final = cast(  # cast-ok: additional_headers is always a dict[str, object] when present
+            "dict[str, object]", self._hidden_params.get("additional_headers") or {}
+        )
+        self._hidden_params = {  # mutable-ok: matches _hidden_params' existing dict[str, object] shape
+            **self._hidden_params,
+            **fallback_hidden_params,
+            "additional_headers": {**existing_headers, **fallback_headers},  # mutable-ok: same shape
+        }
+
+
 class RoutingArgs(enum.Enum):
     ttl = 60  # 1min (RPM/TPM expire key)
 
@@ -4806,6 +4906,264 @@ class Router:
             )
         return response
 
+    async def _aanthropic_messages_streaming_iterator(
+        self,
+        response: AsyncIterator[bytes],
+        initial_kwargs: dict[str, Any],  # mutable-ok: mutated in-place before re-entering the fallback chain
+    ) -> AsyncIterator[bytes]:
+        """
+        Wrap an anthropic_messages (/v1/messages) streaming response so a
+        mid-stream provider error triggers the Router's fallback chain
+        (parity with _acompletion_streaming_iterator for the
+        chat-completions path). See #24004.
+
+        anthropic_messages goes through _ageneric_api_call_with_fallbacks
+        rather than _acompletion, so the returned byte iterator is never
+        wrapped by the chat-completions fallback handler. Two failure
+        shapes land here:
+          - the completion-bridge path (deployments with no native
+            /v1/messages endpoint, via
+            LiteLLMMessagesToCompletionTransformationHandler) already
+            raises MidStreamFallbackError out of its underlying
+            CustomStreamWrapper; this wrapper only needs to catch it.
+          - a native Anthropic/Bedrock passthrough never raises anything
+            for a provider SSE `event: error` frame (e.g. `overloaded_error`,
+            `internal_server_error`) - it is forwarded to the client as-is -
+            so this wrapper detects it via parse_anthropic_error_event and
+            raises MidStreamFallbackError itself.
+
+        Only an error before any real content (a content_block_delta frame)
+        has reached the caller triggers a fallback attempt, mirroring the
+        restriction _acompletion_streaming_iterator applies: once generated
+        output has already reached the caller, retrying would start a
+        second, overlapping Anthropic message lifecycle on the same SSE
+        stream, so the error is left to propagate instead of being retried
+        invisibly. A non-retriable client error (4xx other than 429) is
+        never worth a fallback attempt either, so it is also left to
+        propagate.
+
+        Lifecycle/bookkeeping frames (message_start, content_block_start,
+        ping, ...) do not by themselves disqualify a fallback attempt -
+        Anthropic routinely sends message_start before an overload error -
+        but they are BUFFERED rather than forwarded immediately, since
+        forwarding one and then appending a fallback attempt's own
+        message_start would produce two overlapping message lifecycles on
+        one SSE stream. Buffered frames are flushed, in order, the moment
+        real content arrives (the primary attempt has committed by then
+        anyway) or once the stream ends without ever producing content or
+        an error.
+        """
+        from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
+            aclose_if_supported,
+            parse_anthropic_error_event,
+        )
+
+        source_iterator: Final = response
+
+        async def stream_with_fallbacks() -> AsyncGenerator[bytes, None]:
+            from litellm.exceptions import MidStreamFallbackError
+
+            # Lifecycle/bookkeeping frames (message_start, content_block_start,
+            # ping, ...) are held back rather than forwarded immediately:
+            # Anthropic routinely sends message_start before an overload
+            # error, and once a byte reaches the client a fallback attempt
+            # can only append its OWN message_start, producing two
+            # overlapping message lifecycles on one SSE stream. Buffered
+            # frames are flushed the moment real content (content_block_delta)
+            # arrives - at that point the primary attempt has committed and a
+            # clean retry is no longer possible anyway - or once the primary
+            # stream ends without ever producing content. A `ping` keepalive
+            # is dropped outright rather than buffered, since it can recur
+            # indefinitely on a slow-starting connection and carries nothing
+            # worth preserving; hitting MAX_BUFFERED_PRE_CONTENT_ANTHROPIC_CHUNKS
+            # forces the same early commit as real content arriving, so a
+            # hostile or pathological upstream can't grow the buffer forever.
+            has_generated_content = False  # rebind-ok: set once real content is seen, or the buffer cap is hit
+            buffered_lifecycle_chunks: tuple[bytes, ...] = ()  # rebind-ok: flushed once committed or on decline
+            model: Final = cast(str, initial_kwargs.get("model"))  # cast-ok: kwargs always carries the model group
+            try:
+                async for chunk in source_iterator:
+                    if _anthropic_stream_should_drop_pre_content_ping(chunk, has_generated_content):
+                        continue
+                    if _anthropic_stream_commits_now(chunk, has_generated_content, len(buffered_lifecycle_chunks)):
+                        has_generated_content = True  # rebind-ok: real content seen, or the buffer cap was hit
+                    error_event = parse_anthropic_error_event(chunk)
+                    retriable_pending_error = (  # rebind-ok: freshly computed each iteration, never carried over
+                        not has_generated_content
+                        and error_event is not None
+                        and _is_retriable_anthropic_status(error_event[2])
+                    )
+                    if not has_generated_content and not retriable_pending_error and error_event is None:
+                        buffered_lifecycle_chunks = (*buffered_lifecycle_chunks, chunk)
+                        continue
+                    if retriable_pending_error:
+                        assert error_event is not None  # guard-ok: retriable_pending_error implies this
+                        _error_type, message, status_code = error_event
+                        raise MidStreamFallbackError(
+                            message=message,
+                            model=model,
+                            llm_provider="anthropic",
+                            original_exception=litellm.exceptions.APIError(
+                                status_code=status_code,
+                                message=message,
+                                llm_provider="anthropic",
+                                model=model,
+                            ),
+                            is_pre_first_chunk=True,
+                        )
+                    for buffered_chunk in buffered_lifecycle_chunks:
+                        yield buffered_chunk
+                    buffered_lifecycle_chunks = ()
+                    yield chunk
+                for buffered_chunk in buffered_lifecycle_chunks:
+                    yield buffered_chunk
+            except MidStreamFallbackError as e:
+                if _anthropic_stream_should_decline_fallback(has_generated_content, e):
+                    for buffered_chunk in buffered_lifecycle_chunks:
+                        yield buffered_chunk
+                    if e.original_exception is not None:
+                        raise e.original_exception from e
+                    raise
+                async for item in self._aanthropic_messages_fallback_attempt(e, initial_kwargs, wrapper):
+                    yield item
+            finally:
+                with anyio.CancelScope(shield=True), contextlib.suppress(BaseException):
+                    await aclose_if_supported(source_iterator)
+
+        # Referenced by stream_with_fallbacks via closure - assigned here, before
+        # the generator body ever runs, so the reference resolves fine despite
+        # being defined textually after the function that captures it.
+        wrapper: Final = FallbackAwareAnthropicMessagesStream(stream_with_fallbacks(), source_iterator)
+        return wrapper
+
+    async def _aanthropic_messages_fallback_attempt(
+        self,
+        e: "MidStreamFallbackError",
+        initial_kwargs: dict[str, Any],  # mutable-ok: mutated in-place before re-entering the fallback chain
+        wrapper: "FallbackAwareAnthropicMessagesStream",
+    ) -> AsyncGenerator[bytes, None]:
+        """
+        Re-enters the Router's fallback chain for a mid-stream
+        anthropic_messages error and yields whatever the fallback attempt
+        produces. Split out of _aanthropic_messages_streaming_iterator to
+        keep each function's cyclomatic complexity within the repo's C901
+        budget.
+        """
+        from litellm.exceptions import MidStreamFallbackError
+        from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
+            aclose_if_supported,
+            anthropic_messages_response_as_sse_events,
+        )
+
+        fallback_response = None  # rebind-ok: pre-init so finally can close it if a fallback was actually attempted
+        try:
+            model_group: Final = cast(str, initial_kwargs.get("model"))  # cast-ok: model group
+            fallbacks: Final[list | None] = initial_kwargs.get(  # mutable-ok: matches the common_utils list|None param
+                "fallbacks", self.fallbacks
+            )
+            context_window_fallbacks: Final[list | None] = initial_kwargs.get(  # mutable-ok: matches the param below
+                "context_window_fallbacks", self.context_window_fallbacks
+            )
+            content_policy_fallbacks: Final[list | None] = initial_kwargs.get(  # mutable-ok: matches the param below
+                "content_policy_fallbacks", self.content_policy_fallbacks
+            )
+            initial_kwargs["original_function"] = self._ageneric_api_call_with_fallbacks_helper
+            self._update_kwargs_before_fallbacks(
+                model=model_group,
+                kwargs=initial_kwargs,
+                metadata_variable_name="litellm_metadata",
+            )
+            fallback_response = await self.async_function_with_fallbacks_common_utils(  # rebind-ok: set on success
+                e=e,
+                disable_fallbacks=False,
+                fallbacks=fallbacks,
+                context_window_fallbacks=context_window_fallbacks,
+                content_policy_fallbacks=content_policy_fallbacks,
+                model_group=model_group,
+                args=(),
+                kwargs=initial_kwargs,
+                include_fallback_errors=initial_kwargs.get("include_fallback_errors", False) is True,
+            )
+            fallback_hidden_params, fallback_headers = Router._prepare_fallback_hidden_params(fallback_response)
+            wrapper.merge_fallback_hidden_params(fallback_hidden_params, fallback_headers)
+            if hasattr(fallback_response, "__aiter__"):
+                async for fallback_item in fallback_response:
+                    yield fallback_item
+            else:
+                # A fallback can resolve to a complete AnthropicMessagesResponse
+                # dict even for a streaming request (e.g. an agentic tool-use
+                # interception loop) - yielding it as-is would put a raw dict
+                # into a byte stream, so it's synthesized into the SSE
+                # lifecycle a real stream would have sent instead.
+                for event in anthropic_messages_response_as_sse_events(
+                    cast("AnthropicMessagesResponse", fallback_response)  # cast-ok: non-streaming shape by elimination
+                ):
+                    yield event
+        except Exception as fallback_error:
+            verbose_router_logger.error("Anthropic messages streaming fallback also failed: %s", fallback_error)
+            if isinstance(fallback_error, MidStreamFallbackError) and fallback_error.original_exception is not None:
+                raise fallback_error.original_exception from fallback_error
+            raise
+        finally:
+            if fallback_response is not None:
+                with anyio.CancelScope(shield=True), contextlib.suppress(BaseException):
+                    await aclose_if_supported(fallback_response)
+
+    async def _aanthropic_messages_with_streaming_fallbacks(
+        self,
+        original_function: Callable,
+        **kwargs: object,  # kwargs-ok: forwarded verbatim to original_function, shape varies per call site
+    ) -> Union["AnthropicMessagesResponse", AsyncIterator[bytes]]:
+        """
+        _ageneric_api_call_with_fallbacks for anthropic_messages, with the
+        addition of mid-stream fallback handling (see
+        _aanthropic_messages_streaming_iterator). Parity with
+        _aresponses_with_streaming_fallbacks for the Responses API.
+        """
+        from litellm.litellm_core_utils.core_helpers import safe_deep_copy
+
+        # Snapshot the request kwargs before the primary attempt mutates them
+        # in place: _update_kwargs_with_deployment writes deployment-specific
+        # fields (deployment, model_info, api_base, tags, ...) into the
+        # SAME litellm_metadata/metadata dicts a shallow .copy() would still
+        # share, leaking primary-deployment metadata into the mid-stream
+        # fallback request. safe_deep_copy avoids deep-copying the full
+        # kwargs (which can hold non-deepcopyable logging handles/clients).
+        fallback_kwargs: Final[dict[str, object]] = kwargs.copy()  # mutable-ok: mutated below before re-entry
+        if isinstance(fallback_kwargs.get("litellm_metadata"), dict):
+            fallback_kwargs["litellm_metadata"] = safe_deep_copy(fallback_kwargs["litellm_metadata"])
+        if isinstance(fallback_kwargs.get("metadata"), dict):
+            fallback_kwargs["metadata"] = safe_deep_copy(fallback_kwargs["metadata"])
+        fallback_kwargs["original_generic_function"] = original_function
+
+        response: Final = await self._ageneric_api_call_with_fallbacks(original_function=original_function, **kwargs)
+
+        if kwargs.get("stream") and hasattr(response, "__aiter__"):
+            return await self._aanthropic_messages_streaming_iterator(
+                response=cast("AsyncIterator[bytes]", response),  # cast-ok: stream=True always returns a byte iterator
+                initial_kwargs=fallback_kwargs,
+            )
+        return response
+
+    async def _dispatch_generic_call_type(
+        self,
+        call_type: str,
+        original_function: Callable,
+        **kwargs: object,  # kwargs-ok: forwarded verbatim to the per-call-type helper, shape varies per call site
+    ):
+        """
+        factory_function's shared dispatch for call types with no
+        call-specific handling, except anthropic_messages: kept out of
+        factory_function's own async_wrapper (already at the repo's C901
+        complexity ceiling) so routing its mid-stream fallback handling
+        (#24004) doesn't add another branch there.
+        """
+        if call_type == "anthropic_messages":
+            return await self._aanthropic_messages_with_streaming_fallbacks(
+                original_function=original_function, **kwargs
+            )
+        return await self._ageneric_api_call_with_fallbacks(original_function=original_function, **kwargs)
+
     def _generic_api_call_with_fallbacks(self, model: str, original_function: Callable, **kwargs):
         """
         Make a generic LLM API call through the router, this allows you to use retries/fallbacks with litellm router
@@ -5992,7 +6350,8 @@ class Router:
                 "aget_skill",
                 "adelete_skill",
             ):
-                return await self._ageneric_api_call_with_fallbacks(
+                return await self._dispatch_generic_call_type(
+                    call_type=call_type,
                     original_function=original_function,
                     **kwargs,
                 )
diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py
index f33bb3dda8b..652c1f077a9 100644
--- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py
+++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py
@@ -11,6 +11,10 @@ from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterato
     BaseAnthropicMessagesStreamingIterator,
     _incomplete_stream_error_sse_event,
     _is_message_stop_chunk,
+    _is_provider_error_chunk,
+    anthropic_messages_response_as_sse_events,
+    is_anthropic_content_delta_chunk,
+    parse_anthropic_error_event,
 )
 
 
@@ -157,6 +161,96 @@ def test_is_message_stop_chunk_ignores_substring_in_payload():
     assert _is_message_stop_chunk(delta_frame_with_substring) is False
 
 
+def test_parse_anthropic_error_event_from_dict_chunk():
+    """Regression for #24004: dict-shaped error chunks parse to
+    (type, message, status) so the Router can decide whether to fall back."""
+    chunk = {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}
+    assert parse_anthropic_error_event(chunk) == ("overloaded_error", "Overloaded", 503)
+    assert _is_provider_error_chunk(chunk) is True
+
+
+def test_parse_anthropic_error_event_from_sse_bytes():
+    """Regression for #24004: a raw `event: error` SSE frame (what a native
+    Anthropic/Bedrock passthrough forwards verbatim today) must parse
+    identically to the dict shape so the Router can raise a fallback."""
+    sse_chunk = (
+        b"event: error\n"
+        b'data: {"type": "error", "error": {"type": "internal_server_error", "message": "boom"}}\n\n'
+    )
+    assert parse_anthropic_error_event(sse_chunk) == ("internal_server_error", "boom", 500)
+    assert _is_provider_error_chunk(sse_chunk) is True
+
+
+def test_parse_anthropic_error_event_defaults_status_for_unknown_type():
+    chunk = {"type": "error", "error": {"type": "some_future_error_type", "message": "?"}}
+    assert parse_anthropic_error_event(chunk) == ("some_future_error_type", "?", 500)
+
+
+def test_parse_anthropic_error_event_missing_message_falls_back_to_type():
+    chunk = {"type": "error", "error": {"type": "overloaded_error"}}
+    assert parse_anthropic_error_event(chunk) == ("overloaded_error", "overloaded_error", 503)
+
+
+def test_parse_anthropic_error_event_non_string_error_type_returns_none():
+    """A malformed error body whose `type` field isn't a string (e.g. an
+    upstream bug sends null or a number) must not be treated as an error
+    event rather than crashing or forwarding a garbage error_type."""
+    chunk = {"type": "error", "error": {"type": None, "message": "boom"}}
+    assert parse_anthropic_error_event(chunk) is None
+
+
+def test_decoded_sse_data_line_swallows_invalid_json():
+    """A `data:` line that isn't valid JSON (a malformed/truncated frame)
+    must not be treated as an error event or raise, just be ignored."""
+    malformed_frame = b"event: error\ndata: {not valid json\n\n"
+    assert parse_anthropic_error_event(malformed_frame) is None
+    assert _is_provider_error_chunk(malformed_frame) is False
+
+
+class TestIsAnthropicContentDeltaChunk:
+    def test_dict_content_block_delta(self):
+        assert is_anthropic_content_delta_chunk({"type": "content_block_delta"}) is True
+
+    def test_dict_other_type(self):
+        assert is_anthropic_content_delta_chunk({"type": "message_start"}) is False
+
+    def test_bytes_content_block_delta(self):
+        assert is_anthropic_content_delta_chunk(b"event: content_block_delta\ndata: {}\n\n") is True
+
+    def test_bytes_other_event(self):
+        assert is_anthropic_content_delta_chunk(b"event: message_start\ndata: {}\n\n") is False
+
+    def test_neither_dict_nor_bytes(self):
+        assert is_anthropic_content_delta_chunk("content_block_delta") is False
+        assert is_anthropic_content_delta_chunk(None) is False
+
+
+@pytest.mark.parametrize(
+    "chunk",
+    [
+        {"type": "content_block_delta", "delta": {"type": "text_delta", "text": "hi"}},
+        b'event: content_block_delta\ndata: {"type": "content_block_delta"}\n\n',
+        b"raw-bytes",
+        "error",
+        None,
+    ],
+)
+def test_parse_anthropic_error_event_non_error_chunks_return_none(chunk):
+    assert parse_anthropic_error_event(chunk) is None
+    assert _is_provider_error_chunk(chunk) is False
+
+
+def test_parse_anthropic_error_event_ignores_substring_in_payload():
+    """A content_block_delta whose partial_json happens to contain the
+    literal string `"type": "error"` must not be misread as an error event."""
+    delta_frame_with_substring = (
+        b"event: content_block_delta\n"
+        b'data: {"type": "content_block_delta", "delta": '
+        b'{"type": "input_json_delta", "partial_json": "\\"type\\": \\"error\\""}}\n\n'
+    )
+    assert parse_anthropic_error_event(delta_frame_with_substring) is None
+
+
 @pytest.mark.asyncio
 async def test_async_sse_wrapper_emits_error_when_bytes_stream_only_mentions_message_stop_in_payload():
     """
@@ -307,3 +401,117 @@ def test_incomplete_stream_error_sse_event_is_valid_anthropic_error():
         "error": {"type": "api_error", "message": INCOMPLETE_STREAM_ERROR_MESSAGE},
     }
     assert event.endswith("\n\n")
+
+
+def _decode_sse_events(events: tuple[bytes, ...]) -> list[tuple[str, dict]]:
+    decoded = []
+    for event in events:
+        assert isinstance(event, bytes)
+        lines = event.decode().split("\n")
+        assert lines[0].startswith("event: ")
+        decoded.append((lines[0].removeprefix("event: "), json.loads(lines[1].removeprefix("data: "))))
+    return decoded
+
+
+def test_anthropic_messages_response_as_sse_events_text_block():
+    response = {
+        "id": "msg_1",
+        "model": "claude-haiku",
+        "role": "assistant",
+        "type": "message",
+        "stop_reason": "end_turn",
+        "stop_sequence": None,
+        "content": [{"type": "text", "text": "hello"}],
+        "usage": {"input_tokens": 3, "output_tokens": 2},
+    }
+    decoded = _decode_sse_events(anthropic_messages_response_as_sse_events(response))
+    types = [event_type for event_type, _ in decoded]
+    assert types == [
+        "message_start",
+        "content_block_start",
+        "content_block_delta",
+        "content_block_stop",
+        "message_delta",
+        "message_stop",
+    ]
+    # message_start must not carry generated content itself, matching a real
+    # streaming response - it arrives via the content_block_delta that follows.
+    assert decoded[0][1]["message"]["content"] == []
+    assert decoded[0][1]["message"]["id"] == "msg_1"
+    # Bugbot regression: message_start must not carry the completed response's
+    # final stop_reason/stop_sequence/output_tokens - a real stream keeps those
+    # null/zero until message_delta, so a client could otherwise treat the
+    # message as already finished, or double-count output tokens.
+    assert decoded[0][1]["message"]["stop_reason"] is None
+    assert decoded[0][1]["message"]["stop_sequence"] is None
+    assert decoded[0][1]["message"]["usage"] == {"input_tokens": 3, "output_tokens": 0}
+    assert decoded[1][1]["content_block"] == {"type": "text", "text": ""}
+    assert decoded[2][1]["delta"] == {"type": "text_delta", "text": "hello"}
+    assert decoded[4][1]["delta"]["stop_reason"] == "end_turn"
+    assert decoded[4][1]["usage"] == {"input_tokens": 3, "output_tokens": 2}
+
+
+def test_anthropic_messages_response_as_sse_events_tool_use_block():
+    response = {
+        "id": "msg_2",
+        "content": [{"type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": {"city": "NYC"}}],
+        "stop_reason": "tool_use",
+    }
+    decoded = _decode_sse_events(anthropic_messages_response_as_sse_events(response))
+    content_block_start = dict(decoded)["content_block_start"]
+    assert content_block_start["content_block"] == {
+        "type": "tool_use",
+        "id": "toolu_1",
+        "name": "get_weather",
+        "input": {},
+    }
+    content_block_delta = dict(decoded)["content_block_delta"]
+    assert json.loads(content_block_delta["delta"]["partial_json"]) == {"city": "NYC"}
+    assert content_block_delta["delta"]["type"] == "input_json_delta"
+
+
+def test_anthropic_messages_response_as_sse_events_thinking_block_emits_signature_delta():
+    """Bugbot regression: a thinking block's real `signature` must reach the
+    client via a trailing signature_delta, not be silently dropped - Anthropic
+    rejects a replayed assistant message (a follow-up turn, a tool-use
+    continuation) whose thinking block lacks its original signature."""
+    response = {
+        "id": "msg_5",
+        "content": [{"type": "thinking", "thinking": "let me think", "signature": "sig-abc123"}],
+        "stop_reason": "end_turn",
+    }
+    decoded = _decode_sse_events(anthropic_messages_response_as_sse_events(response))
+    deltas = [payload["delta"] for event_type, payload in decoded if event_type == "content_block_delta"]
+    assert deltas == [
+        {"type": "thinking_delta", "thinking": "let me think"},
+        {"type": "signature_delta", "signature": "sig-abc123"},
+    ]
+
+
+def test_anthropic_messages_response_as_sse_events_thinking_block_without_signature_omits_delta():
+    response = {
+        "id": "msg_6",
+        "content": [{"type": "thinking", "thinking": "let me think", "signature": None}],
+        "stop_reason": "end_turn",
+    }
+    decoded = _decode_sse_events(anthropic_messages_response_as_sse_events(response))
+    deltas = [payload["delta"] for event_type, payload in decoded if event_type == "content_block_delta"]
+    assert deltas == [{"type": "thinking_delta", "thinking": "let me think"}]
+
+
+def test_anthropic_messages_response_as_sse_events_multiple_blocks_are_indexed():
+    response = {
+        "id": "msg_3",
+        "content": [{"type": "text", "text": "a"}, {"type": "text", "text": "b"}],
+    }
+    decoded = _decode_sse_events(anthropic_messages_response_as_sse_events(response))
+    starts = [payload for event_type, payload in decoded if event_type == "content_block_start"]
+    assert [s["index"] for s in starts] == [0, 1]
+    deltas = [payload for event_type, payload in decoded if event_type == "content_block_delta"]
+    assert [d["delta"]["text"] for d in deltas] == ["a", "b"]
+
+
+def test_anthropic_messages_response_as_sse_events_no_content_blocks():
+    response = {"id": "msg_4", "content": [], "stop_reason": "end_turn"}
+    decoded = _decode_sse_events(anthropic_messages_response_as_sse_events(response))
+    assert [event_type for event_type, _ in decoded] == ["message_start", "message_delta", "message_stop"]
diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py
index 0b82ab971bf..56fd6df446f 100644
--- a/tests/test_litellm/test_router.py
+++ b/tests/test_litellm/test_router.py
@@ -12,8 +12,17 @@ import pytest
 
 
 import litellm
+from litellm import Router
 from litellm.exceptions import MidStreamFallbackError
 from litellm.integrations.custom_logger import CustomLogger
+from litellm.router import (
+    MAX_BUFFERED_PRE_CONTENT_ANTHROPIC_CHUNKS,
+    FallbackAwareAnthropicMessagesStream,
+    _anthropic_stream_commits_now,
+    _anthropic_stream_should_decline_fallback,
+    _anthropic_stream_should_drop_pre_content_ping,
+    _is_retriable_anthropic_status,
+)
 
 
 def test_update_kwargs_does_not_mutate_defaults_and_merges_metadata():
@@ -9224,3 +9233,1095 @@ class TestAddDeploymentApiBaseProviderResolution:
         deployment = router.get_deployment_by_model_group_name("openai-via-gateway")
         assert deployment is not None
         assert deployment.litellm_params.custom_llm_provider == "openai"
+
+# =====================================================================
+# anthropic_messages mid-stream-fallback helpers, added for #24004
+# (mid-stream fallback not supported for anthropic_messages route type).
+#
+# anthropic_messages goes through _ageneric_api_call_with_fallbacks rather
+# than _acompletion, so its returned iterator was never wrapped by the chat
+# completions fallback handler: an SSE `event: error` frame from a native
+# Anthropic/Bedrock passthrough passed through to the client silently, and a
+# MidStreamFallbackError raised by the completion-bridge path's
+# CustomStreamWrapper (e.g. a Vertex AI transport drop) propagated
+# unhandled.
+#
+# Targets the helpers introduced on Router:
+#   - _aanthropic_messages_streaming_iterator
+#   - _aanthropic_messages_fallback_attempt
+#   - _aanthropic_messages_with_streaming_fallbacks
+#   - _dispatch_generic_call_type
+# =====================================================================
+
+
+async def _anthropic_messages_empty_generator():
+    return
+    yield  # pragma: no cover - makes this an async generator
+
+
+def _anthropic_messages_make_wrapper() -> FallbackAwareAnthropicMessagesStream:
+    """A minimal wrapper for tests that call _aanthropic_messages_fallback_attempt
+    directly, bypassing _aanthropic_messages_streaming_iterator."""
+    return FallbackAwareAnthropicMessagesStream(_anthropic_messages_empty_generator(), object())
+
+
+def _anthropic_messages_make_router() -> Router:
+    return Router(
+        model_list=[
+            {
+                "model_name": "primary",
+                "litellm_params": {
+                    "model": "anthropic/claude-sonnet-4-5",
+                    "api_key": "sk-test",
+                },
+            },
+            {
+                "model_name": "fallback",
+                "litellm_params": {
+                    "model": "bedrock/anthropic.claude-sonnet-4-5",
+                },
+            },
+        ]
+    )
+
+
+class _AnthropicMessagesFakeByteStream:
+    """Minimal AsyncIterator[bytes], carrying _hidden_params like
+    AnthropicMessagesStreamingResponse does."""
+
+    def __init__(self, chunks: list) -> None:
+        self._chunks = list(chunks)
+        self._hidden_params = {"additional_headers": {"x-amzn-requestid": "req-1"}}
+        self.closed = False
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self) -> bytes:
+        if not self._chunks:
+            raise StopAsyncIteration
+        return self._chunks.pop(0)
+
+    async def aclose(self) -> None:
+        self.closed = True
+
+
+class _AnthropicMessagesRaisingByteStream:
+    """Simulates the completion-bridge path: no error SSE chunk is ever
+    yielded, the underlying CustomStreamWrapper raises MidStreamFallbackError
+    directly out of the iterator instead (a Vertex AI transport drop)."""
+
+    def __init__(self, chunks: list, error: Exception) -> None:
+        self._chunks = list(chunks)
+        self._error = error
+        self._hidden_params: dict = {}
+        self.closed = False
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self) -> bytes:
+        if self._chunks:
+            return self._chunks.pop(0)
+        raise self._error
+
+    async def aclose(self) -> None:
+        self.closed = True
+
+
+class _AnthropicMessagesFallbackByteStream:
+    def __init__(self, chunks: list, hidden_params: dict | None = None) -> None:
+        self._chunks = list(chunks)
+        self._hidden_params = hidden_params if hidden_params is not None else {}
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self) -> bytes:
+        if not self._chunks:
+            raise StopAsyncIteration
+        return self._chunks.pop(0)
+
+
+def _anthropic_messages_overloaded_error_chunk() -> bytes:
+    return (
+        b"event: error\n"
+        b'data: {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}\n\n'
+    )
+
+
+def _anthropic_messages_invalid_request_error_chunk() -> bytes:
+    return (
+        b"event: error\n"
+        b'data: {"type": "error", "error": {"type": "invalid_request_error", "message": "bad request"}}\n\n'
+    )
+
+
+def _anthropic_messages_rate_limit_error_chunk() -> bytes:
+    return (
+        b"event: error\n"
+        b'data: {"type": "error", "error": {"type": "rate_limit_error", "message": "Too many requests"}}\n\n'
+    )
+
+
+def _anthropic_messages_content_chunk(text: str = "hi") -> bytes:
+    payload = f'{{"type": "content_block_delta", "delta": {{"type": "text_delta", "text": "{text}"}}}}'
+    return f"event: content_block_delta\ndata: {payload}\n\n".encode()
+
+
+def _anthropic_messages_message_start_chunk() -> bytes:
+    """A lifecycle/bookkeeping frame Anthropic sends before any real content -
+    routinely the very first event before an overload error."""
+    return b'event: message_start\ndata: {"type": "message_start", "message": {"id": "msg_1"}}\n\n'
+
+
+def _anthropic_messages_ping_chunk() -> bytes:
+    return b'event: ping\ndata: {"type": "ping"}\n\n'
+
+
+# -------- _aanthropic_messages_streaming_iterator (passthrough) --------
+
+
+@pytest.mark.asyncio
+async def test_anthropic_messages_streaming_iterator_passthrough():
+    """Without any error chunk, the wrapper forwards every chunk unchanged
+    and carries the source iterator's _hidden_params through (so response
+    headers like Bedrock's request-id keep flowing to the client)."""
+    router = _anthropic_messages_make_router()
+    source = _AnthropicMessagesFakeByteStream(
+        [_anthropic_messages_content_chunk("hi"), _anthropic_messages_content_chunk(" there")]
+    )
+
+    wrapped = await router._aanthropic_messages_streaming_iterator(
+        response=source, initial_kwargs={"model": "primary"}
+    )
+
+    collected = [chunk async for chunk in wrapped]
+    assert collected == [_anthropic_messages_content_chunk("hi"), _anthropic_messages_content_chunk(" there")]
+    assert wrapped._hidden_params["additional_headers"]["x-amzn-requestid"] == "req-1"
+
+
+@pytest.mark.asyncio
+async def test_anthropic_messages_streaming_iterator_flushes_buffered_lifecycle_frames_in_order():
+    """Regression: lifecycle frames held back to guard against a mid-stream
+    fallback must still reach the client, in order, once real content
+    arrives - buffering them for the fallback-safety check must not silently
+    drop them on the happy path."""
+    router = _anthropic_messages_make_router()
+    message_stop = b'event: message_stop\ndata: {"type": "message_stop"}\n\n'
+    source = _AnthropicMessagesFakeByteStream(
+        [_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("hi"), message_stop]
+    )
+
+    wrapped = await router._aanthropic_messages_streaming_iterator(
+        response=source, initial_kwargs={"model": "primary"}
+    )
+
+    collected = [chunk async for chunk in wrapped]
+    assert collected == [_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("hi"), message_stop]
+
+
+@pytest.mark.asyncio
+async def test_anthropic_messages_streaming_iterator_flushes_buffered_frames_on_stream_end():
+    """Regression: if the primary stream ends with only lifecycle frames and
+    no content and no error, the buffered frames must still reach the
+    client rather than being silently swallowed."""
+    router = _anthropic_messages_make_router()
+    message_stop = b'event: message_stop\ndata: {"type": "message_stop"}\n\n'
+    source = _AnthropicMessagesFakeByteStream([_anthropic_messages_message_start_chunk(), message_stop])
+
+    wrapped = await router._aanthropic_messages_streaming_iterator(
+        response=source, initial_kwargs={"model": "primary"}
+    )
+
+    collected = [chunk async for chunk in wrapped]
+    assert collected == [_anthropic_messages_message_start_chunk(), message_stop]
+
+    with pytest.raises(StopAsyncIteration):
+        await wrapped.__anext__()
+
+
+@pytest.mark.asyncio
+async def test_anthropic_messages_content_coalesced_with_error_in_one_physical_chunk_skips_fallback():
+    """Greptile review round: transport-level buffering can coalesce a real
+    content_block_delta and a following retriable error into ONE physical
+    read from the source iterator. Since the whole chunk (content and error
+    together) is forwarded to the client atomically, the client genuinely
+    receives the content - so no fallback must be attempted, exactly as if
+    the two events had arrived as separate reads."""
+    router = _anthropic_messages_make_router()
+    coalesced_chunk = _anthropic_messages_content_chunk("partial") + _anthropic_messages_overloaded_error_chunk()
+    source = _AnthropicMessagesFakeByteStream([coalesced_chunk])
+
+    with patch.object(
+        router,
+        "async_function_with_fallbacks_common_utils",
+        new=AsyncMock(return_value=_AnthropicMessagesFallbackByteStream([])),
+    ) as mock_fallback:
+        wrapped = await router._aanthropic_messages_streaming_iterator(
+            response=source,
+            initial_kwargs={"model": "primary"},
+        )
+        collected = [chunk async for chunk in wrapped]
+
+    assert collected == [coalesced_chunk]
+    mock_fallback.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_anthropic_messages_ping_keepalive_never_buffered_or_forwarded():
+    """Bugbot regression: a `ping` keepalive carries no content and must be
+    dropped outright before any real content arrives, rather than buffered -
+    otherwise a slow-starting connection sending many pings could grow the
+    pre-content buffer without bound."""
+    router = _anthropic_messages_make_router()
+    source = _AnthropicMessagesFakeByteStream(
+        [_anthropic_messages_ping_chunk(), _anthropic_messages_content_chunk("hi")]
+    )
+
+    wrapped = await router._aanthropic_messages_streaming_iterator(response=source, initial_kwargs={"model": "primary"})
+    collected = [chunk async for chunk in wrapped]
+
+    assert _anthropic_messages_ping_chunk() not in collected
+    assert collected == [_anthropic_messages_content_chunk("hi")]
+
+
+@pytest.mark.asyncio
+async def test_anthropic_messages_pre_content_buffer_cap_forces_commit():
+    """Bugbot regression: a hostile or pathological upstream that never emits
+    real content or an error must not grow the pre-content lifecycle buffer
+    without bound - hitting MAX_BUFFERED_PRE_CONTENT_ANTHROPIC_CHUNKS commits
+    to the primary stream early, exactly as real content arriving would."""
+    router = _anthropic_messages_make_router()
+    lifecycle_chunk = _anthropic_messages_message_start_chunk()
+    error_chunk = _anthropic_messages_overloaded_error_chunk()
+    chunks = [lifecycle_chunk] * (MAX_BUFFERED_PRE_CONTENT_ANTHROPIC_CHUNKS + 5) + [error_chunk]
+    source = _AnthropicMessagesFakeByteStream(chunks)
+
+    with patch.object(
+        router,
+        "async_function_with_fallbacks_common_utils",
+        new=AsyncMock(return_value=_AnthropicMessagesFallbackByteStream([])),
+    ) as mock_fallback:
+        wrapped = await router._aanthropic_messages_streaming_iterator(
+            response=source, initial_kwargs={"model": "primary"}
+        )
+        collected = [chunk async for chunk in wrapped]
+
+    mock_fallback.assert_not_awaited()
+    assert collected.count(lifecycle_chunk) == MAX_BUFFERED_PRE_CONTENT_ANTHROPIC_CHUNKS + 5
+    assert collected[-1] == error_chunk
+
+
+@pytest.mark.asyncio
+async def test_anthropic_messages_ping_coalesced_with_content_in_one_physical_chunk_is_forwarded():
+    """Greptile/Bugbot regression: transport-level buffering can coalesce a
+    `ping` keepalive and a real content_block_delta into ONE physical read.
+    The pre-content ping-drop must only discard PURE ping frames - dropping
+    the whole coalesced chunk would silently lose generated content."""
+    router = _anthropic_messages_make_router()
+    coalesced_chunk = _anthropic_messages_ping_chunk() + _anthropic_messages_content_chunk("hi")
+    source = _AnthropicMessagesFakeByteStream([coalesced_chunk])
+
+    wrapped = await router._aanthropic_messages_streaming_iterator(response=source, initial_kwargs={"model": "primary"})
+    collected = [chunk async for chunk in wrapped]
+
+    assert collected == [coalesced_chunk]
+
+
+@pytest.mark.asyncio
+async def test_anthropic_messages_ping_coalesced_with_retriable_error_still_falls_back():
+    """Greptile/Bugbot regression: a physical chunk coalescing a `ping` with a
+    retriable `event: error` must not be discarded as a keepalive - the error
+    inside it must still trigger the mid-stream fallback."""
+    router = _anthropic_messages_make_router()
+    coalesced_chunk = _anthropic_messages_ping_chunk() + _anthropic_messages_overloaded_error_chunk()
+    source = _AnthropicMessagesFakeByteStream([coalesced_chunk])
+    fallback_stream = _AnthropicMessagesFallbackByteStream([_anthropic_messages_content_chunk("fallback answer")])
+
+    with patch.object(
+        router,
+        "async_function_with_fallbacks_common_utils",
+        new=AsyncMock(return_value=fallback_stream),
+    ) as mock_fallback:
+        wrapped = await router._aanthropic_messages_streaming_iterator(
+            response=source, initial_kwargs={"model": "primary"}
+        )
+        collected = [chunk async for chunk in wrapped]
+
+    mock_fallback.assert_awaited_once()
+    assert collected == [_anthropic_messages_content_chunk("fallback answer")]
+
+
+# -------- _aanthropic_messages_fallback_attempt --------
+
+
+@pytest.mark.asyncio
+async def test_aanthropic_messages_fallback_attempt_yields_fallback_stream():
+    """Direct-call regression: the fallback-attempt helper re-enters the
+    Router's fallback chain and forwards whatever the fallback produces."""
+    router = _anthropic_messages_make_router()
+    fallback_stream = _AnthropicMessagesFallbackByteStream([_anthropic_messages_content_chunk("fallback answer")])
+    error = MidStreamFallbackError(message="overloaded", model="primary", llm_provider="anthropic")
+
+    with patch.object(
+        router,
+        "async_function_with_fallbacks_common_utils",
+        new=AsyncMock(return_value=fallback_stream),
+    ) as mock_fallback:
+        collected = [
+            chunk
+            async for chunk in router._aanthropic_messages_fallback_attempt(
+                error,
+                {"model": "primary", "messages": [{"role": "user", "content": "hi"}]},
+                _anthropic_messages_make_wrapper(),
+            )
+        ]
+
+    assert collected == [_anthropic_messages_content_chunk("fallback answer")]
+    mock_fallback.assert_awaited_once()
+    assert mock_fallback.await_args.kwargs["e"] is error
+
+
+@pytest.mark.asyncio
+async def test_aanthropic_messages_fallback_attempt_raises_original_exception_on_double_failure():
+    """Direct-call regression: when the fallback attempt itself fails with a
+    MidStreamFallbackError wrapping a real provider exception, that real
+    exception must surface rather than the internal wrapper exception."""
+    router = _anthropic_messages_make_router()
+    error = MidStreamFallbackError(message="overloaded", model="primary", llm_provider="anthropic")
+    original_exception = litellm.APIError(
+        status_code=503, message="fallback also overloaded", llm_provider="bedrock", model="fallback"
+    )
+    fallback_failure = MidStreamFallbackError(
+        message="fallback failed", model="fallback", llm_provider="bedrock", original_exception=original_exception
+    )
+
+    with patch.object(
+        router,
+        "async_function_with_fallbacks_common_utils",
+        new=AsyncMock(side_effect=fallback_failure),
+    ):
+        with pytest.raises(litellm.APIError) as exc_info:
+            async for _ in router._aanthropic_messages_fallback_attempt(
+                error, {"model": "primary"}, _anthropic_messages_make_wrapper()
+            ):
+                pass
+
+    assert exc_info.value is original_exception
+
+
+@pytest.mark.asyncio
+async def test_aanthropic_messages_fallback_attempt_yields_non_streaming_fallback_response():
+    """Bugbot regression: a fallback that resolves to a non-streaming
+    response (no __aiter__, e.g. an agentic tool-use interception loop) must
+    be synthesized into a valid SSE byte sequence, not yielded as a raw dict
+    into a byte stream - the generator is typed AsyncGenerator[bytes, None]
+    and every item reaching the client must be a real SSE frame."""
+    router = _anthropic_messages_make_router()
+    error = MidStreamFallbackError(message="overloaded", model="primary", llm_provider="anthropic")
+    non_streaming_response = {"id": "msg_1", "type": "message", "content": [{"type": "text", "text": "hi"}]}
+
+    with patch.object(
+        router,
+        "async_function_with_fallbacks_common_utils",
+        new=AsyncMock(return_value=non_streaming_response),
+    ):
+        collected = [
+            item
+            async for item in router._aanthropic_messages_fallback_attempt(
+                error, {"model": "primary"}, _anthropic_messages_make_wrapper()
+            )
+        ]
+
+    assert all(isinstance(item, bytes) for item in collected)
+    event_types = [item.split(b"\n")[0].removeprefix(b"event: ") for item in collected]
+    assert event_types == [
+        b"message_start",
+        b"content_block_start",
+        b"content_block_delta",
+        b"content_block_stop",
+        b"message_delta",
+        b"message_stop",
+    ]
+    assert b'"text": "hi"' in collected[2]
+
+
+@pytest.mark.asyncio
+async def test_aanthropic_messages_fallback_attempt_reraises_plain_exception_on_double_failure():
+    """Direct-call regression: when the fallback attempt fails with a plain
+    exception (not a MidStreamFallbackError), that exception itself must
+    propagate unchanged."""
+    router = _anthropic_messages_make_router()
+    error = MidStreamFallbackError(message="overloaded", model="primary", llm_provider="anthropic")
+    fallback_failure = ValueError("no healthy deployments")
+
+    with patch.object(
+        router,
+        "async_function_with_fallbacks_common_utils",
+        new=AsyncMock(side_effect=fallback_failure),
+    ):
+        with pytest.raises(ValueError, match="no healthy deployments") as exc_info:
+            async for _ in router._aanthropic_messages_fallback_attempt(
+                error, {"model": "primary"}, _anthropic_messages_make_wrapper()
+            ):
+                pass
+
+    assert exc_info.value is fallback_failure
+
+
+# -------- _aanthropic_messages_with_streaming_fallbacks --------
+
+
+@pytest.mark.asyncio
+async def test_aanthropic_messages_with_streaming_fallbacks_non_streaming_passthrough():
+    """A non-streaming response (plain dict) is returned unchanged, never wrapped."""
+    router = _anthropic_messages_make_router()
+    plain_response = {"id": "msg_1", "type": "message"}
+
+    async def fake_original(**_kwargs):
+        return plain_response
+
+    with patch.object(
+        router,
+        "_ageneric_api_call_with_fallbacks",
+        new=AsyncMock(return_value=plain_response),
+    ):
+        out = await router._aanthropic_messages_with_streaming_fallbacks(
+            original_function=fake_original,
+            model="primary",
+            stream=False,
+        )
+    assert out is plain_response
+
+
+@pytest.mark.asyncio
+async def test_aanthropic_messages_with_streaming_fallbacks_wraps_streaming_iterator():
+    """A streaming response is wrapped via _aanthropic_messages_streaming_iterator."""
+    router = _anthropic_messages_make_router()
+    streaming_iter = _AnthropicMessagesFakeByteStream([_anthropic_messages_content_chunk()])
+    wrapped_marker = object()
+
+    async def fake_original(**_kwargs):
+        return streaming_iter
+
+    with (
+        patch.object(
+            router,
+            "_ageneric_api_call_with_fallbacks",
+            new=AsyncMock(return_value=streaming_iter),
+        ),
+        patch.object(
+            router,
+            "_aanthropic_messages_streaming_iterator",
+            new=AsyncMock(return_value=wrapped_marker),
+        ) as mock_wrap,
+    ):
+        out = await router._aanthropic_messages_with_streaming_fallbacks(
+            original_function=fake_original,
+            model="primary",
+            stream=True,
+        )
+    assert out is wrapped_marker
+    mock_wrap.assert_awaited_once()
+
+
+# -------- mid-stream error handling --------
+
+
+@pytest.mark.asyncio
+async def test_anthropic_messages_fallback_on_pre_first_chunk_error_event():
+    """Regression for #24004: a retriable SSE `event: error` frame
+    (overloaded_error/internal_server_error) that arrives before any real
+    content must trigger the router's fallback chain instead of passing
+    through to the client silently."""
+    router = _anthropic_messages_make_router()
+    source = _AnthropicMessagesFakeByteStream([_anthropic_messages_overloaded_error_chunk()])
+    fallback_stream = _AnthropicMessagesFallbackByteStream([_anthropic_messages_content_chunk("fallback answer")])
+
+    with patch.object(
+        router,
+        "async_function_with_fallbacks_common_utils",
+        new=AsyncMock(return_value=fallback_stream),
+    ) as mock_fallback:
+        wrapped = await router._aanthropic_messages_streaming_iterator(
+            response=source,
+            initial_kwargs={"model": "primary", "messages": [{"role": "user", "content": "hi"}]},
+        )
+        collected = [chunk async for chunk in wrapped]
+
+    assert collected == [_anthropic_messages_content_chunk("fallback answer")]
+    mock_fallback.assert_awaited_once()
+    raised = mock_fallback.await_args.kwargs["e"]
+    assert isinstance(raised, MidStreamFallbackError)
+    assert raised.status_code == 503
+    assert raised.is_pre_first_chunk is True
+    assert source.closed is True
+
+
+@pytest.mark.asyncio
+async def test_anthropic_messages_mid_stream_error_preserves_real_status_code():
+    """Bugbot regression: the MidStreamFallbackError raised for a detected SSE
+    `event: error` frame must carry the error's REAL parsed status code
+    (via original_exception), not silently default to 503 for every error
+    type - a rate_limit_error (429) must surface as 429, not 503."""
+    router = _anthropic_messages_make_router()
+    source = _AnthropicMessagesFakeByteStream([_anthropic_messages_rate_limit_error_chunk()])
+    fallback_stream = _AnthropicMessagesFallbackByteStream([_anthropic_messages_content_chunk("fallback answer")])
+
+    with patch.object(
+        router,
+        "async_function_with_fallbacks_common_utils",
+        new=AsyncMock(return_value=fallback_stream),
+    ) as mock_fallback:
+        wrapped = await router._aanthropic_messages_streaming_iterator(
+            response=source,
+            initial_kwargs={"model": "primary", "messages": [{"role": "user", "content": "hi"}]},
+        )
+        [chunk async for chunk in wrapped]
+
+    raised = mock_fallback.await_args.kwargs["e"]
+    assert isinstance(raised, MidStreamFallbackError)
+    assert raised.status_code == 429
+    assert raised.original_exception is not None
+    assert raised.original_exception.status_code == 429
+    assert raised.original_exception.llm_provider == "anthropic"
+
+
+def test_merge_fallback_hidden_params_direct_call():
+    """Direct-call regression: merge_fallback_hidden_params combines the
+    fallback's hidden params/headers with whatever was already present,
+    with the fallback's values winning on key collisions."""
+    wrapper = FallbackAwareAnthropicMessagesStream(
+        _anthropic_messages_empty_generator(),
+        _AnthropicMessagesFakeByteStream([]),  # carries {"additional_headers": {"x-amzn-requestid": "req-1"}}
+    )
+    wrapper.merge_fallback_hidden_params(
+        {"model_id": "fallback-deployment"},
+        {"x-amzn-requestid": "req-2", "x-fallback-only": "yes"},
+    )
+    assert wrapper._hidden_params["model_id"] == "fallback-deployment"
+    assert wrapper._hidden_params["additional_headers"] == {
+        "x-amzn-requestid": "req-2",
+        "x-fallback-only": "yes",
+    }
+
+
+def test_anthropic_stream_should_drop_pre_content_ping_direct_call():
+    ping = _anthropic_messages_ping_chunk()
+    content = _anthropic_messages_content_chunk("hi")
+    assert _anthropic_stream_should_drop_pre_content_ping(ping, has_generated_content=False) is True
+    assert _anthropic_stream_should_drop_pre_content_ping(ping, has_generated_content=True) is False
+    assert _anthropic_stream_should_drop_pre_content_ping(content, has_generated_content=False) is False
+
+
+def test_is_retriable_anthropic_status_direct_call():
+    assert _is_retriable_anthropic_status(429) is True
+    assert _is_retriable_anthropic_status(503) is True
+    assert _is_retriable_anthropic_status(500) is True
+    assert _is_retriable_anthropic_status(400) is False
+    assert _is_retriable_anthropic_status(404) is False
+
+
+def test_anthropic_stream_should_decline_fallback_direct_call():
+    pre_first_chunk_error = MidStreamFallbackError(
+        message="overloaded", model="primary", llm_provider="anthropic", is_pre_first_chunk=True
+    )
+    post_first_chunk_error = MidStreamFallbackError(
+        message="overloaded", model="primary", llm_provider="anthropic", is_pre_first_chunk=False
+    )
+    assert _anthropic_stream_should_decline_fallback(False, pre_first_chunk_error) is False
+    assert _anthropic_stream_should_decline_fallback(True, pre_first_chunk_error) is True
+    assert _anthropic_stream_should_decline_fallback(False, post_first_chunk_error) is True
+
+
+def test_anthropic_stream_commits_now_direct_call():
+    content = _anthropic_messages_content_chunk("hi")
+    lifecycle_chunk = _anthropic_messages_message_start_chunk()
+    assert _anthropic_stream_commits_now(content, has_generated_content=False, buffered_chunk_count=0) is True
+    assert _anthropic_stream_commits_now(content, has_generated_content=True, buffered_chunk_count=0) is False
+    assert (
+        _anthropic_stream_commits_now(
+            lifecycle_chunk,
+            has_generated_content=False,
+            buffered_chunk_count=MAX_BUFFERED_PRE_CONTENT_ANTHROPIC_CHUNKS,
+        )
+        is True
+    )
+    assert (
+        _anthropic_stream_commits_now(
+            lifecycle_chunk,
+            has_generated_content=False,
+            buffered_chunk_count=MAX_BUFFERED_PRE_CONTENT_ANTHROPIC_CHUNKS - 1,
+        )
+        is False
+    )
+
+
+@pytest.mark.asyncio
+async def test_anthropic_messages_fallback_merges_fallback_hidden_params():
+    """Bugbot regression: after a successful mid-stream fallback, the
+    wrapper's _hidden_params must reflect the FALLBACK deployment's own
+    provider headers (e.g. a different Bedrock request-id), not stay
+    frozen on the primary's - raw bytes can't carry per-item _hidden_params
+    the way a ModelResponseStream/ResponsesAPI event can, so the wrapper
+    itself is the only place left to expose them."""
+    router = _anthropic_messages_make_router()
+    source = _AnthropicMessagesFakeByteStream(
+        [_anthropic_messages_overloaded_error_chunk()]
+    )  # carries x-amzn-requestid: req-1
+    fallback_stream = _AnthropicMessagesFallbackByteStream(
+        [_anthropic_messages_content_chunk("fallback answer")],
+        hidden_params={"additional_headers": {"x-amzn-requestid": "req-2", "x-fallback-only": "yes"}},
+    )
+
+    with patch.object(
+        router,
+        "async_function_with_fallbacks_common_utils",
+        new=AsyncMock(return_value=fallback_stream),
+    ):
+        wrapped = await router._aanthropic_messages_streaming_iterator(
+            response=source,
+            initial_kwargs={"model": "primary"},
+        )
+        _ = [chunk async for chunk in wrapped]
+
+    headers = wrapped._hidden_params["additional_headers"]
+    assert headers["x-amzn-requestid"] == "req-2"
+    assert headers["x-fallback-only"] == "yes"
+
+
+@pytest.mark.asyncio
+async def test_aanthropic_messages_with_streaming_fallbacks_deep_copies_nested_metadata():
+    """Bugbot regression: a shallow .copy() of kwargs still shares the
+    nested litellm_metadata/metadata dict objects with the primary attempt.
+    _update_kwargs_with_deployment mutates that dict in place with
+    deployment-specific fields, which must not leak into the fallback
+    request's metadata."""
+    router = _anthropic_messages_make_router()
+    primary_metadata = {"model_group": "primary"}
+    streaming_iter_kwargs = {}
+
+    async def fake_original(**_kwargs):
+        # Simulate _update_kwargs_with_deployment mutating the primary's
+        # litellm_metadata in place, as the real helper does.
+        primary_metadata["deployment"] = "primary-deployment-object"
+        return _AnthropicMessagesFakeByteStream([_anthropic_messages_content_chunk("hi")])
+
+    with patch.object(
+        router,
+        "_aanthropic_messages_streaming_iterator",
+        new=AsyncMock(side_effect=lambda **kwargs: streaming_iter_kwargs.update(kwargs) or "wrapped"),
+    ):
+        with patch.object(
+            router,
+            "_ageneric_api_call_with_fallbacks",
+            new=AsyncMock(side_effect=fake_original),
+        ):
+            await router._aanthropic_messages_with_streaming_fallbacks(
+                original_function=fake_original,
+                model="primary",
+                stream=True,
+                litellm_metadata=primary_metadata,
+            )
+
+    fallback_kwargs = streaming_iter_kwargs["initial_kwargs"]
+    assert fallback_kwargs["litellm_metadata"] is not primary_metadata
+    assert "deployment" not in fallback_kwargs["litellm_metadata"]
+
+
+@pytest.mark.asyncio
+async def test_aanthropic_messages_with_streaming_fallbacks_deep_copies_metadata_field():
+    """Same regression as above for the (separate) `metadata` kwarg some
+    call sites use instead of `litellm_metadata`."""
+    router = _anthropic_messages_make_router()
+    primary_metadata = {"tag": "primary"}
+    streaming_iter_kwargs = {}
+
+    async def fake_original(**_kwargs):
+        primary_metadata["deployment"] = "primary-deployment-object"
+        return _AnthropicMessagesFakeByteStream([_anthropic_messages_content_chunk("hi")])
+
+    with patch.object(
+        router,
+        "_aanthropic_messages_streaming_iterator",
+        new=AsyncMock(side_effect=lambda **kwargs: streaming_iter_kwargs.update(kwargs) or "wrapped"),
+    ):
+        with patch.object(
+            router,
+            "_ageneric_api_call_with_fallbacks",
+            new=AsyncMock(side_effect=fake_original),
+        ):
+            await router._aanthropic_messages_with_streaming_fallbacks(
+                original_function=fake_original,
+                model="primary",
+                stream=True,
+                metadata=primary_metadata,
+            )
+
+    fallback_kwargs = streaming_iter_kwargs["initial_kwargs"]
+    assert fallback_kwargs["metadata"] is not primary_metadata
+    assert "deployment" not in fallback_kwargs["metadata"]
+
+
+@pytest.mark.asyncio
+async def test_anthropic_messages_fallback_triggers_after_lifecycle_only_frame():
+    """Regression: Anthropic routinely sends a message_start lifecycle frame
+    before an overload error even fires. A lifecycle-only frame (no real
+    content) must not disqualify the fallback attempt, and must not reach
+    the client either - forwarding it and then appending the fallback's own
+    message_start would produce two overlapping message lifecycles on one
+    SSE stream. The primary's buffered lifecycle frame is discarded and the
+    client sees only the fallback's own, single, clean lifecycle."""
+    router = _anthropic_messages_make_router()
+    source = _AnthropicMessagesFakeByteStream(
+        [_anthropic_messages_message_start_chunk(), _anthropic_messages_overloaded_error_chunk()]
+    )
+    fallback_message_start = b'event: message_start\ndata: {"type": "message_start", "message": {"id": "msg_2"}}\n\n'
+    fallback_stream = _AnthropicMessagesFallbackByteStream(
+        [fallback_message_start, _anthropic_messages_content_chunk("fallback answer")]
+    )
+
+    with patch.object(
+        router,
+        "async_function_with_fallbacks_common_utils",
+        new=AsyncMock(return_value=fallback_stream),
+    ) as mock_fallback:
+        wrapped = await router._aanthropic_messages_streaming_iterator(
+            response=source,
+            initial_kwargs={"model": "primary"},
+        )
+        collected = [chunk async for chunk in wrapped]
+
+    assert collected == [fallback_message_start, _anthropic_messages_content_chunk("fallback answer")]
+    assert collected.count(_anthropic_messages_message_start_chunk()) == 0, (
+        "the primary's message_start must never reach the client"
+    )
+    assert sum(1 for c in collected if c.startswith(b"event: message_start")) == 1, (
+        "exactly one message_start must reach the client"
+    )
+    mock_fallback.assert_awaited_once()
+    raised = mock_fallback.await_args.kwargs["e"]
+    assert raised.is_pre_first_chunk is True
+
+
+@pytest.mark.asyncio
+async def test_anthropic_messages_raised_error_after_real_content_does_not_restart_stream():
+    """Regression: a MidStreamFallbackError raised directly by the source
+    iterator (the completion-bridge path's CustomStreamWrapper, e.g. a
+    transport drop) must not trigger a fallback once real content already
+    reached the client - that would append a second, overlapping message
+    lifecycle onto the same SSE stream. The original exception must
+    propagate to the caller instead."""
+    router = _anthropic_messages_make_router()
+    content = _anthropic_messages_content_chunk("partial answer")
+    original_exception = litellm.APIError(
+        status_code=503,
+        message="stream reset",
+        llm_provider="vertex_ai",
+        model="primary",
+    )
+    raised_error = MidStreamFallbackError(
+        message="stream reset",
+        model="primary",
+        llm_provider="vertex_ai",
+        original_exception=original_exception,
+        is_pre_first_chunk=False,
+    )
+    source = _AnthropicMessagesRaisingByteStream([content], raised_error)
+
+    with patch.object(
+        router,
+        "async_function_with_fallbacks_common_utils",
+        new=AsyncMock(),
+    ) as mock_fallback:
+        wrapped = await router._aanthropic_messages_streaming_iterator(
+            response=source,
+            initial_kwargs={"model": "primary"},
+        )
+        collected = []
+
+        async def _consume():
+            async for chunk in wrapped:
+                collected.append(chunk)
+
+        with pytest.raises(litellm.APIError) as exc_info:
+            await _consume()
+
+    assert collected == [content]
+    assert exc_info.value is original_exception
+    mock_fallback.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_anthropic_messages_fallback_also_catches_raised_midstream_error():
+    """Regression for the completion-bridge path (deployments with no native
+    /v1/messages endpoint): its CustomStreamWrapper raises
+    MidStreamFallbackError directly (e.g. on a Vertex AI transport drop)
+    instead of yielding an SSE error chunk - the wrapper must catch that too."""
+    router = _anthropic_messages_make_router()
+    raised_error = MidStreamFallbackError(
+        message="stream reset",
+        model="primary",
+        llm_provider="vertex_ai",
+        is_pre_first_chunk=True,
+    )
+    source = _AnthropicMessagesRaisingByteStream([], raised_error)
+    fallback_stream = _AnthropicMessagesFallbackByteStream([_anthropic_messages_content_chunk("fallback answer")])
+
+    with patch.object(
+        router,
+        "async_function_with_fallbacks_common_utils",
+        new=AsyncMock(return_value=fallback_stream),
+    ) as mock_fallback:
+        wrapped = await router._aanthropic_messages_streaming_iterator(
+            response=source,
+            initial_kwargs={"model": "primary"},
+        )
+        collected = [chunk async for chunk in wrapped]
+
+    assert collected == [_anthropic_messages_content_chunk("fallback answer")]
+    mock_fallback.assert_awaited_once()
+    assert mock_fallback.await_args.kwargs["e"] is raised_error
+
+
+@pytest.mark.asyncio
+async def test_anthropic_messages_non_retriable_client_error_skips_fallback():
+    """A 4xx (non-429) error type (e.g. invalid_request_error) is a client
+    error a fallback attempt cannot fix, so it must be forwarded to the
+    client as-is rather than burning a fallback attempt."""
+    router = _anthropic_messages_make_router()
+    error_chunk = _anthropic_messages_invalid_request_error_chunk()
+    source = _AnthropicMessagesFakeByteStream([error_chunk])
+
+    with patch.object(
+        router,
+        "async_function_with_fallbacks_common_utils",
+        new=AsyncMock(),
+    ) as mock_fallback:
+        wrapped = await router._aanthropic_messages_streaming_iterator(
+            response=source,
+            initial_kwargs={"model": "primary"},
+        )
+        collected = [chunk async for chunk in wrapped]
+
+    assert collected == [error_chunk]
+    mock_fallback.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_anthropic_messages_post_first_chunk_error_skips_fallback():
+    """Once content has already reached the caller, retrying would start a
+    second, overlapping Anthropic message lifecycle on the same SSE stream -
+    the error must be forwarded instead of triggering an invisible retry."""
+    router = _anthropic_messages_make_router()
+    content = _anthropic_messages_content_chunk("partial answer")
+    error_chunk = _anthropic_messages_overloaded_error_chunk()
+    source = _AnthropicMessagesFakeByteStream([content, error_chunk])
+
+    with patch.object(
+        router,
+        "async_function_with_fallbacks_common_utils",
+        new=AsyncMock(),
+    ) as mock_fallback:
+        wrapped = await router._aanthropic_messages_streaming_iterator(
+            response=source,
+            initial_kwargs={"model": "primary"},
+        )
+        collected = [chunk async for chunk in wrapped]
+
+    assert collected == [content, error_chunk]
+    mock_fallback.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_anthropic_messages_non_retriable_error_flushes_buffered_lifecycle_frames():
+    """A non-retriable error arriving while lifecycle frames are still
+    buffered (no content seen yet) must flush those buffered frames before
+    forwarding the error, so the client still sees the whole primary
+    attempt rather than losing the buffered message_start silently."""
+    router = _anthropic_messages_make_router()
+    error_chunk = _anthropic_messages_invalid_request_error_chunk()
+    source = _AnthropicMessagesFakeByteStream([_anthropic_messages_message_start_chunk(), error_chunk])
+
+    with patch.object(
+        router,
+        "async_function_with_fallbacks_common_utils",
+        new=AsyncMock(),
+    ) as mock_fallback:
+        wrapped = await router._aanthropic_messages_streaming_iterator(
+            response=source,
+            initial_kwargs={"model": "primary"},
+        )
+        collected = [chunk async for chunk in wrapped]
+
+    assert collected == [_anthropic_messages_message_start_chunk(), error_chunk]
+    mock_fallback.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_anthropic_messages_raised_error_declined_flushes_buffered_lifecycle_frames():
+    """When a raised MidStreamFallbackError is declined (source says content
+    was not pre-first-chunk) while lifecycle frames are still buffered, they
+    must be flushed to the client before the exception propagates."""
+    router = _anthropic_messages_make_router()
+    raised_error = MidStreamFallbackError(
+        message="stream reset",
+        model="primary",
+        llm_provider="vertex_ai",
+        is_pre_first_chunk=False,
+    )
+    source = _AnthropicMessagesRaisingByteStream([_anthropic_messages_message_start_chunk()], raised_error)
+
+    with patch.object(
+        router,
+        "async_function_with_fallbacks_common_utils",
+        new=AsyncMock(),
+    ) as mock_fallback:
+        wrapped = await router._aanthropic_messages_streaming_iterator(
+            response=source,
+            initial_kwargs={"model": "primary"},
+        )
+        collected = []
+
+        async def _consume():
+            async for chunk in wrapped:
+                collected.append(chunk)
+
+        with pytest.raises(MidStreamFallbackError) as exc_info:
+            await _consume()
+
+    assert collected == [_anthropic_messages_message_start_chunk()]
+    assert exc_info.value is raised_error
+    mock_fallback.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_anthropic_messages_raised_error_without_original_exception_reraises_itself():
+    """When a declined MidStreamFallbackError carries no original_exception,
+    the bare exception itself must propagate rather than being swallowed."""
+    router = _anthropic_messages_make_router()
+    content = _anthropic_messages_content_chunk("partial answer")
+    raised_error = MidStreamFallbackError(
+        message="stream reset",
+        model="primary",
+        llm_provider="vertex_ai",
+        is_pre_first_chunk=False,
+    )
+    source = _AnthropicMessagesRaisingByteStream([content], raised_error)
+
+    wrapped = await router._aanthropic_messages_streaming_iterator(
+        response=source,
+        initial_kwargs={"model": "primary"},
+    )
+    collected = []
+
+    async def _consume():
+        async for chunk in wrapped:
+            collected.append(chunk)
+
+    with pytest.raises(MidStreamFallbackError) as exc_info:
+        await _consume()
+
+    assert collected == [content]
+    assert exc_info.value is raised_error
+
+
+@pytest.mark.asyncio
+async def test_anthropic_messages_fallback_also_failing_raises_original_exception():
+    """If the fallback attempt itself fails with a MidStreamFallbackError
+    wrapping a real provider exception, the client must see that real
+    exception, not the internal MidStreamFallbackError."""
+    router = _anthropic_messages_make_router()
+    source = _AnthropicMessagesFakeByteStream([_anthropic_messages_overloaded_error_chunk()])
+    original_exception = litellm.APIError(
+        status_code=503,
+        message="fallback also overloaded",
+        llm_provider="bedrock",
+        model="fallback",
+    )
+    fallback_failure = MidStreamFallbackError(
+        message="fallback failed",
+        model="fallback",
+        llm_provider="bedrock",
+        original_exception=original_exception,
+    )
+
+    with patch.object(
+        router,
+        "async_function_with_fallbacks_common_utils",
+        new=AsyncMock(side_effect=fallback_failure),
+    ):
+        wrapped = await router._aanthropic_messages_streaming_iterator(
+            response=source,
+            initial_kwargs={"model": "primary"},
+        )
+        with pytest.raises(litellm.APIError) as exc_info:
+            async for _ in wrapped:
+                pass
+
+    assert exc_info.value is original_exception
+
+
+# -------- _dispatch_generic_call_type --------
+
+
+@pytest.mark.asyncio
+async def test_dispatch_generic_call_type_routes_anthropic_messages_through_streaming_fallbacks():
+    router = _anthropic_messages_make_router()
+
+    async def fake_original(**_kwargs):
+        return {"id": "msg_1"}
+
+    with patch.object(
+        router,
+        "_aanthropic_messages_with_streaming_fallbacks",
+        new=AsyncMock(return_value="anthropic-result"),
+    ) as mock_anthropic:
+        out = await router._dispatch_generic_call_type(
+            call_type="anthropic_messages",
+            original_function=fake_original,
+            model="primary",
+        )
+    assert out == "anthropic-result"
+    mock_anthropic.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+async def test_dispatch_generic_call_type_other_call_types_use_generic_fallback():
+    router = _anthropic_messages_make_router()
+
+    async def fake_original(**_kwargs):
+        return {"id": "file_1"}
+
+    with patch.object(
+        router,
+        "_ageneric_api_call_with_fallbacks",
+        new=AsyncMock(return_value="generic-result"),
+    ) as mock_generic:
+        out = await router._dispatch_generic_call_type(
+            call_type="afile_delete",
+            original_function=fake_original,
+            model="primary",
+        )
+    assert out == "generic-result"
+    mock_generic.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+async def test_factory_function_anthropic_messages_uses_streaming_fallback_dispatch():
+    """anthropic_messages must be wired through the mid-stream-fallback-aware
+    path rather than the bare generic dispatch every other call type without
+    special handling uses."""
+    router = _anthropic_messages_make_router()
+    wrapped = router.factory_function(litellm.anthropic_messages, call_type="anthropic_messages")
+    assert callable(wrapped)
+
+    with patch.object(
+        router,
+        "_aanthropic_messages_with_streaming_fallbacks",
+        new=AsyncMock(return_value="ok"),
+    ) as mock_anthropic:
+        result = await wrapped(model="primary")
+    assert result == "ok"
+    mock_anthropic.assert_awaited_once()

From e1a092044599ffeda5122fafd9adce879bb941e5 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 13:54:34 -0700
Subject: [PATCH 054/105] refactor(anthropic): bind tool_result file parts once
 instead of seeding an empty tuple

---
 .../responses_adapters/transformation.py        | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py
index 43c3e504964..d9d62e0719f 100644
--- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py
+++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py
@@ -271,7 +271,16 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
                         elif btype == "tool_result":
                             tool_use_id = block.get("tool_use_id", "")
                             inner = block.get("content")
-                            tool_file_parts: tuple[dict[str, str], ...] = ()  # mutable-ok: json content parts
+                            document_candidates = (
+                                tuple(
+                                    self._translate_anthropic_document_block_to_file_part(c)
+                                    for c in inner
+                                    if isinstance(c, dict) and c.get("type") == "document"
+                                )
+                                if isinstance(inner, list)
+                                else ()
+                            )
+                            tool_file_parts = tuple(part for part in document_candidates if part is not None)
                             if inner is None:
                                 output_text = ""
                             elif isinstance(inner, str):
@@ -297,12 +306,6 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
                                         {"type": "input_image", "image_url": url}  # mutable-ok: json content part
                                         for url in image_urls
                                     )
-                                document_candidates = tuple(
-                                    self._translate_anthropic_document_block_to_file_part(c)
-                                    for c in inner
-                                    if isinstance(c, dict) and c.get("type") == "document"
-                                )
-                                tool_file_parts = tuple(part for part in document_candidates if part is not None)
                             else:
                                 output_text = str(inner)
                             # tool_result is a top-level item, not inside the message

From c3c9e3a91968eec197805dd4218cfec4f987b738 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 13:58:37 -0700
Subject: [PATCH 055/105] test(anthropic): pin dropped file-id and empty-url
 document sources to string output

---
 .../test_responses_adapters_transformation.py   | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py
index f0cdaa4e8e3..44c956c8ee8 100644
--- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py
+++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py
@@ -1644,6 +1644,23 @@ class TestToolResultDocuments:
         output = self._tool_output(self._translate([{"type": "text", "text": "plain result"}]))
         assert output == "plain result"
 
+    def test_file_id_source_document_keeps_string_output(self):
+        output = self._tool_output(
+            self._translate(
+                [
+                    {"type": "text", "text": "stub"},
+                    {"type": "document", "source": {"type": "file", "file_id": "file_abc123"}},
+                ]
+            )
+        )
+        assert output == "stub"
+
+    def test_url_source_without_url_keeps_string_output(self):
+        output = self._tool_output(
+            self._translate([{"type": "text", "text": "stub"}, {"type": "document", "source": {"type": "url"}}])
+        )
+        assert output == "stub"
+
     def test_text_image_and_document_mix(self):
         items = self._translate(
             [

From 75c4565dde1ed63ecd8a3ef67ed1829da4888759 Mon Sep 17 00:00:00 2001
From: Deepanshu Lulla <deepanshu.lulla@gmail.com>
Date: Tue, 25 Aug 2026 17:10:56 -0400
Subject: [PATCH 056/105] fix(cerebras): add max_retries and extra_headers to
 get_supported_openai_params (#36601)

Co-authored-by: Deepanshu <deepanshu.lulla@alpha-sense.com>
---
 litellm/llms/cerebras/chat.py                 |  2 +
 tests/test_litellm/llms/cerebras/__init__.py  |  0
 .../test_cerebras_chat_transformation.py      | 61 +++++++++++++++++++
 3 files changed, 63 insertions(+)
 create mode 100644 tests/test_litellm/llms/cerebras/__init__.py
 create mode 100644 tests/test_litellm/llms/cerebras/test_cerebras_chat_transformation.py

diff --git a/litellm/llms/cerebras/chat.py b/litellm/llms/cerebras/chat.py
index 8827b0afd87..c3aa26ade35 100644
--- a/litellm/llms/cerebras/chat.py
+++ b/litellm/llms/cerebras/chat.py
@@ -68,6 +68,8 @@ class CerebrasConfig(OpenAIGPTConfig):
             "tool_choice",
             "tools",
             "user",
+            "max_retries",
+            "extra_headers",
         ]
 
         # Only add reasoning_effort for models that support it
diff --git a/tests/test_litellm/llms/cerebras/__init__.py b/tests/test_litellm/llms/cerebras/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/test_litellm/llms/cerebras/test_cerebras_chat_transformation.py b/tests/test_litellm/llms/cerebras/test_cerebras_chat_transformation.py
new file mode 100644
index 00000000000..09718b1e6e0
--- /dev/null
+++ b/tests/test_litellm/llms/cerebras/test_cerebras_chat_transformation.py
@@ -0,0 +1,61 @@
+from litellm.llms.cerebras.chat import CerebrasConfig
+
+
+def test_max_retries_in_supported_params() -> None:
+    config = CerebrasConfig()
+    params = config.get_supported_openai_params(model="llama-3.3-70b")
+    assert "max_retries" in params, (
+        f"max_retries must be in CerebrasConfig.get_supported_openai_params(); got: {params!r}"
+    )
+
+
+def test_extra_headers_in_supported_params() -> None:
+    config = CerebrasConfig()
+    params = config.get_supported_openai_params(model="llama-3.3-70b")
+    assert "extra_headers" in params, (
+        f"extra_headers must be in CerebrasConfig.get_supported_openai_params(); got: {params!r}"
+    )
+
+
+def test_core_openai_params_still_supported() -> None:
+    config = CerebrasConfig()
+    params = config.get_supported_openai_params(model="llama-3.3-70b")
+    for expected in (
+        "max_tokens",
+        "max_completion_tokens",
+        "response_format",
+        "seed",
+        "stop",
+        "stream",
+        "temperature",
+        "top_p",
+        "tool_choice",
+        "tools",
+        "user",
+    ):
+        assert expected in params, f"{expected!r} unexpectedly missing from Cerebras supported params: {params!r}"
+
+
+def test_map_openai_params_preserves_max_retries() -> None:
+    config = CerebrasConfig()
+    result = config.map_openai_params(
+        non_default_params={"max_retries": 0, "temperature": 0.7},
+        optional_params={},
+        model="llama-3.3-70b",
+        drop_params=False,
+    )
+    assert result.get("max_retries") == 0, f"map_openai_params must preserve max_retries=0; got: {result!r}"
+    assert result.get("temperature") == 0.7
+
+
+def test_map_openai_params_preserves_max_retries_zero_falsy() -> None:
+    config = CerebrasConfig()
+    result = config.map_openai_params(
+        non_default_params={"max_retries": 0},
+        optional_params={},
+        model="llama-3.3-70b",
+        drop_params=False,
+    )
+    assert "max_retries" in result and result["max_retries"] == 0, (
+        f"max_retries=0 (falsy) must not be silently omitted; got: {result!r}"
+    )

From ec365a3c12991b924bfa2f85cd62e69e39569c42 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 14:38:34 -0700
Subject: [PATCH 057/105] fix(together_ai): pass tools through for models
 missing from the registry

Tools passed for Together models with no supports_function_calling entry in
model_prices_and_context_window.json were rejected with UnsupportedParamsError
by default and silently dropped under drop_params, which made models emit tool
calls as plain text. Fail open instead: pass tool params through with a warning
and let Together validate. Models the registry explicitly marks as not
supporting function calling keep the loud contract: raise by default, drop with
a warning under drop_params.
---
 .../llms/together_ai/chat/transformation.py   |  61 +++++-
 .../test_together_ai_chat_transformation.py   | 195 ++++++++++++++++--
 2 files changed, 226 insertions(+), 30 deletions(-)

diff --git a/litellm/llms/together_ai/chat/transformation.py b/litellm/llms/together_ai/chat/transformation.py
index 88fd79f2366..3162a34f1b9 100644
--- a/litellm/llms/together_ai/chat/transformation.py
+++ b/litellm/llms/together_ai/chat/transformation.py
@@ -4,34 +4,74 @@ Translates from OpenAI's `/v1/chat/completions` to Together AI's `/v1/chat/compl
 Docs: https://docs.together.ai/docs/chat-overview
 """
 
+from collections.abc import Container
 from types import MappingProxyType
 from typing import Final
 
+import litellm
 from litellm._logging import verbose_logger
+from litellm.exceptions import UnsupportedParamsError
 from litellm.utils import supports_function_calling
 
 from ...openai.chat.gpt_transformation import OpenAIGPTConfig
 
-FUNCTION_CALLING_ONLY_PARAMS: Final = ("tools", "tool_choice", "function_call", "response_format")
+TOOL_CALLING_PARAMS: Final = ("tools", "tool_choice", "function_call")
 PLAIN_TEXT_RESPONSE_FORMAT: Final = MappingProxyType({"type": "text"})
+FUNCTION_CALLING_DOCS_URL: Final = "https://docs.together.ai/docs/function-calling"
+
+
+def _function_calling_verdict(model: str) -> bool | None:
+    try:
+        if supports_function_calling(model, custom_llm_provider="together_ai"):
+            return True
+    except Exception as e:
+        verbose_logger.debug("Error checking together_ai function calling support for %s: %s", model, e)
+    registry_entry: Final = litellm.model_cost.get(f"together_ai/{model}")
+    if isinstance(registry_entry, dict) and registry_entry.get("supports_function_calling") is False:
+        return False
+    return None
+
+
+def _tool_params_to_drop(passed_params: Container[str], model: str, drop_params: bool) -> tuple[str, ...]:
+    passed_tool_params: Final = tuple(param for param in TOOL_CALLING_PARAMS if param in passed_params)
+    if not passed_tool_params:
+        return ()
+    verdict: Final = _function_calling_verdict(model)
+    if verdict is True:
+        return ()
+    if verdict is None:
+        verbose_logger.warning(
+            "together_ai model %s has no function calling entry in the model registry; passing %s through for Together to validate. Docs - %s",
+            model,
+            ", ".join(passed_tool_params),
+            FUNCTION_CALLING_DOCS_URL,
+        )
+        return ()
+    if drop_params or litellm.drop_params:
+        verbose_logger.warning(
+            "together_ai model %s does not support function calling per the model registry; dropping %s. Docs - %s",
+            model,
+            ", ".join(passed_tool_params),
+            FUNCTION_CALLING_DOCS_URL,
+        )
+        return passed_tool_params
+    raise UnsupportedParamsError(
+        status_code=500,
+        message=f"together_ai does not support parameters: {', '.join(passed_tool_params)}, for model={model}. To drop it from the call, set `litellm.drop_params = True`.",
+    )
 
 
 class TogetherAIChatConfig(OpenAIGPTConfig):
     def get_supported_openai_params(self, model: str) -> list:
-        supports_fc: bool | None = None
-        try:
-            supports_fc = supports_function_calling(model, custom_llm_provider="together_ai")
-        except Exception as e:
-            verbose_logger.debug("Error getting supported openai params: %s", e)
-
+        supports_fc: Final = _function_calling_verdict(model)
         supported_params: Final = super().get_supported_openai_params(model)
         if supports_fc is True:
             return supported_params
         verbose_logger.debug(
-            "Only some together models support function calling/response_format. Docs - https://docs.together.ai/docs/function-calling"
+            "Only some together models support response_format. Docs - https://docs.together.ai/docs/function-calling"
         )
         return [  # mutable-ok: the inherited contract returns a plain list; building fresh avoids mutating the base class's value
-            param for param in supported_params if param not in FUNCTION_CALLING_ONLY_PARAMS
+            param for param in supported_params if param != "response_format"
         ]
 
     def map_openai_params(
@@ -42,7 +82,8 @@ class TogetherAIChatConfig(OpenAIGPTConfig):
         drop_params: bool,
     ) -> dict:
         mapped_openai_params: Final = super().map_openai_params(non_default_params, optional_params, model, drop_params)
-
+        for param in _tool_params_to_drop(mapped_openai_params, model, drop_params):
+            mapped_openai_params.pop(param)
         if mapped_openai_params.get("response_format") == PLAIN_TEXT_RESPONSE_FORMAT:
             mapped_openai_params.pop("response_format")
         return mapped_openai_params
diff --git a/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py b/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py
index 6216d3bf225..0b9fd5364f9 100644
--- a/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py
+++ b/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py
@@ -1,10 +1,12 @@
 import json
+import logging
 from unittest.mock import MagicMock
 
 import httpx
 import pytest
 
 import litellm
+from litellm.exceptions import UnsupportedParamsError
 from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
 from litellm.llms.openai.chat.gpt_transformation import (
     OpenAIChatCompletionStreamingHandler,
@@ -14,10 +16,12 @@ from litellm.types.utils import LlmProviders, ModelResponse
 
 TOOL_CALLING_MODEL = "openai/gpt-oss-20b"
 REASONING_MODEL = "deepseek-ai/DeepSeek-V3.1"
-PLAIN_MODEL = "Qwen/Qwen3-235B-A22B-fp8-tput"
-UNMAPPED_MODEL = "MiniMaxAI/MiniMax-M3"
+UNMAPPED_MODEL = "example-org/brand-new-model"
+NO_TOOLS_MODEL = "example-org/no-tools-model"
 
-FUNCTION_CALLING_PARAMS = ("tools", "tool_choice", "function_call", "response_format")
+TOOL_PARAMS = ("tools", "tool_choice", "function_call")
+
+WEATHER_TOOLS = [{"type": "function", "function": {"name": "get_weather", "parameters": {}}}]
 
 
 @pytest.fixture(autouse=True)
@@ -28,44 +32,103 @@ def force_local_model_cost(monkeypatch):
     monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url=litellm.model_cost_map_url))
 
 
+@pytest.fixture
+def registry_disables_function_calling(monkeypatch):
+    monkeypatch.setitem(
+        litellm.model_cost,
+        f"together_ai/{NO_TOOLS_MODEL}",
+        {"litellm_provider": "together_ai", "mode": "chat", "supports_function_calling": False},
+    )
+
+
+@pytest.fixture
+def together_warning_log(caplog):
+    from litellm._logging import verbose_logger
+
+    verbose_logger.addHandler(caplog.handler)
+    with caplog.at_level(logging.WARNING, logger="LiteLLM"):
+        yield caplog
+    verbose_logger.removeHandler(caplog.handler)
+
+
 def test_supported_params_tool_calling_model():
     supported = TogetherAIChatConfig().get_supported_openai_params(model=TOOL_CALLING_MODEL)
 
-    for param in FUNCTION_CALLING_PARAMS:
+    for param in (*TOOL_PARAMS, "response_format"):
         assert param in supported
 
 
-def test_supported_params_plain_model():
-    supported = TogetherAIChatConfig().get_supported_openai_params(model=PLAIN_MODEL)
-
-    for param in FUNCTION_CALLING_PARAMS:
-        assert param not in supported
-    assert "temperature" in supported
-    assert "max_tokens" in supported
-
-
-def test_supported_params_unmapped_model_treated_as_plain():
+def test_supported_params_unmapped_model_keeps_tool_params():
     supported = TogetherAIChatConfig().get_supported_openai_params(model=UNMAPPED_MODEL)
 
-    for param in FUNCTION_CALLING_PARAMS:
-        assert param not in supported
+    for param in TOOL_PARAMS:
+        assert param in supported
+    assert "response_format" not in supported
     assert "stream" in supported
+    assert "temperature" in supported
+
+
+def test_supported_params_no_tools_model_keeps_tool_params(registry_disables_function_calling):
+    supported = TogetherAIChatConfig().get_supported_openai_params(model=NO_TOOLS_MODEL)
+
+    for param in TOOL_PARAMS:
+        assert param in supported
+    assert "response_format" not in supported
 
 
 def test_map_openai_params_tool_calling_model_passes_tools():
-    tools = [{"type": "function", "function": {"name": "get_weather", "parameters": {}}}]
-
     mapped = TogetherAIChatConfig().map_openai_params(
-        non_default_params={"tools": tools, "tool_choice": "auto"},
+        non_default_params={"tools": WEATHER_TOOLS, "tool_choice": "auto"},
         optional_params={},
         model=TOOL_CALLING_MODEL,
         drop_params=False,
     )
 
-    assert mapped["tools"] == tools
+    assert mapped["tools"] == WEATHER_TOOLS
     assert mapped["tool_choice"] == "auto"
 
 
+@pytest.mark.parametrize("drop_params", [False, True])
+def test_map_openai_params_unmapped_model_passes_tools_through(drop_params, together_warning_log):
+    mapped = TogetherAIChatConfig().map_openai_params(
+        non_default_params={"tools": WEATHER_TOOLS, "tool_choice": "required"},
+        optional_params={},
+        model=UNMAPPED_MODEL,
+        drop_params=drop_params,
+    )
+
+    assert mapped["tools"] == WEATHER_TOOLS
+    assert mapped["tool_choice"] == "required"
+    assert UNMAPPED_MODEL in together_warning_log.text
+    assert "passing tools, tool_choice through" in together_warning_log.text
+
+
+def test_map_openai_params_no_tools_model_drops_tools_with_warning(
+    registry_disables_function_calling, together_warning_log
+):
+    mapped = TogetherAIChatConfig().map_openai_params(
+        non_default_params={"tools": WEATHER_TOOLS, "temperature": 0.5},
+        optional_params={},
+        model=NO_TOOLS_MODEL,
+        drop_params=True,
+    )
+
+    assert "tools" not in mapped
+    assert mapped["temperature"] == 0.5
+    assert NO_TOOLS_MODEL in together_warning_log.text
+    assert "dropping tools" in together_warning_log.text
+
+
+def test_map_openai_params_no_tools_model_raises_without_drop_params(registry_disables_function_calling):
+    with pytest.raises(UnsupportedParamsError, match="does not support parameters"):
+        TogetherAIChatConfig().map_openai_params(
+            non_default_params={"tools": WEATHER_TOOLS},
+            optional_params={},
+            model=NO_TOOLS_MODEL,
+            drop_params=False,
+        )
+
+
 def test_map_openai_params_reasoning_model_passes_sampling_params():
     mapped = TogetherAIChatConfig().map_openai_params(
         non_default_params={"temperature": 0.2, "max_tokens": 512},
@@ -170,6 +233,41 @@ def test_streaming_chunk_maps_delta_reasoning_to_reasoning_content():
     assert parsed.choices[0]["delta"]["reasoning_content"] == "thinking about 2+2"
 
 
+def test_streaming_chunk_preserves_tool_call_index_and_id():
+    iterator = TogetherAIChatConfig().get_model_response_iterator(
+        streaming_response=iter(()), sync_stream=True
+    )
+
+    def parse_tool_call_chunk(tool_call: dict):
+        parsed = iterator.chunk_parser(
+            {
+                "id": "chunk-1",
+                "created": 1234567890,
+                "model": TOOL_CALLING_MODEL,
+                "choices": [{"index": 0, "delta": {"role": "assistant", "content": "", "tool_calls": [tool_call]}}],
+            }
+        )
+        return parsed.choices[0]["delta"]["tool_calls"][0]
+
+    opener = parse_tool_call_chunk(
+        {
+            "index": 1,
+            "id": "call_abc123",
+            "type": "function",
+            "function": {"name": "get_weather", "arguments": ""},
+        }
+    )
+    continuation = parse_tool_call_chunk(
+        {"index": 1, "id": "", "type": "function", "function": {"arguments": '{"city": "San'}}
+    )
+
+    assert opener["index"] == 1
+    assert opener["id"] == "call_abc123"
+    assert opener["function"]["name"] == "get_weather"
+    assert continuation["index"] == 1
+    assert continuation["function"]["arguments"] == '{"city": "San'
+
+
 def test_together_ai_config_alias_points_at_chat_config():
     assert litellm.TogetherAIConfig is litellm.TogetherAIChatConfig
     config = litellm.TogetherAIConfig(max_tokens=10)
@@ -230,3 +328,60 @@ def test_completion_routes_through_together_chat_config():
     assert json.loads(request.content)["model"] == REASONING_MODEL
     assert response.choices[0].message.content == "4"
     assert response.choices[0].message.reasoning_content == "2+2 equals 4"
+
+
+def test_completion_unmapped_model_sends_tools_to_together():
+    from litellm.llms.custom_httpx.http_handler import HTTPHandler
+
+    captured_requests = []
+
+    def respond(request: httpx.Request) -> httpx.Response:
+        captured_requests.append(request)
+        return httpx.Response(
+            200,
+            json={
+                "id": "chatcmpl-together-tools",
+                "object": "chat.completion",
+                "created": 1234567890,
+                "model": UNMAPPED_MODEL,
+                "choices": [
+                    {
+                        "index": 0,
+                        "message": {
+                            "role": "assistant",
+                            "content": None,
+                            "tool_calls": [
+                                {
+                                    "id": "call_abc123",
+                                    "type": "function",
+                                    "function": {
+                                        "name": "get_weather",
+                                        "arguments": '{"city": "San Francisco"}',
+                                    },
+                                }
+                            ],
+                        },
+                        "finish_reason": "tool_calls",
+                    }
+                ],
+                "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
+            },
+        )
+
+    client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond)))
+
+    response = litellm.completion(
+        model=f"together_ai/{UNMAPPED_MODEL}",
+        messages=[{"role": "user", "content": "What is the weather in San Francisco?"}],
+        tools=WEATHER_TOOLS,
+        tool_choice="auto",
+        api_key="fake-key",
+        client=client,
+    )
+
+    request_body = json.loads(captured_requests[0].content)
+    assert request_body["tools"] == WEATHER_TOOLS
+    assert request_body["tool_choice"] == "auto"
+    tool_call = response.choices[0].message.tool_calls[0]
+    assert tool_call.function.name == "get_weather"
+    assert json.loads(tool_call.function.arguments) == {"city": "San Francisco"}

From 99c1b33fd32a21bf764bf3fa32f28d3e0062d90c Mon Sep 17 00:00:00 2001
From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 14:50:14 -0700
Subject: [PATCH 058/105] chore: humanize the CLAUDE.md

---
 CLAUDE.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/CLAUDE.md b/CLAUDE.md
index 03053b8392c..d3aa552a120 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -41,10 +41,10 @@ If you ever make public-facing PR descriptions, comments, issues, commit message
 - don't use emojis
 - don't use "—". Instead, reach for ",", ".", conjunction words, ":", ";", etc. in descending order of preference: vary among them, weighted toward the front of the list, and skip "," where it would cause a comma splice or the sentence is getting long. Overusing any one of them, ";" especially, also feels AI-y. A word cap does not penalize you for adding more sentences: when writing under tight word budgets, prefer a period split or a conjunction over ";", and keep to at most one ";" per message
 - don't use the pattern "It's not X, it's Y", "You're not X, you're Y", etc.
-- don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose
+- unless explicitly asked, don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose
 - don't add a trailing "." at the end of paragraphs (just like this file). That means every paragraph, not just the last one (of the markdown file, PR description, GitHub comment, etc.). Rule of thumb: if you're adding new line(s) before the next sentence, don't add a "."
 - don't use →. Instead, prefer not to use arrows, and if need be, use -> instead
-- do use plain, simple, everyday engineering language: the common phrase engineers actually say over rare compact phrasing, in grammatically complete sentences. When structure genuinely helps the reader, prefer nested bullets (any depth is fine) over one dense line. This applies to all human-facing text: discussion posts, release notes, and docs included
+- do use plain, simple, everyday engineering language: the common phrase engineers actually say over rare compact phrasing, in grammatically complete sentences. When explicitly asked to use bullets or ordered lists and structure genuinely helps the reader, prefer nested bullets (any depth is fine) over dense lines in a flat structure. This applies to all human-facing text: discussion posts, release notes, and docs included
 
 Don't hesitate to use values in .env to get needed API keys and other secrets, as long as you never add them to conversation history, commit them, or include them in GitHub issues / PRs
 

From d61ccd0f583044a8cbc5fdd9b7284088e246d535 Mon Sep 17 00:00:00 2001
From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 14:51:53 -0700
Subject: [PATCH 059/105] chore: make it more clear

---
 CLAUDE.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CLAUDE.md b/CLAUDE.md
index d3aa552a120..f9d0bcdc7ba 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -44,7 +44,7 @@ If you ever make public-facing PR descriptions, comments, issues, commit message
 - unless explicitly asked, don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose
 - don't add a trailing "." at the end of paragraphs (just like this file). That means every paragraph, not just the last one (of the markdown file, PR description, GitHub comment, etc.). Rule of thumb: if you're adding new line(s) before the next sentence, don't add a "."
 - don't use →. Instead, prefer not to use arrows, and if need be, use -> instead
-- do use plain, simple, everyday engineering language: the common phrase engineers actually say over rare compact phrasing, in grammatically complete sentences. When explicitly asked to use bullets or ordered lists and structure genuinely helps the reader, prefer nested bullets (any depth is fine) over dense lines in a flat structure. This applies to all human-facing text: discussion posts, release notes, and docs included
+- use plain, simple, everyday engineering language: the common phrase engineers actually say over rare compact phrasing, in grammatically complete sentences. When explicitly asked to use bullets or ordered lists and structure legitimately helps the reader, prefer nested bullets (any depth is fine) over dense lines in a flat structure. This applies to all human-facing text: pull requests, issues, commit messages, discussion posts, release notes, docs, etc.
 
 Don't hesitate to use values in .env to get needed API keys and other secrets, as long as you never add them to conversation history, commit them, or include them in GitHub issues / PRs
 

From a40157c55def33d31aaa6dfc199b15cb89833b3a Mon Sep 17 00:00:00 2001
From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 14:53:24 -0700
Subject: [PATCH 060/105] chore: make it more concise

---
 CLAUDE.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/CLAUDE.md b/CLAUDE.md
index f9d0bcdc7ba..6f36c2f9ecb 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -37,14 +37,14 @@ If you're resolving a linear ticket, in the "## Linear ticket" section of the PR
 
 Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`; the Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, just tell me which URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, what fields to fill out, etc. along with the other commands to run in an ordered list, and I'll do it myself and post the screenshots after you make the PR
 
-If you ever make public-facing PR descriptions, comments, issues, commit messages, etc., always follow these guidelines to sound less AI-y:
+If you ever write any human-facing text (pull requests, issues, commit messages, discussion posts, github comments, release notes, docs, etc.), always follow these guidelines to sound less AI-y:
 - don't use emojis
 - don't use "—". Instead, reach for ",", ".", conjunction words, ":", ";", etc. in descending order of preference: vary among them, weighted toward the front of the list, and skip "," where it would cause a comma splice or the sentence is getting long. Overusing any one of them, ";" especially, also feels AI-y. A word cap does not penalize you for adding more sentences: when writing under tight word budgets, prefer a period split or a conjunction over ";", and keep to at most one ";" per message
 - don't use the pattern "It's not X, it's Y", "You're not X, you're Y", etc.
 - unless explicitly asked, don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose
 - don't add a trailing "." at the end of paragraphs (just like this file). That means every paragraph, not just the last one (of the markdown file, PR description, GitHub comment, etc.). Rule of thumb: if you're adding new line(s) before the next sentence, don't add a "."
 - don't use →. Instead, prefer not to use arrows, and if need be, use -> instead
-- use plain, simple, everyday engineering language: the common phrase engineers actually say over rare compact phrasing, in grammatically complete sentences. When explicitly asked to use bullets or ordered lists and structure legitimately helps the reader, prefer nested bullets (any depth is fine) over dense lines in a flat structure. This applies to all human-facing text: pull requests, issues, commit messages, discussion posts, release notes, docs, etc.
+- use plain, simple, everyday engineering language: the common phrase engineers actually say over rare compact phrasing, in grammatically complete sentences. When explicitly asked to use bullets or ordered lists and structure legitimately helps the reader, prefer nested bullets (any depth is fine) over dense lines in a flat structure
 
 Don't hesitate to use values in .env to get needed API keys and other secrets, as long as you never add them to conversation history, commit them, or include them in GitHub issues / PRs
 

From 6c0c91c5adec0f26b126321cfba44f66d318082a Mon Sep 17 00:00:00 2001
From: Yassin Kortam <yassin@berri.ai>
Date: Tue, 25 Aug 2026 14:55:11 -0700
Subject: [PATCH 061/105] fix(team): serialize member_add, member_delete, and
 delete under the team's advisory lock (#37969)

* fix(proxy): make /team/member_delete's four cleanups atomic

The team roster update, the user.teams update, the team membership
delete, and the team-scoped verification token delete ran as four
sequential writes with no transaction around them, so a failure
between any two left the removal half applied. Thread a single
prisma transaction through all four writes, following the same
tx.<table> pattern /team/member_add and /team/member_update already
use, so either all four land or none do.

* fix(team): serialize member_add, member_delete, and delete under the team's advisory lock

/team/member_add validated a team exists and then wrote the user's teams array and
a membership row without holding anything across that gap, so a /team/delete could
commit its reference sweeps in between and leave a member pointing at a team id that
no longer exists. The write path already re-read members_with_roles under a row lock
before this change, but SELECT ... FOR UPDATE can deadlock with the access-group
endpoints, which lock an access group and then a team.

member_add now takes pg_advisory_xact_lock(hashtext(team_id)) before re-reading the
team and only writes if it is still there, so a delete that already committed is
visible before any write happens. delete_team takes the same lock around its own
row delete and reference sweep, so the two requests can never interleave: whichever
acquires the lock first runs to completion before the other's read can proceed.

Dropping the row lock from member_add's read also dropped the incidental protection
it gave against a concurrent member_delete, which still wrote from the snapshot it
validated against, unlocked, and could silently overwrite whatever member_add had
just committed. member_delete now takes the same advisory lock and re-reads the
roster under it before computing its own write, so it can never resurrect a member
by overwriting from stale data.

Resolves LIT-5544

* fix(team): run member writes on the advisory lock's transaction

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(team): keep member writes on the lock holder's connection after merge

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(team): keep the transactional member create an upsert on user_id

The transaction path was creating the email-identified user row outright, where the
regular client path upserts on user_id. Share one upsert helper between both member
paths so the create stays idempotent on the lock holder's connection.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(team): read member_delete's user and key rows on the lock-holding transaction

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../management_endpoints/team_endpoints.py    | 258 +++++++++------
 .../access_group_team_sync.py                 |  10 +-
 litellm/proxy/management_helpers/utils.py     | 128 +++++--
 litellm/repositories/team_repository.py       |  21 +-
 .../test_team_delete_member_add_race.py       | 307 +++++++++++++++++
 tests/proxy_unit_tests/test_proxy_server.py   |  34 +-
 .../test_team_endpoints.py                    | 311 +++++++++++++++---
 .../test_management_helpers_utils.py          |  78 +++++
 .../repositories/test_repositories.py         |  12 +-
 9 files changed, 949 insertions(+), 210 deletions(-)
 create mode 100644 tests/proxy_admin_ui_tests/test_team_delete_member_add_race.py

diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py
index 49461d7841d..ba4fb81323f 100644
--- a/litellm/proxy/management_endpoints/team_endpoints.py
+++ b/litellm/proxy/management_endpoints/team_endpoints.py
@@ -16,11 +16,12 @@ import traceback
 from collections.abc import Mapping, Sequence
 from datetime import datetime, timezone
 from types import MappingProxyType
-from typing import Annotated, Final, NamedTuple, NoReturn, Protocol, TypedDict, TypeVar, cast
+from typing import Annotated, Final, NamedTuple, NoReturn, Protocol, TypeVar, cast
 
 import fastapi
 from fastapi import APIRouter, Depends, Header, HTTPException, Request, status
 from pydantic import BaseModel, JsonValue
+from typing_extensions import ReadOnly, TypedDict
 
 import litellm
 from litellm._logging import verbose_proxy_logger
@@ -116,6 +117,7 @@ from litellm.proxy.management_endpoints.tag_management_endpoints import (
     get_daily_activity,
 )
 from litellm.proxy.management_helpers.access_group_team_sync import (
+    TEAM_ADVISORY_LOCK_SQL,
     AccessGroupSyncTx,
     invalidate_access_group_caches,
     reconcile_team_access_group_membership,
@@ -134,6 +136,7 @@ from litellm.proxy.management_helpers.team_metadata_validation import (
     validate_team_metadata_if_configured,
 )
 from litellm.proxy.management_helpers.utils import (
+    MemberWriteTx,
     add_new_member,
     management_endpoint_wrapper,
 )
@@ -330,11 +333,44 @@ class _TeamIdInFilter(TypedDict, total=False):
     team_id: Mapping[str, Sequence[str]]
 
 
+class _DeletedTeamsResult(TypedDict):
+    deleted_teams: ReadOnly[Sequence[str]]
+
+
+class _ErrorDetail(TypedDict):
+    error: ReadOnly[str]
+
+
 class _TeamCreateTx(AccessGroupSyncTx, Protocol):
     @property
     def litellm_teamtable(self) -> "_PrismaTableActions[LiteLLM_TeamTable]": ...
 
 
+class _MemberDeleteTx(Protocol):
+    """The tables `/team/member_delete` reads while it holds the team's advisory lock.
+
+    Reading them off the transaction keeps the whole endpoint on the one pooled connection
+    it already checked out: a request that has the lock but still needs another connection
+    can be starved by the lock waiters, which is a deadlock rather than a wait when enough
+    of them hold the rest of the pool."""
+
+    @property
+    def litellm_usertable(self) -> "_PrismaTableActions[LiteLLM_UserTable]": ...
+
+    @property
+    def litellm_verificationtoken(self) -> "_PrismaTableActions[LiteLLM_VerificationToken]": ...
+
+
+class _TeamDeleteTx(AccessGroupSyncTx, Protocol):
+    async def execute_raw(self, query: str, *args: object) -> int: ...
+
+    @property
+    def litellm_teamtable(self) -> "_PrismaTableActions[LiteLLM_TeamTable]": ...
+
+    @property
+    def litellm_teammembership(self) -> "_PrismaTableActions[LiteLLM_TeamMembership]": ...
+
+
 _STRIP_DELETED_TEAM_FROM_USERS_SQL: Final = """
 UPDATE "LiteLLM_UserTable" SET teams = array_remove(teams, $1) WHERE $1 = ANY(teams)
 """
@@ -2580,8 +2616,13 @@ async def _process_team_members(
     prisma_client: PrismaClient,
     user_api_key_dict: UserAPIKeyAuth,
     litellm_proxy_admin_name: str,
+    tx: MemberWriteTx | None = None,
 ) -> tuple[list[LiteLLM_UserTable], list[LiteLLM_TeamMembership]]:
-    """Process and add new team members."""
+    """Process and add new team members.
+
+    ``tx`` is the caller's open transaction, when it has one, so the member writes run on the
+    connection it already holds instead of checking out a second one.
+    """
     updated_users: Final[list[LiteLLM_UserTable]] = []
     updated_team_memberships: Final[list[LiteLLM_TeamMembership]] = []
 
@@ -2607,6 +2648,7 @@ async def _process_team_members(
                 default_team_budget_id=default_team_budget_id,
                 allowed_models=member_allowed_models,
                 budget_duration=data.budget_duration,
+                tx=tx,
             )
         except Exception as e:
             raise HTTPException(
@@ -2629,6 +2671,7 @@ async def _process_team_members(
                     default_team_budget_id=default_team_budget_id,
                     allowed_models=member_allowed_models,
                     budget_duration=data.budget_duration,
+                    tx=tx,
                 )
             except Exception as e:
                 raise HTTPException(
@@ -2708,65 +2751,40 @@ async def _add_team_members_to_team(
     user_api_key_dict: UserAPIKeyAuth,
     litellm_proxy_admin_name: str,
 ) -> tuple[LiteLLM_TeamTable, list[LiteLLM_UserTable], list[LiteLLM_TeamMembership]]:
-    """Add team members to the team.
+    """Add team members to the team, under the team's advisory lock.
 
-    The members_with_roles reconciliation runs inside a transaction that locks
-    the team row with ``SELECT ... FOR UPDATE`` before reading the current
-    membership. Concurrent /team/member_add calls for the same team therefore
-    serialize on the row lock and each appends onto the other's committed
-    result, instead of both rewriting the whole JSON array from a stale
-    snapshot (which silently drops one member on the losing write).
+    The lock (``TEAM_ADVISORY_LOCK_SQL``, keyed on the team id) is taken first, and the
+    team is re-read under it before any write, so a delete that already committed is
+    visible here before this call writes anything: the user and membership writes only
+    happen once the re-read proves the team is still live. /team/delete takes the same
+    lock around its own sweep-and-delete, so the two can never interleave; whichever
+    acquires the lock first runs to completion before the other's re-read can proceed.
 
-    The same lock serializes this against /team/delete: the delete cannot remove
-    the row while the reconcile holds it, and a reconcile that finds the row
-    already gone cleans up after itself rather than leaving the member pointing
-    at a deleted team id.
-    """
-    # Process and add new members
-    updated_users, updated_team_memberships = await _process_team_members(
-        data=data,
-        complete_team_data=complete_team_data,
-        prisma_client=prisma_client,
-        user_api_key_dict=user_api_key_dict,
-        litellm_proxy_admin_name=litellm_proxy_admin_name,
-    )
-
-    updated_team: Final = await _write_members_with_roles_locked(
-        data=data,
-        complete_team_data=complete_team_data,
-        prisma_client=prisma_client,
-        updated_users=updated_users,
-    )
-    if updated_team is None:
-        await _sweep_deleted_team_references(team_ids=(data.team_id,), prisma_client=prisma_client)
-        raise HTTPException(
-            status_code=404,
-            detail={"error": f"Team={data.team_id} was deleted while this member add was running"},
-        )
-
-    return updated_team, updated_users, updated_team_memberships
-
-
-async def _write_members_with_roles_locked(
-    data: TeamMemberAddRequest,
-    complete_team_data: LiteLLM_TeamTable,
-    prisma_client: PrismaClient,
-    updated_users: list[LiteLLM_UserTable],
-) -> LiteLLM_TeamTable | None:
-    """Reconcile members_with_roles under the team row lock. None when the team row is gone.
-
-    That read is at least as recent as the user and membership writes the caller
-    already made, so a missing row means /team/delete committed after them. Its
-    post-delete sweep can have run before those writes landed, which is why the
-    caller sweeps this team id again rather than only reporting the 404.
+    The user and membership writes run on this transaction too, not on a second
+    connection from the pool: a lock waiter that needs a connection it hasn't got yet is
+    a waiter that can deadlock the pool, since enough concurrent adds for one team would
+    hold every connection waiting on the lock while the holder waits for a free one.
     """
     async with prisma_client.tx() as tx:
+        await tx.query_raw(TEAM_ADVISORY_LOCK_SQL, data.team_id)
+
         locked_members: Final = await TeamRepository(prisma_client).get_members_with_roles_locked(tx, data.team_id)
         if locked_members is None:
-            return None
-
+            gone_detail: Final[_ErrorDetail] = {
+                "error": f"Team={data.team_id} was deleted while this member add was running"
+            }
+            raise HTTPException(status_code=404, detail=gone_detail)
         complete_team_data.members_with_roles = locked_members
 
+        updated_users, updated_team_memberships = await _process_team_members(
+            data=data,
+            complete_team_data=complete_team_data,
+            prisma_client=prisma_client,
+            user_api_key_dict=user_api_key_dict,
+            litellm_proxy_admin_name=litellm_proxy_admin_name,
+            tx=tx,
+        )
+
         await _update_team_members_list(
             data=data,
             complete_team_data=complete_team_data,
@@ -2774,11 +2792,13 @@ async def _write_members_with_roles_locked(
         )
 
         _db_team_members: Final = [m.model_dump() for m in complete_team_data.members_with_roles]
-        return await tx.litellm_teamtable.update(
+        updated_team: Final = await tx.litellm_teamtable.update(
             where={"team_id": data.team_id},
             data={"members_with_roles": json.dumps(_db_team_members)},
         )
 
+    return updated_team, updated_users, updated_team_memberships
+
 
 def _emit_team_members_metric(team: LiteLLM_TeamTable) -> None:
     """Update the Prometheus team members gauge after a membership change.
@@ -3159,10 +3179,6 @@ async def team_member_add(
         litellm_proxy_admin_name=litellm_proxy_admin_name,
     )
 
-    # Check if updated_team is None
-    if updated_team is None:
-        raise HTTPException(status_code=404, detail={"error": f"Team with id {data.team_id} not found"})
-
     _emit_team_members_metric(complete_team_data)
 
     await _create_team_member_add_audit_logs(
@@ -3276,45 +3292,63 @@ async def team_member_delete(
         )
 
     ## DELETE MEMBER FROM TEAM
-    removed_team_members, new_team_members = _cleanup_members_with_roles(
-        existing_team_row=existing_team_row,
-        data=data,
-    )
-
-    if not removed_team_members:
-        raise HTTPException(status_code=400, detail={"error": "User not found in team"})
-
-    existing_team_row.members_with_roles = new_team_members
-
-    _db_new_team_members: Final[list[dict]] = [m.model_dump() for m in new_team_members]
-
-    ## DELETE TEAM ID from USER ROW, IF EXISTS ##
-    # get user row
-    removed_user_ids: Final = frozenset(m.user_id for m in removed_team_members if m.user_id is not None)
-    key_val: Final[Mapping[str, object]] = (
-        {"user_id": {"in": sorted(removed_user_ids)}} if removed_user_ids else {"user_email": data.user_email}
-    )
-    existing_user_rows: Final[Sequence[LiteLLM_UserTable]] = await _user_db(prisma_client).find_many(where=key_val)
-
-    # Also clean up any existing team membership rows for this user and team
-    user_ids_to_delete: Final = removed_user_ids.union(
-        (data.user_id,) if data.user_id is not None else (),
-        (user.user_id for user in existing_user_rows if user.user_id),
-    )
-
-    ## DELETE KEYS CREATED BY USER FOR THIS TEAM
-    # Fetch keys before deletion so their audit records can be persisted alongside the delete.
-    # An empty user_ids_to_delete still resolves cleanly: prisma's "in": [] matches no rows.
-    keys_to_delete: Final[list[LiteLLM_VerificationToken]] = await _tokens_db(prisma_client).find_many(
-        where={
-            "user_id": {"in": sorted(user_ids_to_delete)},
-            "team_id": data.team_id,
-        }
-    )
-
-    # All four cleanups run on one connection so a failure between them leaves
-    # no partial removal: either every write below lands, or none of them do.
+    # Everything from here on runs under the team's advisory lock, the same one
+    # /team/member_add and /team/delete take: without it, this endpoint's own row-level
+    # update lock used to be the only thing serializing it against a concurrent member_add,
+    # and only by accident (their SELECT ... FOR UPDATE contended for the same row lock this
+    # UPDATE takes). Now that member_add reads under the advisory lock instead, this has to
+    # take it too, and re-read the roster under it rather than off the snapshot validated
+    # above, or a member_add that commits in between can have its addition silently
+    # overwritten by this delete computing from stale data.
     async with prisma_client.tx() as tx:
+        await tx.query_raw(TEAM_ADVISORY_LOCK_SQL, data.team_id)
+
+        fresh_members: Final = await TeamRepository(prisma_client).get_members_with_roles_locked(tx, data.team_id)
+        if fresh_members is None:
+            raise HTTPException(
+                status_code=400,
+                detail={"error": f"Team id={data.team_id} does not exist in db"},
+            )
+
+        removed_team_members, new_team_members = _cleanup_members_with_roles(
+            existing_team_row=LiteLLM_TeamTable(team_id=data.team_id, members_with_roles=fresh_members),
+            data=data,
+        )
+
+        if not removed_team_members:
+            raise HTTPException(status_code=400, detail={"error": "User not found in team"})
+
+        existing_team_row.members_with_roles = new_team_members
+
+        _db_new_team_members: Final[list[dict]] = [m.model_dump() for m in new_team_members]
+
+        ## DELETE TEAM ID from USER ROW, IF EXISTS ##
+        # get user row
+        removed_user_ids: Final = frozenset(m.user_id for m in removed_team_members if m.user_id is not None)
+        key_val: Final[Mapping[str, object]] = (
+            {"user_id": {"in": sorted(removed_user_ids)}} if removed_user_ids else {"user_email": data.user_email}
+        )
+        member_tx: Final[_MemberDeleteTx] = tx
+        existing_user_rows: Final[Sequence[LiteLLM_UserTable]] = await member_tx.litellm_usertable.find_many(
+            where=key_val
+        )
+
+        # Also clean up any existing team membership rows for this user and team
+        user_ids_to_delete: Final = removed_user_ids.union(
+            (data.user_id,) if data.user_id is not None else (),
+            (user.user_id for user in existing_user_rows if user.user_id),
+        )
+
+        ## DELETE KEYS CREATED BY USER FOR THIS TEAM
+        # Fetch keys before deletion so their audit records can be persisted alongside the delete.
+        # An empty user_ids_to_delete still resolves cleanly: prisma's "in": [] matches no rows.
+        keys_to_delete: Final[list[LiteLLM_VerificationToken]] = await member_tx.litellm_verificationtoken.find_many(
+            where={
+                "user_id": {"in": sorted(user_ids_to_delete)},
+                "team_id": data.team_id,
+            }
+        )
+
         await tx.litellm_teamtable.update(
             where={"team_id": data.team_id},
             data={"members_with_roles": json.dumps(_db_new_team_members)},
@@ -4009,7 +4043,21 @@ async def delete_team(
     await _sweep_deleted_team_references(team_ids=data.team_ids, prisma_client=prisma_client)
 
     ## DELETE TEAMS
-    deleted_teams: Final = await prisma_client.delete_data(team_id_list=data.team_ids, table_name="team")
+    # Both the delete and the reconcile sweep run under every team's advisory lock
+    # (TEAM_ADVISORY_LOCK_SQL, the same one /team/member_add takes before its own writes),
+    # sorted so two overlapping batch deletes always request their locks in the same order.
+    # A member_add mid-flight for one of these teams either finishes its write and releases
+    # the lock before this transaction starts, in which case this sweep reaches what it wrote,
+    # or is still waiting on the lock, in which case its own re-read happens after this commits
+    # and sees the row gone before it writes anything.
+    delete_filter: Final[_TeamIdInFilter] = {"team_id": {"in": data.team_ids}}
+    async with prisma_client.tx() as tx:
+        for team_id in sorted(data.team_ids):
+            await tx.query_raw(TEAM_ADVISORY_LOCK_SQL, team_id)
+        await tx.litellm_teamtable.delete_many(where=delete_filter)
+        await _sweep_deleted_team_references_tx(team_ids=data.team_ids, tx=tx)
+
+    deleted_teams: Final[_DeletedTeamsResult] = {"deleted_teams": data.team_ids}
 
     # Evict AFTER the rows are gone. Both writers of these keys (`_cache_team_object` and
     # `get_team_object_by_alias`) hydrate from the db, so evicting first leaves a window where a
@@ -4022,12 +4070,6 @@ async def delete_team(
         proxy_logging_obj=proxy_logging_obj,
     )
 
-    # Sweep again now the team is gone. A `/team/member_add` that landed between the first sweep
-    # and the delete would have re-appended the reference; an add still in flight sees the row
-    # missing under its own row lock and sweeps what it wrote. Both passes are idempotent, and
-    # keeping the first one means a failure here still leaves a team the admin can retry deleting.
-    await _sweep_deleted_team_references(team_ids=data.team_ids, prisma_client=prisma_client)
-
     for deleted_team in team_rows:
         await sync_team_access_group_membership(prisma_client=prisma_client, team_id=deleted_team.team_id)
 
@@ -4056,6 +4098,16 @@ async def _sweep_deleted_team_references(team_ids: Sequence[str], prisma_client:
     _ = await _team_membership_db(prisma_client).delete_many(where=_TeamIdInFilter(team_id={"in": tuple(team_ids)}))
 
 
+async def _sweep_deleted_team_references_tx(team_ids: Sequence[str], tx: _TeamDeleteTx) -> None:
+    """Same sweep as `_sweep_deleted_team_references`, run on the transaction that holds
+    every id's advisory lock and deletes the team rows, so it commits or rolls back with them."""
+    for team_id in team_ids:
+        _ = await tx.execute_raw(_STRIP_DELETED_TEAM_FROM_USERS_SQL, team_id)
+
+    membership_filter: Final[_TeamIdInFilter] = {"team_id": {"in": tuple(team_ids)}}
+    _ = await tx.litellm_teammembership.delete_many(where=membership_filter)
+
+
 async def _invalidate_deleted_key_cache(
     keys: Sequence[LiteLLM_VerificationToken],
     user_api_key_cache: UserApiKeyCache,
diff --git a/litellm/proxy/management_helpers/access_group_team_sync.py b/litellm/proxy/management_helpers/access_group_team_sync.py
index 55c0346e375..664e36c9f10 100644
--- a/litellm/proxy/management_helpers/access_group_team_sync.py
+++ b/litellm/proxy/management_helpers/access_group_team_sync.py
@@ -23,9 +23,11 @@ from pydantic import BaseModel, TypeAdapter
 from litellm.proxy.auth.auth_checks import _delete_cache_access_object
 
 # hashtext collisions only cost two unrelated teams a little serialization, and the
-# lock is never taken by the access-group endpoints, so it cannot join their
-# access-group-then-team lock order to form a cycle.
-_LOCK_TEAM_SQL: Final = "SELECT pg_advisory_xact_lock(hashtext($1)) IS NULL AS locked"
+# lock is never taken by the access-group endpoints as a SELECT ... FOR UPDATE row lock,
+# so it cannot join their access-group-then-team lock order to form a cycle. team_endpoints
+# reuses this exact statement to serialize /team/member_add and /team/delete against each
+# other and against this mirror, rather than defining a second, divergent lock on the same key.
+TEAM_ADVISORY_LOCK_SQL: Final = "SELECT pg_advisory_xact_lock(hashtext($1)) IS NULL AS locked"
 
 _READ_TEAM_SQL: Final = 'SELECT access_group_ids FROM "LiteLLM_TeamTable" WHERE team_id = $1'
 
@@ -138,7 +140,7 @@ async def reconcile_team_access_group_membership(tx: AccessGroupSyncTx, team_id:
     concurrent write for a different team cannot be lost the way a read-modify-write of
     the whole array can, and the pair commits together or not at all.
     """
-    await tx.query_raw(_LOCK_TEAM_SQL, team_id)
+    await tx.query_raw(TEAM_ADVISORY_LOCK_SQL, team_id)
     team_rows: Final = _TeamRows.validate_python(await tx.query_raw(_READ_TEAM_SQL, team_id))
     desired: Final = (team_rows[0].access_group_ids or ()) if team_rows else ()
     affected: Final = _AffectedGroups.validate_python(await tx.query_raw(_AFFECTED_SQL, team_id, desired))
diff --git a/litellm/proxy/management_helpers/utils.py b/litellm/proxy/management_helpers/utils.py
index cb30ce90c7f..e2d7262fb69 100644
--- a/litellm/proxy/management_helpers/utils.py
+++ b/litellm/proxy/management_helpers/utils.py
@@ -34,7 +34,7 @@ from litellm.proxy._types import (  # key request types; user request types; tea
 )
 from litellm.proxy.common_utils.http_parsing_utils import _read_request_body
 from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time
-from litellm.proxy.utils import PrismaClient
+from litellm.proxy.utils import PrismaClient, jsonify_object
 from litellm.repositories.budget_repository import BudgetRepository
 from litellm.repositories.table_repositories import TeamMembershipRepository
 from litellm.repositories.user_repository import UserRepository
@@ -79,6 +79,8 @@ class _PrismaUserTable(Protocol):
         self, *, where: Mapping[str, object], data: Mapping[str, Mapping[str, object]]
     ) -> _PrismaUserRecord | None: ...
 
+    async def find_many(self, *, where: Mapping[str, object]) -> Sequence[_PrismaUserRecord]: ...
+
 
 class _PrismaTeamMembershipTable(Protocol):
     """Team membership table actions the management helpers issue."""
@@ -86,6 +88,73 @@ class _PrismaTeamMembershipTable(Protocol):
     async def create(self, *, data: Mapping[str, object], include: Mapping[str, bool]) -> _PrismaRecord: ...
 
 
+class MemberWriteTx(Protocol):
+    """Transaction surface `add_new_member` writes through when the caller owns one.
+
+    A caller already holding a transaction, and with it a pooled connection plus that
+    transaction's locks, passes it here so these writes reuse that connection rather than
+    checking out another one that lock waiters may already have drained from the pool.
+    """
+
+    @property
+    def litellm_usertable(self) -> _PrismaUserTable: ...
+
+    @property
+    def litellm_budgettable(self) -> _PrismaBudgetTable: ...
+
+    @property
+    def litellm_teammembership(self) -> _PrismaTeamMembershipTable: ...
+
+
+def _user_table(prisma_client: PrismaClient, tx: MemberWriteTx | None) -> _PrismaUserTable:
+    return tx.litellm_usertable if tx is not None else UserRepository(prisma_client).table
+
+
+def _budget_table(prisma_client: PrismaClient, tx: MemberWriteTx | None) -> _PrismaBudgetTable:
+    return tx.litellm_budgettable if tx is not None else BudgetRepository(prisma_client).table
+
+
+def _team_membership_table(prisma_client: PrismaClient, tx: MemberWriteTx | None) -> _PrismaTeamMembershipTable:
+    return tx.litellm_teammembership if tx is not None else TeamMembershipRepository(prisma_client).table
+
+
+async def _find_users_by_email(
+    prisma_client: PrismaClient, tx: MemberWriteTx | None, user_email: str
+) -> Sequence[_PrismaUserRecord]:
+    if tx is not None:
+        return await tx.litellm_usertable.find_many(where={"user_email": user_email})
+    rows: Final[Sequence[_PrismaUserRecord] | None] = await prisma_client.get_data(
+        key_val={"user_email": user_email},
+        table_name="user",
+        query_type="find_all",
+    )
+    return rows if rows is not None else ()
+
+
+async def _upsert_user_row(
+    user_table: _PrismaUserTable, user_id: str, create_data: Mapping[str, object]
+) -> _PrismaUserRecord | None:
+    """Insert the user row if it is absent, leaving an existing row as it is.
+
+    Upserting keeps concurrent provisioning of the same new user from racing on create.
+    The update branch re-states user_id rather than being empty because Prisma only
+    compiles an upsert down to INSERT ... ON CONFLICT when the update is non-empty, and
+    otherwise falls back to a racy SELECT-then-INSERT.
+    """
+    return await user_table.upsert(
+        where={"user_id": user_id},
+        data={"create": create_data, "update": {"user_id": user_id}},
+    )
+
+
+async def _create_user_row(
+    prisma_client: PrismaClient, tx: MemberWriteTx | None, user_data: dict[str, object]
+) -> _PrismaUserRecord | None:
+    if tx is not None:
+        return await _upsert_user_row(tx.litellm_usertable, str(user_data["user_id"]), jsonify_object(user_data))
+    return await prisma_client.insert_data(data=user_data, table_name="user")
+
+
 def get_new_internal_user_defaults(user_id: str, user_email: str | None = None) -> dict[str, object]:
     user_info: Final = litellm.default_internal_user_params or {}
 
@@ -206,6 +275,7 @@ async def _clone_team_default_budget_for_member(
     user_api_key_dict: UserAPIKeyAuth,
     litellm_proxy_admin_name: str,
     budget_duration_override: str | None = None,
+    tx: MemberWriteTx | None = None,
 ) -> str | None:
     """
     Create a new budget row that copies the values from the team's default
@@ -220,7 +290,7 @@ async def _clone_team_default_budget_for_member(
     member while keeping the default's other limits, so an admin can set a
     member's reset cadence without discarding the team default's max_budget.
     """
-    budget_table: Final[_PrismaBudgetTable] = BudgetRepository(prisma_client).table
+    budget_table: Final[_PrismaBudgetTable] = _budget_table(prisma_client, tx)
     default_budget: Final = await budget_table.find_unique(where={"budget_id": default_team_budget_id})
     if default_budget is None:
         return None
@@ -248,7 +318,7 @@ async def _clone_team_default_budget_for_member(
     if cloned_data.get("budget_duration"):
         cloned_data["budget_reset_at"] = get_budget_reset_time(cloned_data["budget_duration"])
 
-    new_budget: Final[_PrismaBudgetRecord] = await BudgetRepository(prisma_client).table.create(data=cloned_data)
+    new_budget: Final[_PrismaBudgetRecord] = await budget_table.create(data=cloned_data)
     return new_budget.budget_id
 
 
@@ -260,6 +330,7 @@ async def _resolve_member_budget_id(
     allowed_models: list[str] | None,
     budget_duration: str | None,
     default_team_budget_id: str | None,
+    tx: MemberWriteTx | None = None,
 ) -> str | None:
     """
     Resolve the budget a new team member should be linked to.
@@ -279,6 +350,7 @@ async def _resolve_member_budget_id(
             user_api_key_dict=user_api_key_dict,
             litellm_proxy_admin_name=litellm_proxy_admin_name,
             budget_duration_override=budget_duration,
+            tx=tx,
         )
 
     if not has_explicit_limit and budget_duration is None:
@@ -295,12 +367,14 @@ async def _resolve_member_budget_id(
     if budget_duration is not None:
         budget_data["budget_duration"] = budget_duration
         budget_data["budget_reset_at"] = get_budget_reset_time(budget_duration=budget_duration)
-    budget_table: Final[_PrismaBudgetTable] = BudgetRepository(prisma_client).table
+    budget_table: Final[_PrismaBudgetTable] = _budget_table(prisma_client, tx)
     response: Final = await budget_table.create(data=budget_data)
     return response.budget_id
 
 
-async def _append_team_id_if_absent(prisma_client: PrismaClient, user_id: str, team_id: str) -> None:
+async def _append_team_id_if_absent(
+    prisma_client: PrismaClient, user_id: str, team_id: str, tx: MemberWriteTx | None = None
+) -> None:
     """Append team_id to a user's teams array, only if it is not already present.
 
     The row-level filter makes the append a no-op once the team is present, so
@@ -309,7 +383,7 @@ async def _append_team_id_if_absent(prisma_client: PrismaClient, user_id: str, t
     number of teams a user belongs to). Teams added concurrently for a different
     team id are unaffected, since each update filters on its own team id.
     """
-    user_table: Final[_PrismaUserTable] = UserRepository(prisma_client).table
+    user_table: Final[_PrismaUserTable] = _user_table(prisma_client, tx)
     await user_table.update_many(
         where={"user_id": user_id, "NOT": {"teams": {"has": team_id}}},
         data={"teams": {"push": [team_id]}},
@@ -326,6 +400,7 @@ async def add_new_member(
     default_team_budget_id: str | None = None,
     allowed_models: list[str] | None = None,
     budget_duration: str | None = None,
+    tx: MemberWriteTx | None = None,
 ) -> tuple[LiteLLM_UserTable, LiteLLM_TeamMembership | None]:
     """
     Add a new member to a team
@@ -334,49 +409,41 @@ async def add_new_member(
     - add team member w/ budget to team member table
 
     Returns created/existing user + team membership w/ budget id
+
+    Callers already inside a transaction pass it as ``tx`` so every write here runs on that
+    connection instead of borrowing more from the pool while the caller's locks are held.
     """
     returned_user: LiteLLM_UserTable | None = None
     returned_team_membership: LiteLLM_TeamMembership | None = None
     ## ADD TEAM ID, to USER TABLE IF NEW ##
     if new_member.user_id is not None:
         new_user_defaults = get_new_internal_user_defaults(user_id=new_member.user_id)
-        # Upsert ensures the user row exists atomically (no create race when the
-        # same new user is provisioned concurrently), seeding teams on create.
-        # The teams append lives in the filtered update below rather than the
-        # upsert's update branch so an already-existing user does not get a
-        # duplicate team id. The update branch still has to write something:
-        # Prisma only compiles an upsert down to INSERT ... ON CONFLICT when it
-        # is non-empty, and falls back to a racy SELECT-then-INSERT when it is
-        # not, so this re-states user_id as a no-op rather than being empty.
-        user_table: Final[_PrismaUserTable] = UserRepository(prisma_client).table
-        _returned_user: _PrismaUserRecord | None = await user_table.upsert(
-            where={"user_id": new_member.user_id},
-            data={
-                "create": {"teams": [team_id], **new_user_defaults},
-                "update": {"user_id": new_member.user_id},
-            },
+        # The teams append lives in the filtered update below rather than the upsert's
+        # update branch so an already-existing user does not get a duplicate team id.
+        _returned_user: _PrismaUserRecord | None = await _upsert_user_row(
+            _user_table(prisma_client, tx),
+            new_member.user_id,
+            {"teams": [team_id], **new_user_defaults},
         )
-        await _append_team_id_if_absent(prisma_client, new_member.user_id, team_id)
+        await _append_team_id_if_absent(prisma_client, new_member.user_id, team_id, tx)
         if _returned_user is not None:
             returned_user = LiteLLM_UserTable.model_validate(_returned_user.model_dump())
     elif new_member.user_email is not None:
         new_user_defaults = get_new_internal_user_defaults(user_id=str(uuid.uuid4()), user_email=new_member.user_email)
         ## user email is not unique acc. to prisma schema -> future improvement
         ### for now: check if it exists in db, if not - insert it
-        existing_user_row: Final[list[_PrismaUserRecord] | None] = await prisma_client.get_data(
-            key_val={"user_email": new_member.user_email},
-            table_name="user",
-            query_type="find_all",
+        existing_user_row: Final[Sequence[_PrismaUserRecord]] = await _find_users_by_email(
+            prisma_client, tx, new_member.user_email
         )
-        if existing_user_row is None or (isinstance(existing_user_row, list) and len(existing_user_row) == 0):
+        if len(existing_user_row) == 0:
             new_user_defaults["teams"] = [team_id]
-            _returned_user = await prisma_client.insert_data(data=new_user_defaults, table_name="user")
+            _returned_user = await _create_user_row(prisma_client, tx, new_user_defaults)
 
             if _returned_user is not None:
                 returned_user = LiteLLM_UserTable.model_validate(_returned_user.model_dump())
         elif len(existing_user_row) == 1:
             user_info: Final = existing_user_row[0]
-            await _append_team_id_if_absent(prisma_client, user_info.user_id, team_id)
+            await _append_team_id_if_absent(prisma_client, user_info.user_id, team_id, tx)
             returned_user = LiteLLM_UserTable.model_validate(user_info.model_dump())
         elif len(existing_user_row) > 1:
             raise HTTPException(
@@ -392,10 +459,11 @@ async def add_new_member(
         allowed_models=allowed_models,
         budget_duration=budget_duration,
         default_team_budget_id=default_team_budget_id,
+        tx=tx,
     )
 
     if _budget_id and returned_user is not None and returned_user.user_id is not None:
-        membership_table: Final[_PrismaTeamMembershipTable] = TeamMembershipRepository(prisma_client).table
+        membership_table: Final[_PrismaTeamMembershipTable] = _team_membership_table(prisma_client, tx)
         _returned_team_membership: Final = await membership_table.create(
             data={
                 "team_id": team_id,
diff --git a/litellm/repositories/team_repository.py b/litellm/repositories/team_repository.py
index 7efd32288e4..d636592f925 100644
--- a/litellm/repositories/team_repository.py
+++ b/litellm/repositories/team_repository.py
@@ -58,19 +58,22 @@ class TeamRepository(BaseRepository[LiteLLM_TeamTable]):
         return LiteLLM_TeamTable.model_validate(data)
 
     async def get_members_with_roles_locked(self, tx: "Prisma", team_id: str) -> list[Member] | None:
-        """Return the team's members_with_roles, locking the row FOR UPDATE.
+        """Return the team's members_with_roles. The caller must already hold
+        ``TEAM_ADVISORY_LOCK_SQL`` for this team_id on ``tx`` before calling this.
 
-        ``None`` when the team row is gone, which a caller holding the lock can
-        only see if a delete committed under it, as opposed to ``[]`` for a team
-        that simply has no members.
+        ``None`` when the team row is gone, which is only possible under that lock if
+        a delete committed before this read, as opposed to ``[]`` for a team that
+        simply has no members.
 
-        Must be called inside a transaction so the row lock is held until
-        commit. This serializes concurrent membership writers on the team row
-        so the losing writer appends onto the winner's committed result instead
-        of overwriting it from a stale snapshot.
+        A plain read is enough here because the advisory lock, not a row lock, is what
+        serializes this against a concurrent writer: ``SELECT ... FOR UPDATE`` would
+        additionally take a row lock on ``LiteLLM_TeamTable``, and the access-group
+        endpoints lock an access group and then a team row, so a team-row-first lock
+        here can deadlock with them. The advisory lock cannot, since those endpoints
+        never take it.
         """
         rows: Final = await tx.query_raw(
-            'SELECT members_with_roles FROM "LiteLLM_TeamTable" WHERE team_id = $1 FOR UPDATE',
+            'SELECT members_with_roles FROM "LiteLLM_TeamTable" WHERE team_id = $1',
             team_id,
         )
         if not rows:
diff --git a/tests/proxy_admin_ui_tests/test_team_delete_member_add_race.py b/tests/proxy_admin_ui_tests/test_team_delete_member_add_race.py
new file mode 100644
index 00000000000..30544a8bb81
--- /dev/null
+++ b/tests/proxy_admin_ui_tests/test_team_delete_member_add_race.py
@@ -0,0 +1,307 @@
+"""
+Real-Postgres coverage for the /team/member_add vs /team/delete race (LIT-5544), and for
+/team/member_delete's participation in the same lock.
+
+A member_add that validated the team before a delete began could previously still commit
+its writes after the delete's reference sweeps had already run, leaving a user record and
+a membership row pointing at a team id that no longer exists. Neither side of that race can
+be forced by a sequential script: it needs one request to be genuinely mid-flight while the
+other commits. A mocked prisma cannot arbitrate that either, since the property under test
+is whether Postgres's own advisory lock actually serializes the two requests.
+
+These tests pin the interleaving the same way test_access_group_team_sync.py does: a second
+real connection holds the team's advisory lock in its own transaction, so the function under
+test is provably blocked on it rather than hoping a sleep lands in the right gap.
+"""
+
+import asyncio
+import json
+import os
+from contextlib import asynccontextmanager
+from datetime import timedelta
+from unittest.mock import MagicMock
+
+import pytest
+from fastapi import HTTPException
+
+from litellm.proxy._types import (
+    DeleteTeamRequest,
+    LitellmUserRoles,
+    Member,
+    TeamMemberAddRequest,
+    UserAPIKeyAuth,
+)
+from litellm.caching.caching import DualCache
+from litellm.proxy.utils import PrismaClient, ProxyLogging
+
+TEAM = "lit5544-race-team"
+USER = "lit5544-race-user"
+_DELETE_SEEDED = 'DELETE FROM "LiteLLM_TeamMembership" WHERE team_id = $1'
+_DELETE_USER = 'DELETE FROM "LiteLLM_UserTable" WHERE user_id = $1'
+_DELETE_TEAM = 'DELETE FROM "LiteLLM_TeamTable" WHERE team_id = $1'
+_LOCK_SQL = "SELECT pg_advisory_xact_lock(hashtext($1)) IS NULL AS locked"
+
+
+@asynccontextmanager
+async def _clean_db():
+    """Connects inside the running test's loop: an async fixture would be torn up on a
+    different loop than the test body, which prisma's engine lock refuses outright."""
+    from prisma import Prisma
+
+    if not os.getenv("DATABASE_URL"):
+        pytest.fail("DATABASE_URL is required; these tests must not silently skip")
+
+    db = Prisma()
+    await db.connect()
+    try:
+        await db.execute_raw(_DELETE_SEEDED, TEAM)
+        await db.execute_raw(_DELETE_USER, USER)
+        await db.execute_raw(_DELETE_TEAM, TEAM)
+        yield db
+    finally:
+        await db.execute_raw(_DELETE_SEEDED, TEAM)
+        await db.execute_raw(_DELETE_USER, USER)
+        await db.execute_raw(_DELETE_TEAM, TEAM)
+        await db.disconnect()
+
+
+@asynccontextmanager
+async def _real_prisma_client():
+    """The full app-level PrismaClient, not the raw generated client: add_new_member reads
+    and writes through PrismaClient.get_data/insert_data, which the raw client doesn't have."""
+    proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache())
+    client = PrismaClient(database_url=os.environ["DATABASE_URL"], proxy_logging_obj=proxy_logging_obj)
+    await client.connect()
+    try:
+        yield client
+    finally:
+        await client.db.disconnect()
+
+
+def _admin_auth():
+    return UserAPIKeyAuth(user_id="lit5544-admin", api_key="sk-lit5544", user_role=LitellmUserRoles.PROXY_ADMIN.value)
+
+
+@pytest.mark.asyncio
+async def test_member_add_blocked_by_delete_writes_no_dangling_reference():
+    """
+    member_add re-reads the team under the advisory lock before writing anything. When a
+    delete already holds that lock and then removes the row, member_add's re-read must see
+    the row gone and raise, without ever calling the write that appends the user/membership
+    references, which is the only way this leaves zero trace after the delete wins.
+    """
+    from litellm.proxy._types import LiteLLM_TeamTable
+    from litellm.proxy.management_endpoints.team_endpoints import (
+        _add_team_members_to_team,
+    )
+
+    async with _clean_db() as db:
+        await db.litellm_teamtable.create(data={"team_id": TEAM, "team_alias": TEAM, "members_with_roles": "[]"})
+
+        async with _real_prisma_client() as prisma_client:
+            from prisma import Prisma
+
+            blocker = Prisma()
+            await blocker.connect()
+            lock_acquired = asyncio.Event()
+
+            async def add_member():
+                lock_acquired.set()
+                await _add_team_members_to_team(
+                    data=TeamMemberAddRequest(
+                        team_id=TEAM,
+                        member=Member(user_id=USER, role="user"),
+                        max_budget_in_team=5.0,
+                    ),
+                    complete_team_data=LiteLLM_TeamTable(team_id=TEAM, members_with_roles=[]),
+                    prisma_client=prisma_client,
+                    user_api_key_dict=_admin_auth(),
+                    litellm_proxy_admin_name="lit5544-admin",
+                )
+
+            try:
+                async with blocker.tx(timeout=timedelta(seconds=30)) as held:
+                    await held.query_raw(_LOCK_SQL, TEAM)
+                    task = asyncio.create_task(add_member())
+                    await lock_acquired.wait()
+                    await asyncio.sleep(0.2)
+                    assert not task.done(), "member_add did not wait on the team's advisory lock"
+
+                    # the delete wins the race: strip the team row while the lock is held
+                    await held.execute_raw(_DELETE_TEAM, TEAM)
+
+                with pytest.raises(HTTPException) as exc_info:
+                    await asyncio.wait_for(task, timeout=30)
+                assert exc_info.value.status_code == 404
+            finally:
+                await blocker.disconnect()
+
+        user_row = await db.litellm_usertable.find_unique(where={"user_id": USER})
+        assert user_row is None, "member_add must not have written a user row for a team that was gone under its lock"
+
+        membership_row = await db.litellm_teammembership.find_first(where={"team_id": TEAM, "user_id": USER})
+        assert membership_row is None
+
+
+@pytest.mark.asyncio
+async def test_member_delete_blocked_by_member_add_removes_from_the_fresh_roster():
+    """
+    team_member_delete takes the same advisory lock and re-reads the roster under it, so a
+    member_add that committed while member_delete was waiting on the lock is not silently
+    undone. Without the re-read, member_delete would compute its new roster from the stale
+    snapshot it validated against before the lock, and its write would overwrite the
+    member_add's addition right back out even though member_add's request already succeeded.
+    """
+    import litellm.proxy.proxy_server as proxy_server_module
+    from litellm.proxy._types import TeamMemberDeleteRequest
+    from litellm.proxy.management_endpoints.team_endpoints import team_member_delete
+
+    other_user = f"{USER}-other"
+    seeded_roster = '[{"user_id": "%s", "user_email": null, "role": "user"}]' % USER
+    winning_add_roster = (
+        '[{"user_id": "%s", "user_email": null, "role": "user"}, '
+        '{"user_id": "%s", "user_email": null, "role": "user"}]' % (USER, other_user)
+    )
+
+    async with _clean_db() as db:
+        await db.litellm_teamtable.create(
+            data={"team_id": TEAM, "team_alias": TEAM, "members_with_roles": seeded_roster}
+        )
+
+        async with _real_prisma_client() as prisma_client:
+            original_prisma_client = proxy_server_module.prisma_client
+            proxy_server_module.prisma_client = prisma_client
+
+            try:
+                from prisma import Prisma
+
+                blocker = Prisma()
+                await blocker.connect()
+                lock_acquired = asyncio.Event()
+
+                async def run_delete():
+                    lock_acquired.set()
+                    return await team_member_delete(
+                        data=TeamMemberDeleteRequest(team_id=TEAM, user_id=USER),
+                        user_api_key_dict=_admin_auth(),
+                    )
+
+                try:
+                    async with blocker.tx(timeout=timedelta(seconds=30)) as held:
+                        await held.query_raw(_LOCK_SQL, TEAM)
+                        task = asyncio.create_task(run_delete())
+                        await lock_acquired.wait()
+                        await asyncio.sleep(0.2)
+                        assert not task.done(), "member_delete did not wait on the team's advisory lock"
+
+                        # member_add wins the race: it adds `other_user` while holding the lock
+                        await held.litellm_teamtable.update(
+                            where={"team_id": TEAM},
+                            data={"members_with_roles": winning_add_roster},
+                        )
+
+                    await asyncio.wait_for(task, timeout=30)
+                finally:
+                    await blocker.disconnect()
+            finally:
+                proxy_server_module.prisma_client = original_prisma_client
+
+        team_row = await db.litellm_teamtable.find_unique(where={"team_id": TEAM})
+        raw_roster = team_row.members_with_roles
+        parsed_roster = json.loads(raw_roster) if isinstance(raw_roster, str) else raw_roster
+        remaining_ids = {m["user_id"] for m in parsed_roster}
+        assert remaining_ids == {other_user}, (
+            "member_delete must remove only the user it targeted from the roster it actually "
+            "committed to, not silently drop the member the winning add just committed"
+        )
+
+
+@pytest.mark.asyncio
+async def test_delete_blocked_by_member_add_sweeps_the_fresh_reference():
+    """
+    A member_add that wins the lock race writes its reference and releases the lock; the
+    delete that was waiting on it must then run its locked sweep against the row as it
+    actually is, not a stale snapshot, and reap that reference rather than leaving it
+    stranded on a team id the delete is about to remove.
+    """
+    import litellm.proxy.proxy_server as proxy_server_module
+    from litellm.proxy._types import LiteLLM_TeamTable
+    from litellm.proxy.management_endpoints.team_endpoints import delete_team
+
+    async with _clean_db() as db:
+        await db.litellm_teamtable.create(data={"team_id": TEAM, "team_alias": TEAM, "members_with_roles": "[]"})
+
+        async with _real_prisma_client() as prisma_client:
+            proxy_logging_obj = prisma_client.proxy_logging_obj
+            original_prisma_client = proxy_server_module.prisma_client
+            original_admin_name = proxy_server_module.litellm_proxy_admin_name
+            original_proxy_logging_obj = proxy_server_module.proxy_logging_obj
+            original_cache = proxy_server_module.user_api_key_cache
+            original_router = proxy_server_module.llm_router
+            proxy_server_module.prisma_client = prisma_client
+            proxy_server_module.litellm_proxy_admin_name = "lit5544-admin"
+            proxy_server_module.proxy_logging_obj = proxy_logging_obj
+            proxy_server_module.user_api_key_cache = original_cache or proxy_logging_obj.internal_usage_cache
+            proxy_server_module.llm_router = None
+
+            async def restore():
+                proxy_server_module.prisma_client = original_prisma_client
+                proxy_server_module.litellm_proxy_admin_name = original_admin_name
+                proxy_server_module.proxy_logging_obj = original_proxy_logging_obj
+                proxy_server_module.user_api_key_cache = original_cache
+                proxy_server_module.llm_router = original_router
+
+            try:
+                from prisma import Prisma
+
+                blocker = Prisma()
+                await blocker.connect()
+                lock_acquired = asyncio.Event()
+
+                async def run_delete():
+                    lock_acquired.set()
+                    return await delete_team(
+                        data=DeleteTeamRequest(team_ids=[TEAM]),
+                        http_request=MagicMock(),
+                        user_api_key_dict=_admin_auth(),
+                        litellm_changed_by="lit5544-admin",
+                    )
+
+                try:
+                    async with blocker.tx(timeout=timedelta(seconds=30)) as held:
+                        await held.query_raw(_LOCK_SQL, TEAM)
+                        task = asyncio.create_task(run_delete())
+                        await lock_acquired.wait()
+                        await asyncio.sleep(0.3)
+                        assert not task.done(), "delete_team did not wait on the team's advisory lock"
+
+                        # member_add wins the race: write the reference while holding the lock
+                        await held.litellm_usertable.upsert(
+                            where={"user_id": USER},
+                            data={
+                                "create": {"user_id": USER, "teams": [TEAM]},
+                                "update": {"teams": {"push": [TEAM]}},
+                            },
+                        )
+                        await held.litellm_teammembership.create(data={"team_id": TEAM, "user_id": USER})
+                        await held.litellm_teamtable.update(
+                            where={"team_id": TEAM},
+                            data={"members_with_roles": '[{"user_id": "%s", "role": "user"}]' % USER},
+                        )
+
+                    await asyncio.wait_for(task, timeout=30)
+                finally:
+                    await blocker.disconnect()
+            finally:
+                await restore()
+
+        team_row = await db.litellm_teamtable.find_unique(where={"team_id": TEAM})
+        assert team_row is None
+
+        user_row = await db.litellm_usertable.find_unique(where={"user_id": USER})
+        assert user_row is not None and TEAM not in user_row.teams, (
+            "delete_team's locked sweep must reap the reference member_add wrote just before losing the lock"
+        )
+
+        membership_row = await db.litellm_teammembership.find_first(where={"team_id": TEAM, "user_id": USER})
+        assert membership_row is None
diff --git a/tests/proxy_unit_tests/test_proxy_server.py b/tests/proxy_unit_tests/test_proxy_server.py
index 21dbf3e090f..ceaabf0a70f 100644
--- a/tests/proxy_unit_tests/test_proxy_server.py
+++ b/tests/proxy_unit_tests/test_proxy_server.py
@@ -1169,6 +1169,22 @@ async def test_create_user_default_budget(prisma_client, user_role):  # noqa: F8
             assert mock_client.call_args.kwargs["data"]["budget_duration"] is None
 
 
+def _member_add_tx_cm(team_table):
+    """Transaction whose member writes land on whatever tables are mocked on `prisma_client.db`"""
+
+    class _Tx:
+        query_raw = AsyncMock(return_value=[{"members_with_roles": []}])
+        litellm_teamtable = team_table
+
+        def __getattr__(self, table_name):
+            return getattr(litellm.proxy.proxy_server.prisma_client.db, table_name)
+
+    tx_cm = MagicMock()
+    tx_cm.__aenter__ = AsyncMock(return_value=_Tx())
+    tx_cm.__aexit__ = AsyncMock(return_value=None)
+    return tx_cm
+
+
 @pytest.mark.parametrize("new_member_method", ["user_id", "user_email"])
 @pytest.mark.asyncio
 @pytest.mark.skip(reason="Requires reliable external DB connection (prisma).")
@@ -1230,7 +1246,7 @@ async def test_create_team_member_add(prisma_client, new_member_method):  # noqa
             )
         )
         mock_litellm_usertable.upsert = mock_client
-        mock_litellm_usertable.find_many = AsyncMock(return_value=None)
+        mock_litellm_usertable.find_many = AsyncMock(return_value=[])
         # Mock find_first for user_email validation (returns None for new users)
         mock_litellm_usertable.find_first = AsyncMock(return_value=None)
         # Mock find_unique for user_id validation (returns None for new users)
@@ -1245,12 +1261,7 @@ async def test_create_team_member_add(prisma_client, new_member_method):  # noqa
             return_value=LiteLLM_TeamTableCachedObj(team_id="1234")
         )
 
-        tx_mock = AsyncMock()
-        tx_mock.query_raw = AsyncMock(return_value=[{"members_with_roles": []}])
-        tx_mock.litellm_teamtable = team_mock_client
-        tx_cm = MagicMock()
-        tx_cm.__aenter__ = AsyncMock(return_value=tx_mock)
-        tx_cm.__aexit__ = AsyncMock(return_value=None)
+        tx_cm = _member_add_tx_cm(team_mock_client)
         original_tx = litellm.proxy.proxy_server.prisma_client.tx
         litellm.proxy.proxy_server.prisma_client.tx = MagicMock(
             return_value=tx_cm
@@ -1432,7 +1443,7 @@ async def test_create_team_member_add_team_admin(
             )
         )
         mock_litellm_usertable.upsert = mock_client
-        mock_litellm_usertable.find_many = AsyncMock(return_value=None)
+        mock_litellm_usertable.find_many = AsyncMock(return_value=[])
         # Mock find_first for user_email validation (returns None for new users)
         mock_litellm_usertable.find_first = AsyncMock(return_value=None)
         # Mock find_unique for user_id validation (returns None for new users)
@@ -1443,12 +1454,7 @@ async def test_create_team_member_add_team_admin(
             return_value=LiteLLM_TeamTableCachedObj(team_id="1234")
         )
 
-        tx_mock = AsyncMock()
-        tx_mock.query_raw = AsyncMock(return_value=[{"members_with_roles": []}])
-        tx_mock.litellm_teamtable = team_mock_client
-        tx_cm = MagicMock()
-        tx_cm.__aenter__ = AsyncMock(return_value=tx_mock)
-        tx_cm.__aexit__ = AsyncMock(return_value=None)
+        tx_cm = _member_add_tx_cm(team_mock_client)
 
         with (
             patch.object(
diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py
index 7f5d3eb0a14..f33854eeb81 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py
@@ -4,7 +4,7 @@ from contextlib import asynccontextmanager
 from datetime import datetime, timezone
 from types import SimpleNamespace
 from typing import Optional, cast
-from unittest.mock import AsyncMock, MagicMock, call, patch
+from unittest.mock import AsyncMock, MagicMock, PropertyMock, call, patch
 
 import pytest
 from fastapi import HTTPException
@@ -58,6 +58,9 @@ from litellm.proxy.management_endpoints.team_endpoints import (
     update_team,
     validate_team_org_change,
 )
+from litellm.proxy.management_helpers.access_group_team_sync import (
+    TEAM_ADVISORY_LOCK_SQL,
+)
 from litellm.proxy.management_helpers.team_member_permission_checks import (
     TeamMemberPermissionChecks,
 )
@@ -75,7 +78,11 @@ client = TestClient(app)
 
 def _wire_team_create_tx(prisma_client):
     """`/team/new` inserts the team and mirrors it onto the access groups in one transaction,
-    so a mocked client has to hand its team table back out of `db.tx()`."""
+    so a mocked client has to hand its team table back out of `db.tx()`.
+
+    A `/team/new` carrying members then adds them under the team's advisory lock, and those
+    writes run on that lock's transaction, so `tx()` has to hand back the mocked tables too
+    for the per-table assertions on `prisma_client.db.*` to keep seeing them."""
 
     @asynccontextmanager
     async def _tx():
@@ -85,18 +92,67 @@ def _wire_team_create_tx(prisma_client):
         )
 
     prisma_client.db.tx = lambda *_args, **_kwargs: _tx()
+    _wire_member_add_tx(prisma_client)
+
+
+def _wire_member_add_tx(prisma_client):
+    """/team/member_add takes the team's advisory lock, re-reads the roster under it, and runs
+    the user, budget, and membership writes on that same transaction, so a mocked client has
+    to hand its own table mocks back out of `tx()`.
+
+    Tables resolve on access, not here, since tests routinely replace `db.<table>` after
+    wiring the transaction."""
+
+    class _Tx:
+        query_raw = AsyncMock(return_value=[{"members_with_roles": []}])
+
+        def __getattr__(self, table_name):
+            return getattr(prisma_client.db, table_name)
+
+    tx = _Tx()
+    tx_cm = MagicMock()
+    tx_cm.__aenter__ = AsyncMock(return_value=tx)
+    tx_cm.__aexit__ = AsyncMock(return_value=None)
+    prisma_client.tx = MagicMock(return_value=tx_cm)
 
 
 def _wire_member_delete_tx(prisma_client):
-    """/team/member_delete's four cleanups run inside one transaction, so a mocked
-    client has to hand back its own table mocks out of `tx()` for the existing
-    per-table assertions to keep seeing the calls."""
+    """/team/member_delete's four cleanups, plus the advisory-lock re-read that now guards
+    them, run inside one transaction, so a mocked client has to hand back its own table
+    mocks (and a `query_raw` that answers the locked re-read from the same team row the
+    test already configured on `find_unique`) out of `tx()` for the existing per-table
+    assertions to keep seeing the calls."""
+
+    async def _query_raw(sql, team_id):
+        if sql != TEAM_ADVISORY_LOCK_SQL:
+            team_row = await prisma_client.db.litellm_teamtable.find_unique(where={"team_id": team_id})
+            if team_row is not None:
+                return [{"members_with_roles": team_row.model_dump()["members_with_roles"]}]
+        return []
+
+    class _Tx:
+        query_raw = staticmethod(_query_raw)
+
+        def __getattr__(self, table_name):
+            return getattr(prisma_client.db, table_name)
+
+    tx = _Tx()
+    tx_cm = MagicMock()
+    tx_cm.__aenter__ = AsyncMock(return_value=tx)
+    tx_cm.__aexit__ = AsyncMock(return_value=None)
+    prisma_client.tx = MagicMock(return_value=tx_cm)
+
+
+def _wire_team_delete_tx(prisma_client):
+    """`/team/delete` deletes the team rows and runs its post-delete reference sweep under
+    every team's advisory lock in one transaction, so a mocked client has to hand its own
+    table mocks (and db-level execute_raw) back out of `tx()` for existing per-table
+    assertions on `prisma_client.db.*` to keep seeing those calls."""
     tx = SimpleNamespace(
         litellm_teamtable=prisma_client.db.litellm_teamtable,
-        litellm_usertable=prisma_client.db.litellm_usertable,
         litellm_teammembership=prisma_client.db.litellm_teammembership,
-        litellm_verificationtoken=prisma_client.db.litellm_verificationtoken,
-        litellm_deletedverificationtoken=prisma_client.db.litellm_deletedverificationtoken,
+        query_raw=AsyncMock(return_value=[]),
+        execute_raw=prisma_client.db.execute_raw,
     )
     tx_cm = MagicMock()
     tx_cm.__aenter__ = AsyncMock(return_value=tx)
@@ -1669,6 +1725,7 @@ async def test_process_team_members_single_member():
             default_team_budget_id="budget-123",
             allowed_models=None,
             budget_duration=None,
+            tx=None,
         )
 
 
@@ -1809,8 +1866,8 @@ async def test_update_team_members_list_duplicate_prevention():
 async def test_add_team_members_reconciles_against_freshly_locked_row():
     """
     Regression: _add_team_members_to_team must build the new members_with_roles
-    from the row it re-reads under a lock inside the write transaction, not from
-    the stale complete_team_data snapshot captured at the start of the request.
+    from the row it re-reads under the team's advisory lock, not from the stale
+    complete_team_data snapshot captured at the start of the request.
 
     Two concurrent /team/member_add calls for the same team read the same
     snapshot; without the locked re-read the losing write rewrites the whole
@@ -1871,24 +1928,89 @@ async def test_add_team_members_reconciles_against_freshly_locked_row():
     written_ids = sorted(m["user_id"] for m in json.loads(captured["data"]["members_with_roles"]))
     assert written_ids == ["alice", "bob", "zed"]
 
-    lock_reads = [call for call in tx.query_raw.call_args_list if "FOR UPDATE" in str(call.args[0])]
-    assert lock_reads, "expected a SELECT ... FOR UPDATE row-lock read before the write"
+    assert tx.query_raw.call_args_list[0].args == (TEAM_ADVISORY_LOCK_SQL, "test-team-lock"), (
+        "expected the team's advisory lock to be acquired before the members_with_roles read"
+    )
+    assert not any("FOR UPDATE" in str(call.args[0]) for call in tx.query_raw.call_args_list), (
+        "a row lock here can deadlock with the access-group endpoints; only the advisory lock is safe"
+    )
 
     assert [m.user_id for m in updated_team.members_with_roles] == ["zed", "alice", "bob"]
 
 
 @pytest.mark.asyncio
-async def test_add_team_members_cleans_up_when_the_team_is_deleted_mid_request():
+async def test_add_team_members_runs_member_writes_on_the_lock_holding_transaction():
+    """
+    Regression pin against exhausting the connection pool with advisory-lock waiters.
+
+    Every concurrent /team/member_add for one team holds a pooled connection while it waits
+    on the team's advisory lock. If the holder's member writes went to the regular client,
+    it would need a second connection to finish, so enough concurrent adds fill the pool
+    with waiters and the holder can never commit or release the lock. The member writes
+    therefore have to run on the transaction that already owns the connection.
+    """
+    from litellm.proxy.management_endpoints.team_endpoints import (
+        _add_team_members_to_team,
+    )
+
+    added_user = MagicMock()
+    added_user.user_id = "bob"
+    added_user.model_dump.return_value = {"user_id": "bob", "teams": ["team-pool"]}
+    created_budget = MagicMock()
+    created_budget.budget_id = "budget-pool"
+    membership = MagicMock()
+    membership.model_dump.return_value = {
+        "team_id": "team-pool",
+        "user_id": "bob",
+        "budget_id": "budget-pool",
+        "litellm_budget_table": None,
+    }
+
+    tx = MagicMock()
+    tx.query_raw = AsyncMock(return_value=[{"members_with_roles": []}])
+    tx.litellm_teamtable.update = AsyncMock(
+        return_value=LiteLLM_TeamTable(team_id="team-pool", members_with_roles=[])
+    )
+    tx.litellm_usertable.upsert = AsyncMock(return_value=added_user)
+    tx.litellm_usertable.update_many = AsyncMock()
+    tx.litellm_budgettable.create = AsyncMock(return_value=created_budget)
+    tx.litellm_teammembership.create = AsyncMock(return_value=membership)
+
+    tx_cm = MagicMock()
+    tx_cm.__aenter__ = AsyncMock(return_value=tx)
+    tx_cm.__aexit__ = AsyncMock(return_value=None)
+
+    prisma_client = MagicMock()
+    prisma_client.tx = MagicMock(return_value=tx_cm)
+    type(prisma_client).db = PropertyMock(
+        side_effect=AssertionError("member writes must not reach for a second pooled connection")
+    )
+
+    _, updated_users, updated_team_memberships = await _add_team_members_to_team(
+        data=TeamMemberAddRequest(
+            team_id="team-pool",
+            member=Member(user_id="bob", role="user"),
+            max_budget_in_team=50.0,
+        ),
+        complete_team_data=LiteLLM_TeamTable(team_id="team-pool", members_with_roles=[]),
+        prisma_client=cast(object, prisma_client),
+        user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN),
+        litellm_proxy_admin_name="admin",
+    )
+
+    assert [user.user_id for user in updated_users] == ["bob"]
+    assert [tm.budget_id for tm in updated_team_memberships] == ["budget-pool"]
+
+
+@pytest.mark.asyncio
+async def test_add_team_members_writes_nothing_when_the_team_is_deleted_mid_request():
     """
     Regression pin for the /team/member_add vs /team/delete race.
 
-    The user row and membership writes land before the reconcile takes the team
-    row lock, so a /team/delete that commits in between has already run its own
-    reference sweep and cannot see them. The empty locked SELECT is the only
-    signal that happened, and leaving it at that would strand the member on a
-    deleted team id, which authorization paths that trust `user.teams` would
-    treat as membership if the id were ever recreated. So the request must sweep
-    the references it just wrote and fail, not report success.
+    The advisory lock is acquired, and the team is gone, before any write is attempted:
+    the empty locked SELECT is proof a /team/delete already committed under the same
+    lock, so this request must fail without writing the user or membership rows in the
+    first place, rather than writing them and then trying to sweep them back out.
     """
     from litellm.proxy.management_endpoints.team_endpoints import (
         _add_team_members_to_team,
@@ -1907,9 +2029,10 @@ async def test_add_team_members_cleans_up_when_the_team_is_deleted_mid_request()
     prisma_client.db.execute_raw = AsyncMock()
     prisma_client.db.litellm_teammembership.delete_many = AsyncMock()
 
+    process_team_members = AsyncMock(return_value=([], []))
     with patch(
         "litellm.proxy.management_endpoints.team_endpoints._process_team_members",
-        new=AsyncMock(return_value=([], [])),
+        new=process_team_members,
     ):
         with pytest.raises(HTTPException) as exc_info:
             await _add_team_members_to_team(
@@ -1924,14 +2047,10 @@ async def test_add_team_members_cleans_up_when_the_team_is_deleted_mid_request()
             )
 
     assert exc_info.value.status_code == 404
+    process_team_members.assert_not_awaited()
     tx.litellm_teamtable.update.assert_not_awaited()
-
-    assert prisma_client.db.execute_raw.await_args_list == [
-        call(_STRIP_DELETED_TEAM_FROM_USERS_SQL, "team-deleted-mid-add")
-    ]
-    prisma_client.db.litellm_teammembership.delete_many.assert_awaited_once_with(
-        where={"team_id": {"in": ("team-deleted-mid-add",)}}
-    )
+    prisma_client.db.execute_raw.assert_not_awaited()
+    prisma_client.db.litellm_teammembership.delete_many.assert_not_awaited()
 
 
 def test_add_new_models_to_team_with_existing_models():
@@ -4246,6 +4365,86 @@ async def test_team_member_delete_cleans_verification_tokens(
     )
 
 
+@pytest.mark.asyncio
+async def test_team_member_delete_reads_on_the_lock_holding_transaction(
+    mock_db_client, mock_admin_auth
+):
+    """
+    Regression pin against exhausting the connection pool with advisory-lock waiters.
+
+    Every concurrent removal for one team holds a pooled connection while it waits on the
+    team's advisory lock, and /team/delete fans its per-member removals out concurrently.
+    A holder whose reads went to the regular client would need a second connection to
+    finish, so enough waiters fill the pool and the holder can never release the lock.
+    Both reads therefore have to run on the transaction that already owns the connection.
+    """
+    from litellm.proxy._types import TeamMemberDeleteRequest
+    from litellm.proxy.management_endpoints.team_endpoints import team_member_delete
+
+    test_team_id = "team-del-pool-123"
+    test_user_id = "user-del-pool-123"
+    roster_entry = {"user_id": test_user_id, "user_email": None, "role": "user"}
+
+    mock_team_row = MagicMock()
+    mock_team_row.model_dump.return_value = {
+        "team_id": test_team_id,
+        "members_with_roles": [roster_entry],
+        "team_member_permissions": [],
+        "metadata": {},
+        "models": [],
+        "spend": 0.0,
+    }
+    mock_db_client.db.litellm_teamtable.find_unique = AsyncMock(
+        return_value=mock_team_row
+    )
+
+    user_row = MagicMock()
+    user_row.user_id = test_user_id
+    user_row.teams = [test_team_id]
+
+    # Both are wired to answer, so the endpoint completes either way and the awaits below
+    # are what tells which connection it read on.
+    pooled_user_read = AsyncMock(return_value=[user_row])
+    pooled_token_read = AsyncMock(return_value=[])
+    mock_db_client.db.litellm_usertable.find_many = pooled_user_read
+    mock_db_client.db.litellm_verificationtoken.find_many = pooled_token_read
+
+    tx = MagicMock()
+    tx.query_raw = AsyncMock(return_value=[{"members_with_roles": [roster_entry]}])
+    tx.litellm_teamtable.update = AsyncMock(return_value=mock_team_row)
+    tx.litellm_usertable.find_many = AsyncMock(return_value=[user_row])
+    tx.litellm_usertable.update = AsyncMock()
+    tx.litellm_teammembership.delete_many = AsyncMock()
+    tx.litellm_verificationtoken.find_many = AsyncMock(return_value=[])
+    tx.litellm_verificationtoken.delete_many = AsyncMock()
+
+    tx_cm = MagicMock()
+    tx_cm.__aenter__ = AsyncMock(return_value=tx)
+    tx_cm.__aexit__ = AsyncMock(return_value=None)
+    mock_db_client.tx = MagicMock(return_value=tx_cm)
+
+    await team_member_delete(
+        data=TeamMemberDeleteRequest(team_id=test_team_id, user_id=test_user_id),
+        user_api_key_dict=mock_admin_auth,
+    )
+
+    tx.litellm_usertable.find_many.assert_awaited_once_with(
+        where={"user_id": {"in": [test_user_id]}}
+    )
+    tx.litellm_verificationtoken.find_many.assert_awaited_once_with(
+        where={"user_id": {"in": [test_user_id]}, "team_id": test_team_id}
+    )
+    pooled_user_read.assert_not_awaited()
+    pooled_token_read.assert_not_awaited()
+
+    tx.litellm_usertable.update.assert_awaited_once_with(
+        where={"user_id": test_user_id}, data={"teams": {"set": []}}
+    )
+    tx.litellm_teammembership.delete_many.assert_awaited_once_with(
+        where={"team_id": test_team_id, "user_id": test_user_id}
+    )
+
+
 @pytest.mark.parametrize(
     "roster_email",
     ["Alice@Example.com", "alice-invited-as@example.com"],
@@ -7411,6 +7610,7 @@ async def test_delete_team_persists_deleted_teams(
     mock_tx_cm.__aenter__ = AsyncMock(return_value=mock_tx)
     mock_tx_cm.__aexit__ = AsyncMock(return_value=False)
     mock_prisma_client.db.tx = MagicMock(return_value=mock_tx_cm)
+    _wire_team_delete_tx(mock_prisma_client)
 
     monkeypatch.setattr(
         "litellm.proxy.proxy_server.prisma_client",
@@ -7481,15 +7681,14 @@ async def test_delete_team_sweeps_references_outside_members_with_roles(
     cache_state_when_rows_deleted = {}
 
     async def record_cache_state_then_delete(*args, **kwargs):
-        if kwargs.get("table_name") == "team":
-            cache_state_when_rows_deleted["doomed_still_cached"] = (
-                fresh_cache.get_cache(key="team_id:team-doomed") is not None
-            )
-        return {"deleted_teams": ["team-doomed"]}
+        cache_state_when_rows_deleted["doomed_still_cached"] = (
+            fresh_cache.get_cache(key="team_id:team-doomed") is not None
+        )
+        return 1
 
     mock_prisma_client = AsyncMock()
     mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=doomed_team)
-    mock_prisma_client.delete_data = AsyncMock(side_effect=record_cache_state_then_delete)
+    mock_prisma_client.delete_data = AsyncMock(return_value={"deleted_keys": 0})
     mock_prisma_client.db.litellm_deletedteamtable.create_many = AsyncMock()
     mock_prisma_client.db.litellm_deletedverificationtoken.create_many = AsyncMock()
     mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[])
@@ -7498,6 +7697,7 @@ async def test_delete_team_sweeps_references_outside_members_with_roles(
     mock_prisma_client.db.execute_raw = mock_execute_raw
     mock_membership_delete_many = AsyncMock()
     mock_prisma_client.db.litellm_teammembership.delete_many = mock_membership_delete_many
+    mock_prisma_client.db.litellm_teamtable.delete_many = AsyncMock(side_effect=record_cache_state_then_delete)
 
     mock_tx = AsyncMock()
     mock_tx.litellm_proxymodeltable.find_many = AsyncMock(return_value=[])
@@ -7506,6 +7706,11 @@ async def test_delete_team_sweeps_references_outside_members_with_roles(
     mock_tx_cm.__aexit__ = AsyncMock(return_value=False)
     mock_prisma_client.db.tx = MagicMock(return_value=mock_tx_cm)
 
+    # The locked delete-and-sweep transaction /team/member_add serializes against, kept
+    # separate from mock_tx above (the BYOK-model-cleanup transaction, unrelated to this lock).
+    _wire_team_delete_tx(mock_prisma_client)
+    mock_lock_tx = mock_prisma_client.tx.return_value.__aenter__.return_value
+
     fresh_cache = UserApiKeyCache()
     for cached_team_id, cached_alias in (
         ("team-doomed", "doomed-team"),
@@ -7539,14 +7744,22 @@ async def test_delete_team_sweeps_references_outside_members_with_roles(
     assert mock_execute_raw.await_args_list == [
         call(_STRIP_DELETED_TEAM_FROM_USERS_SQL, "team-doomed"),
         call(_STRIP_DELETED_TEAM_FROM_USERS_SQL, "team-doomed"),
-    ], "the sweep must run once before the team row is deleted and again after, so a member_add racing the delete cannot leave the reference behind"
+    ], (
+        "the unlocked sweep must run once to catch pre-existing drift, and the locked sweep "
+        "(alongside the delete, under the same advisory lock member_add takes) must run again "
+        "so a member_add that wrote its reference just before losing the lock is still reaped"
+    )
 
-    # same two passes: the second one reaps a membership row inserted while the delete was running
+    # same two passes for the membership rows, the second under the lock alongside the delete
     assert mock_membership_delete_many.await_args_list == [
         call(where={"team_id": {"in": ("team-doomed",)}}),
         call(where={"team_id": {"in": ("team-doomed",)}}),
     ]
 
+    assert mock_lock_tx.query_raw.await_args_list == [call(TEAM_ADVISORY_LOCK_SQL, "team-doomed")], (
+        "the advisory lock must be acquired before the team row is deleted"
+    )
+
     assert fresh_cache.get_cache(key="team_id:team-doomed") is None
     assert fresh_cache.get_cache(key="team_alias:doomed-team") is None
     assert fresh_cache.get_cache(key="team_id:team-kept") is not None
@@ -7596,6 +7809,7 @@ async def test_delete_team_evicts_the_auth_cache_of_the_keys_it_deletes(
     mock_tx_cm.__aenter__ = AsyncMock(return_value=mock_tx)
     mock_tx_cm.__aexit__ = AsyncMock(return_value=False)
     mock_prisma_client.db.tx = MagicMock(return_value=mock_tx_cm)
+    _wire_team_delete_tx(mock_prisma_client)
 
     fresh_cache = UserApiKeyCache()
     fresh_cache.set_cache(key="hashed-doomed-key", value=UserAPIKeyAuth(token="hashed-doomed-key", team_id="team-doomed"))
@@ -7623,14 +7837,17 @@ async def test_delete_team_evicts_the_auth_cache_of_the_keys_it_deletes(
 
 
 @pytest.mark.asyncio
-async def test_delete_team_failing_reconcile_sweep_cannot_strand_the_team_in_cache(
+async def test_delete_team_failing_locked_sweep_rolls_back_the_delete_and_leaves_the_cache_alone(
     monkeypatch,
     disable_audit_logging_for_mocked_team,
 ):
     """
-    The reconcile sweep runs after the team row is committed deleted. If it ran before cache
-    eviction, a sweep failure would return an error with the team gone from the db but still
-    served from cache, which is the exact bug this PR exists to fix.
+    The team delete and its post-delete reconcile sweep run inside one transaction, under the
+    team's advisory lock, so a sweep failure rolls the delete back with it rather than leaving
+    the row gone with the sweep half done. Cache eviction only runs after that transaction
+    commits, so a failure here must leave the team exactly as it was: still in the db, and
+    still cached. Evicting a cache entry for a delete that never actually committed would be
+    the same class of bug this PR exists to fix, just on the other side of the transaction.
     """
     from litellm.proxy._types import DeleteTeamRequest
     from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
@@ -7650,7 +7867,7 @@ async def test_delete_team_failing_reconcile_sweep_cannot_strand_the_team_in_cac
     mock_prisma_client.db.litellm_deletedteamtable.create_many = AsyncMock()
     mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[])
     mock_prisma_client.db.litellm_teammembership.delete_many = AsyncMock()
-    # the first sweep succeeds, the post-delete reconcile sweep blows up
+    # the unlocked pre-delete sweep succeeds, the locked post-delete sweep blows up
     mock_prisma_client.db.execute_raw = AsyncMock(side_effect=[None, ConnectionError("db went away")])
 
     mock_tx = AsyncMock()
@@ -7659,6 +7876,7 @@ async def test_delete_team_failing_reconcile_sweep_cannot_strand_the_team_in_cac
     mock_tx_cm.__aenter__ = AsyncMock(return_value=mock_tx)
     mock_tx_cm.__aexit__ = AsyncMock(return_value=False)
     mock_prisma_client.db.tx = MagicMock(return_value=mock_tx_cm)
+    _wire_team_delete_tx(mock_prisma_client)
 
     fresh_cache = UserApiKeyCache()
     cached_obj = LiteLLM_TeamTableCachedObj(team_id="team-doomed", team_alias="doomed-team")
@@ -7682,9 +7900,10 @@ async def test_delete_team_failing_reconcile_sweep_cannot_strand_the_team_in_cac
             litellm_changed_by="admin-user",
         )
 
-    # the delete committed, so the cache must not still be serving the team
-    assert fresh_cache.get_cache(key="team_id:team-doomed") is None
-    assert fresh_cache.get_cache(key="team_alias:doomed-team") is None
+    # the transaction that deletes the row and runs the locked sweep never committed, so
+    # cache eviction (which only runs after that commit) must never have been reached
+    assert fresh_cache.get_cache(key="team_id:team-doomed") is not None
+    assert fresh_cache.get_cache(key="team_alias:doomed-team") is not None
 
 
 @pytest.mark.asyncio
@@ -7726,6 +7945,7 @@ async def test_delete_team_broadcasts_cache_invalidation_to_other_workers(
     mock_tx_cm.__aenter__ = AsyncMock(return_value=mock_tx)
     mock_tx_cm.__aexit__ = AsyncMock(return_value=False)
     mock_prisma_client.db.tx = MagicMock(return_value=mock_tx_cm)
+    _wire_team_delete_tx(mock_prisma_client)
 
     published = []
 
@@ -7796,6 +8016,7 @@ async def test_delete_team_survives_a_failing_cache_backend(
     mock_tx_cm.__aenter__ = AsyncMock(return_value=mock_tx)
     mock_tx_cm.__aexit__ = AsyncMock(return_value=False)
     mock_prisma_client.db.tx = MagicMock(return_value=mock_tx_cm)
+    _wire_team_delete_tx(mock_prisma_client)
 
     exploding_logging_obj = MagicMock()
     exploding_logging_obj.internal_usage_cache.dual_cache.async_delete_cache = AsyncMock(
@@ -7820,7 +8041,7 @@ async def test_delete_team_survives_a_failing_cache_backend(
     )
 
     assert result == {"deleted_teams": ["team-doomed"]}
-    mock_delete_data.assert_any_await(team_id_list=["team-doomed"], table_name="team")
+    mock_prisma_client.db.litellm_teamtable.delete_many.assert_any_await(where={"team_id": {"in": ["team-doomed"]}})
     assert exploding_logging_obj.internal_usage_cache.dual_cache.async_delete_cache.await_count > 0
 
 
diff --git a/tests/test_litellm/proxy/management_helpers/test_management_helpers_utils.py b/tests/test_litellm/proxy/management_helpers/test_management_helpers_utils.py
index bdc2f9065b9..a6b1fc32eda 100644
--- a/tests/test_litellm/proxy/management_helpers/test_management_helpers_utils.py
+++ b/tests/test_litellm/proxy/management_helpers/test_management_helpers_utils.py
@@ -1120,3 +1120,81 @@ async def test_add_new_member_creates_missing_user_atomically_via_upsert():
     assert upsert_data["create"]["teams"] == ["team-1"]
     assert upsert_data["update"], "empty update branch degrades the upsert to a racy SELECT-then-INSERT"
     assert "teams" not in upsert_data["update"]
+
+
+def _member_write_tx() -> MagicMock:
+    tx = MagicMock()
+    created_user = MagicMock()
+    created_user.user_id = "pool-user"
+    created_user.model_dump.return_value = {
+        "user_id": "pool-user",
+        "user_email": "pool@example.com",
+        "teams": ["team-pool"],
+        "user_role": "internal_user",
+    }
+    created_budget = MagicMock()
+    created_budget.budget_id = "budget-pool"
+    membership = MagicMock()
+    membership.model_dump.return_value = {
+        "team_id": "team-pool",
+        "user_id": "pool-user",
+        "budget_id": "budget-pool",
+        "litellm_budget_table": None,
+    }
+    tx.litellm_usertable.upsert = AsyncMock(return_value=created_user)
+    tx.litellm_usertable.create = AsyncMock(return_value=created_user)
+    tx.litellm_usertable.update_many = AsyncMock()
+    tx.litellm_usertable.find_many = AsyncMock(return_value=[])
+    tx.litellm_budgettable.find_unique = AsyncMock(return_value=None)
+    tx.litellm_budgettable.create = AsyncMock(return_value=created_budget)
+    tx.litellm_teammembership.create = AsyncMock(return_value=membership)
+    return tx
+
+
+@pytest.mark.parametrize(
+    "new_member",
+    [
+        Member(user_id="pool-user", role="user"),
+        Member(user_email="pool@example.com", role="user"),
+    ],
+    ids=["by_user_id", "by_user_email"],
+)
+@pytest.mark.asyncio
+async def test_add_new_member_runs_every_write_on_the_caller_transaction(new_member):
+    """
+    Regression pin against exhausting the connection pool with advisory-lock waiters.
+
+    /team/member_add calls this while holding the team's advisory lock inside a transaction,
+    so it already owns a pooled connection. Any query issued on the regular client here needs
+    a second one, and enough concurrent adds for one team leave every connection parked on the
+    lock while the holder waits for a free one, so nothing ever commits or releases the lock.
+    Given a transaction, every read and write has to go through it.
+    """
+    from litellm.proxy._types import LitellmUserRoles
+
+    tx = _member_write_tx()
+    prisma_client = AsyncMock()
+
+    result_user, result_membership = await add_new_member(
+        new_member=new_member,
+        max_budget_in_team=50.0,
+        prisma_client=prisma_client,
+        team_id="team-pool",
+        user_api_key_dict=UserAPIKeyAuth(
+            user_id="admin_user", user_role=LitellmUserRoles.PROXY_ADMIN
+        ),
+        litellm_proxy_admin_name="admin",
+        tx=tx,
+    )
+
+    assert result_user.user_id == "pool-user"
+    assert result_membership is not None
+    assert result_membership.budget_id == "budget-pool"
+
+    assert tx.litellm_budgettable.create.await_count == 1
+    assert tx.litellm_teammembership.create.await_count == 1
+    assert tx.litellm_usertable.upsert.await_count + tx.litellm_usertable.create.await_count == 1
+
+    prisma_client.db.assert_not_called()
+    prisma_client.get_data.assert_not_awaited()
+    prisma_client.insert_data.assert_not_awaited()
diff --git a/tests/test_litellm/repositories/test_repositories.py b/tests/test_litellm/repositories/test_repositories.py
index 38af52f165c..758d379f22c 100644
--- a/tests/test_litellm/repositories/test_repositories.py
+++ b/tests/test_litellm/repositories/test_repositories.py
@@ -541,17 +541,19 @@ class TestTeamRepository:
 
         assert [m.user_id for m in members] == expected_ids
         sql = tx.query_raw.call_args.args[0]
-        assert "FOR UPDATE" in sql
+        assert "FOR UPDATE" not in sql, (
+            "a row lock here can deadlock with the access-group endpoints; the caller must "
+            "already hold the team's advisory lock, so a plain read is all this needs"
+        )
         assert tx.query_raw.call_args.args[1] == "team-1"
 
     @pytest.mark.asyncio
     async def test_get_members_with_roles_locked_missing_row(self, repo):
         """None, not [], so a caller can tell a deleted team from an empty one.
 
-        /team/member_add reconciles membership under this lock and has to fail,
-        and clean up the references it already wrote, when a /team/delete
-        committed underneath it. An empty list would look like a live team with
-        no members and it would carry on writing.
+        /team/member_add reconciles membership under the team's advisory lock and has to
+        fail, without writing anything, when a /team/delete committed underneath it. An
+        empty list would look like a live team with no members and it would carry on writing.
         """
         tx = MagicMock()
         tx.query_raw = AsyncMock(return_value=[])

From 61ee2f5ecbb602fb42a8f8b9d2f4316bdc0678a4 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 15:00:57 -0700
Subject: [PATCH 062/105] fix(anthropic): carry user-content document blocks
 through the /v1/messages responses bridge

---
 .../responses_adapters/transformation.py      |  7 ++
 .../test_responses_adapters_transformation.py | 81 +++++++++++++++++++
 2 files changed, 88 insertions(+)

diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py
index d9d62e0719f..492e0050cfe 100644
--- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py
+++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py
@@ -214,6 +214,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
           system text        -> message(role=system, input_text)
           user text          -> message(role=user, input_text)
           user image         -> message(role=user, input_image)
+          user document      -> message(role=user, input_file)
           user tool_result   -> function_call_output
           assistant text     -> message(role=assistant, output_text)
           assistant thinking -> reasoning
@@ -268,6 +269,12 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
                                         {"type": "input_image", "image_url": url}, block.get("prompt_cache_breakpoint")
                                     )
                                 )
+                        elif btype == "document":
+                            file_part = self._translate_anthropic_document_block_to_file_part(block)
+                            if file_part:
+                                user_parts.append(
+                                    with_prompt_cache_breakpoint(file_part, block.get("prompt_cache_breakpoint"))
+                                )
                         elif btype == "tool_result":
                             tool_use_id = block.get("tool_use_id", "")
                             inner = block.get("content")
diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py
index 44c956c8ee8..56f106e407c 100644
--- a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py
+++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py
@@ -1688,6 +1688,87 @@ class TestToolResultDocuments:
         ]
 
 
+class TestUserContentDocuments:
+    """Documents in plain user content must survive translation (LIT-6144): each
+    document block becomes an input_file part of the user message, in block order,
+    exactly like image blocks become input_image parts. Untranslatable documents
+    are dropped without disturbing the surrounding parts."""
+
+    PDF_B64 = "JVBERi0xLjQKJSBQT05H"
+    PDF_DATA_URI = "data:application/pdf;base64,JVBERi0xLjQKJSBQT05H"
+    PDF_URL = "https://example.com/report.pdf"
+    EXPLICIT = {"mode": "explicit"}
+
+    def _translate(self, user_content):
+        return _ADAPTER.translate_messages_to_responses_input([{"role": "user", "content": user_content}])
+
+    @staticmethod
+    def _user_content(items):
+        return next(item for item in items if item.get("type") == "message" and item.get("role") == "user")["content"]
+
+    def _base64_document(self, **extra):
+        return {
+            "type": "document",
+            "source": {"type": "base64", "media_type": "application/pdf", "data": self.PDF_B64},
+            **extra,
+        }
+
+    def test_document_then_text_keeps_block_order(self):
+        content = self._user_content(
+            self._translate([self._base64_document(), {"type": "text", "text": "what does the pdf say?"}])
+        )
+        assert content == [
+            {"type": "input_file", "filename": "document.pdf", "file_data": self.PDF_DATA_URI},
+            {"type": "input_text", "text": "what does the pdf say?"},
+        ]
+
+    def test_document_title_becomes_filename(self):
+        content = self._user_content(self._translate([self._base64_document(title="quarterly-report.pdf")]))
+        assert content == [
+            {"type": "input_file", "filename": "quarterly-report.pdf", "file_data": self.PDF_DATA_URI}
+        ]
+
+    def test_url_document_becomes_file_url_part(self):
+        content = self._user_content(
+            self._translate([{"type": "document", "source": {"type": "url", "url": self.PDF_URL}}])
+        )
+        assert content == [{"type": "input_file", "file_url": self.PDF_URL}]
+
+    def test_document_only_content_still_produces_user_message(self):
+        content = self._user_content(self._translate([self._base64_document()]))
+        assert content == [{"type": "input_file", "filename": "document.pdf", "file_data": self.PDF_DATA_URI}]
+
+    def test_empty_base64_data_drops_only_the_document_part(self):
+        content = self._user_content(
+            self._translate(
+                [
+                    {"type": "text", "text": "still here"},
+                    {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": ""}},
+                ]
+            )
+        )
+        assert content == [{"type": "input_text", "text": "still here"}]
+
+    def test_non_dict_source_drops_only_the_document_part(self):
+        content = self._user_content(
+            self._translate([{"type": "text", "text": "still here"}, {"type": "document", "source": self.PDF_URL}])
+        )
+        assert content == [{"type": "input_text", "text": "still here"}]
+
+    def test_document_breakpoint_rides_on_the_file_part(self):
+        content = self._user_content(
+            self._translate([self._base64_document(prompt_cache_breakpoint=self.EXPLICIT)])
+        )
+        assert content == [
+            {
+                "type": "input_file",
+                "filename": "document.pdf",
+                "file_data": self.PDF_DATA_URI,
+                "prompt_cache_breakpoint": self.EXPLICIT,
+            }
+        ]
+
+
 def _contains_key(value, key) -> bool:
     if isinstance(value, dict):
         return key in value or any(_contains_key(v, key) for v in value.values())

From 5ab9e63628088f13024f5a949259bec69855481e Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 15:24:58 -0700
Subject: [PATCH 063/105] fix(together_ai): fail open on response_format
 instead of dropping it for unregistered models

---
 .../llms/together_ai/chat/transformation.py   |  79 ++++++---
 .../test_together_ai_chat_transformation.py   | 158 ++++++++++++++++--
 2 files changed, 195 insertions(+), 42 deletions(-)

diff --git a/litellm/llms/together_ai/chat/transformation.py b/litellm/llms/together_ai/chat/transformation.py
index 3162a34f1b9..5f0ab5e56af 100644
--- a/litellm/llms/together_ai/chat/transformation.py
+++ b/litellm/llms/together_ai/chat/transformation.py
@@ -4,32 +4,47 @@ Translates from OpenAI's `/v1/chat/completions` to Together AI's `/v1/chat/compl
 Docs: https://docs.together.ai/docs/chat-overview
 """
 
-from collections.abc import Container
-from types import MappingProxyType
+from collections.abc import Callable, Container
 from typing import Final
 
 import litellm
 from litellm._logging import verbose_logger
 from litellm.exceptions import UnsupportedParamsError
-from litellm.utils import supports_function_calling
+from litellm.utils import supports_function_calling, supports_response_schema
 
 from ...openai.chat.gpt_transformation import OpenAIGPTConfig
 
 TOOL_CALLING_PARAMS: Final = ("tools", "tool_choice", "function_call")
-PLAIN_TEXT_RESPONSE_FORMAT: Final = MappingProxyType({"type": "text"})
 FUNCTION_CALLING_DOCS_URL: Final = "https://docs.together.ai/docs/function-calling"
+STRUCTURED_OUTPUTS_DOCS_URL: Final = "https://docs.together.ai/docs/inference/chat/structured-outputs"
+
+
+def _registry_verdict(model: str, flag: str, check: Callable[[str], bool]) -> bool | None:
+    try:
+        if check(model):
+            return True
+    except Exception as e:
+        verbose_logger.debug("Error checking together_ai %s for %s: %s", flag, model, e)
+    registry_entry: Final = litellm.model_cost.get(f"together_ai/{model}")
+    if isinstance(registry_entry, dict) and registry_entry.get(flag) is False:
+        return False
+    return None
 
 
 def _function_calling_verdict(model: str) -> bool | None:
-    try:
-        if supports_function_calling(model, custom_llm_provider="together_ai"):
-            return True
-    except Exception as e:
-        verbose_logger.debug("Error checking together_ai function calling support for %s: %s", model, e)
-    registry_entry: Final = litellm.model_cost.get(f"together_ai/{model}")
-    if isinstance(registry_entry, dict) and registry_entry.get("supports_function_calling") is False:
-        return False
-    return None
+    return _registry_verdict(
+        model,
+        "supports_function_calling",
+        lambda checked_model: supports_function_calling(checked_model, custom_llm_provider="together_ai"),
+    )
+
+
+def _response_schema_verdict(model: str) -> bool | None:
+    return _registry_verdict(
+        model,
+        "supports_response_schema",
+        lambda checked_model: supports_response_schema(checked_model, custom_llm_provider="together_ai"),
+    )
 
 
 def _tool_params_to_drop(passed_params: Container[str], model: str, drop_params: bool) -> tuple[str, ...]:
@@ -61,19 +76,33 @@ def _tool_params_to_drop(passed_params: Container[str], model: str, drop_params:
     )
 
 
-class TogetherAIChatConfig(OpenAIGPTConfig):
-    def get_supported_openai_params(self, model: str) -> list:
-        supports_fc: Final = _function_calling_verdict(model)
-        supported_params: Final = super().get_supported_openai_params(model)
-        if supports_fc is True:
-            return supported_params
-        verbose_logger.debug(
-            "Only some together models support response_format. Docs - https://docs.together.ai/docs/function-calling"
+def _drop_response_format(passed_params: Container[str], model: str, drop_params: bool) -> bool:
+    if "response_format" not in passed_params:
+        return False
+    verdict: Final = _response_schema_verdict(model)
+    if verdict is True:
+        return False
+    if verdict is None:
+        verbose_logger.warning(
+            "together_ai model %s has no structured outputs entry in the model registry; passing response_format through for Together to validate. Docs - %s",
+            model,
+            STRUCTURED_OUTPUTS_DOCS_URL,
         )
-        return [  # mutable-ok: the inherited contract returns a plain list; building fresh avoids mutating the base class's value
-            param for param in supported_params if param != "response_format"
-        ]
+        return False
+    if drop_params or litellm.drop_params:
+        verbose_logger.warning(
+            "together_ai model %s does not support structured outputs per the model registry; dropping response_format. Docs - %s",
+            model,
+            STRUCTURED_OUTPUTS_DOCS_URL,
+        )
+        return True
+    raise UnsupportedParamsError(
+        status_code=500,
+        message=f"together_ai does not support parameters: response_format, for model={model}. To drop it from the call, set `litellm.drop_params = True`.",
+    )
 
+
+class TogetherAIChatConfig(OpenAIGPTConfig):
     def map_openai_params(
         self,
         non_default_params: dict,
@@ -84,6 +113,6 @@ class TogetherAIChatConfig(OpenAIGPTConfig):
         mapped_openai_params: Final = super().map_openai_params(non_default_params, optional_params, model, drop_params)
         for param in _tool_params_to_drop(mapped_openai_params, model, drop_params):
             mapped_openai_params.pop(param)
-        if mapped_openai_params.get("response_format") == PLAIN_TEXT_RESPONSE_FORMAT:
+        if _drop_response_format(mapped_openai_params, model, drop_params):
             mapped_openai_params.pop("response_format")
         return mapped_openai_params
diff --git a/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py b/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py
index 0b9fd5364f9..3848a9c7e6c 100644
--- a/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py
+++ b/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py
@@ -18,11 +18,24 @@ TOOL_CALLING_MODEL = "openai/gpt-oss-20b"
 REASONING_MODEL = "deepseek-ai/DeepSeek-V3.1"
 UNMAPPED_MODEL = "example-org/brand-new-model"
 NO_TOOLS_MODEL = "example-org/no-tools-model"
+NO_SCHEMA_MODEL = "example-org/no-schema-model"
 
 TOOL_PARAMS = ("tools", "tool_choice", "function_call")
 
 WEATHER_TOOLS = [{"type": "function", "function": {"name": "get_weather", "parameters": {}}}]
 
+VOICE_NOTE_SCHEMA = {
+    "type": "object",
+    "properties": {"title": {"type": "string"}, "summary": {"type": "string"}},
+    "required": ["title", "summary"],
+    "additionalProperties": False,
+}
+JSON_SCHEMA_RESPONSE_FORMAT = {
+    "type": "json_schema",
+    "json_schema": {"name": "voice_note", "schema": VOICE_NOTE_SCHEMA, "strict": True},
+}
+REGEX_RESPONSE_FORMAT = {"type": "regex", "pattern": "(positive|neutral|negative)"}
+
 
 @pytest.fixture(autouse=True)
 def force_local_model_cost(monkeypatch):
@@ -41,6 +54,15 @@ def registry_disables_function_calling(monkeypatch):
     )
 
 
+@pytest.fixture
+def registry_disables_response_schema(monkeypatch):
+    monkeypatch.setitem(
+        litellm.model_cost,
+        f"together_ai/{NO_SCHEMA_MODEL}",
+        {"litellm_provider": "together_ai", "mode": "chat", "supports_response_schema": False},
+    )
+
+
 @pytest.fixture
 def together_warning_log(caplog):
     from litellm._logging import verbose_logger
@@ -63,7 +85,7 @@ def test_supported_params_unmapped_model_keeps_tool_params():
 
     for param in TOOL_PARAMS:
         assert param in supported
-    assert "response_format" not in supported
+    assert "response_format" in supported
     assert "stream" in supported
     assert "temperature" in supported
 
@@ -73,7 +95,7 @@ def test_supported_params_no_tools_model_keeps_tool_params(registry_disables_fun
 
     for param in TOOL_PARAMS:
         assert param in supported
-    assert "response_format" not in supported
+    assert "response_format" in supported
 
 
 def test_map_openai_params_tool_calling_model_passes_tools():
@@ -141,21 +163,17 @@ def test_map_openai_params_reasoning_model_passes_sampling_params():
     assert mapped["max_tokens"] == 512
 
 
-def test_map_openai_params_drops_text_response_format():
-    mapped = TogetherAIChatConfig().map_openai_params(
-        non_default_params={"response_format": {"type": "text"}, "temperature": 0.5},
-        optional_params={},
-        model=REASONING_MODEL,
-        drop_params=False,
-    )
-
-    assert "response_format" not in mapped
-    assert mapped["temperature"] == 0.5
-
-
-def test_map_openai_params_keeps_json_response_format():
-    response_format = {"type": "json_object"}
-
+@pytest.mark.parametrize(
+    "response_format",
+    [
+        {"type": "text"},
+        {"type": "json_object"},
+        {"type": "json_object", "schema": VOICE_NOTE_SCHEMA},
+        JSON_SCHEMA_RESPONSE_FORMAT,
+        REGEX_RESPONSE_FORMAT,
+    ],
+)
+def test_map_openai_params_schema_model_passes_response_format_through(response_format):
     mapped = TogetherAIChatConfig().map_openai_params(
         non_default_params={"response_format": response_format},
         optional_params={},
@@ -166,6 +184,46 @@ def test_map_openai_params_keeps_json_response_format():
     assert mapped["response_format"] == response_format
 
 
+@pytest.mark.parametrize("drop_params", [False, True])
+def test_map_openai_params_unmapped_model_passes_response_format_through(drop_params, together_warning_log):
+    mapped = TogetherAIChatConfig().map_openai_params(
+        non_default_params={"response_format": JSON_SCHEMA_RESPONSE_FORMAT},
+        optional_params={},
+        model=UNMAPPED_MODEL,
+        drop_params=drop_params,
+    )
+
+    assert mapped["response_format"] == JSON_SCHEMA_RESPONSE_FORMAT
+    assert UNMAPPED_MODEL in together_warning_log.text
+    assert "passing response_format through" in together_warning_log.text
+
+
+def test_map_openai_params_no_schema_model_drops_response_format_with_warning(
+    registry_disables_response_schema, together_warning_log
+):
+    mapped = TogetherAIChatConfig().map_openai_params(
+        non_default_params={"response_format": JSON_SCHEMA_RESPONSE_FORMAT, "temperature": 0.5},
+        optional_params={},
+        model=NO_SCHEMA_MODEL,
+        drop_params=True,
+    )
+
+    assert "response_format" not in mapped
+    assert mapped["temperature"] == 0.5
+    assert NO_SCHEMA_MODEL in together_warning_log.text
+    assert "dropping response_format" in together_warning_log.text
+
+
+def test_map_openai_params_no_schema_model_raises_without_drop_params(registry_disables_response_schema):
+    with pytest.raises(UnsupportedParamsError, match="response_format"):
+        TogetherAIChatConfig().map_openai_params(
+            non_default_params={"response_format": JSON_SCHEMA_RESPONSE_FORMAT},
+            optional_params={},
+            model=NO_SCHEMA_MODEL,
+            drop_params=False,
+        )
+
+
 def _transform_response(message: dict) -> ModelResponse:
     raw_response_json = {
         "id": "chatcmpl-test",
@@ -385,3 +443,69 @@ def test_completion_unmapped_model_sends_tools_to_together():
     tool_call = response.choices[0].message.tool_calls[0]
     assert tool_call.function.name == "get_weather"
     assert json.loads(tool_call.function.arguments) == {"city": "San Francisco"}
+
+
+def _capture_completion_request(model: str, **completion_kwargs) -> dict:
+    from litellm.llms.custom_httpx.http_handler import HTTPHandler
+
+    captured_requests = []
+
+    def respond(request: httpx.Request) -> httpx.Response:
+        captured_requests.append(request)
+        return httpx.Response(
+            200,
+            json={
+                "id": "chatcmpl-together-structured",
+                "object": "chat.completion",
+                "created": 1234567890,
+                "model": model,
+                "choices": [
+                    {
+                        "index": 0,
+                        "message": {"role": "assistant", "content": '{"title": "t", "summary": "s"}'},
+                        "finish_reason": "stop",
+                    }
+                ],
+                "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
+            },
+        )
+
+    client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond)))
+    litellm.completion(
+        model=f"together_ai/{model}",
+        messages=[{"role": "user", "content": "Summarize with a title and summary."}],
+        api_key="fake-key",
+        client=client,
+        **completion_kwargs,
+    )
+    return json.loads(captured_requests[0].content)
+
+
+def test_completion_unmapped_model_sends_json_schema_to_together():
+    request_body = _capture_completion_request(
+        UNMAPPED_MODEL, response_format=JSON_SCHEMA_RESPONSE_FORMAT, drop_params=True
+    )
+
+    assert request_body["response_format"] == JSON_SCHEMA_RESPONSE_FORMAT
+
+
+def test_completion_pydantic_response_format_sends_json_schema_to_together():
+    from pydantic import BaseModel
+
+    class VoiceNote(BaseModel):
+        title: str
+        summary: str
+
+    request_body = _capture_completion_request(TOOL_CALLING_MODEL, response_format=VoiceNote)
+
+    sent = request_body["response_format"]
+    assert sent["type"] == "json_schema"
+    assert sent["json_schema"]["name"] == "VoiceNote"
+    assert sent["json_schema"]["strict"] is True
+    assert sent["json_schema"]["schema"]["required"] == ["title", "summary"]
+
+
+def test_completion_regex_response_format_sends_pattern_to_together():
+    request_body = _capture_completion_request(TOOL_CALLING_MODEL, response_format=REGEX_RESPONSE_FORMAT)
+
+    assert request_body["response_format"] == REGEX_RESPONSE_FORMAT

From 3ebd18b057cc3ea301c3a001ed0e693bc019b61e Mon Sep 17 00:00:00 2001
From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 25 Aug 2026 22:37:02 +0000
Subject: [PATCH 064/105] fix(ui): stack policy flow builder below the popup
 layer so guardrail options render

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../pipeline_flow_builder.test.tsx            | 23 ++++++++++++++++++-
 .../_components/pipeline_flow_builder.tsx     |  2 +-
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/pipeline_flow_builder.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/pipeline_flow_builder.test.tsx
index 59bd63fd0ed..3af5b31c41f 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/pipeline_flow_builder.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/pipeline_flow_builder.test.tsx
@@ -3,7 +3,7 @@ import { describe, expect, it, vi } from "vitest";
 import { fireEvent, screen } from "@testing-library/react";
 import userEvent from "@testing-library/user-event";
 import { renderWithProviders } from "@/../tests/test-utils";
-import PipelineFlowBuilder, { PipelineInfoDisplay } from "./pipeline_flow_builder";
+import PipelineFlowBuilder, { FlowBuilderPage, PipelineInfoDisplay } from "./pipeline_flow_builder";
 import { GuardrailPipeline, PipelineStep } from "@/components/policies/types";
 import { Guardrail } from "@/components/guardrails/types";
 
@@ -167,3 +167,24 @@ describe("PipelineFlowBuilder", () => {
     expect(screen.getAllByRole("combobox").length).toBeGreaterThan(0);
   });
 });
+
+describe("FlowBuilderPage", () => {
+  it("stacks its full-screen shell below the portalled popup layer", () => {
+    const portalLayerZIndex = 50;
+    const { container } = renderWithProviders(
+      <FlowBuilderPage
+        onBack={vi.fn()}
+        onSuccess={vi.fn()}
+        accessToken="sk-test"
+        availableGuardrails={guardrails}
+        createPolicy={vi.fn()}
+        updatePolicy={vi.fn()}
+      />,
+    );
+
+    const shell = container.firstElementChild as HTMLElement;
+
+    expect(shell).toHaveStyle({ position: "fixed" });
+    expect(Number(shell.style.zIndex)).toBeLessThan(portalLayerZIndex);
+  });
+});
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/pipeline_flow_builder.tsx b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/pipeline_flow_builder.tsx
index 8af8ac7a414..c649bb4844b 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/pipeline_flow_builder.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/policies/_components/pipeline_flow_builder.tsx
@@ -1631,7 +1631,7 @@ export const FlowBuilderPage: React.FC<FlowBuilderPageProps> = ({
         right: 0,
         bottom: 0,
         backgroundColor: "var(--color-muted)",
-        zIndex: 1000,
+        zIndex: 40,
         display: "flex",
         flexDirection: "column",
         overflow: "hidden",

From 8298cbe1f71f771492c3adfcd0569bb21219ce92 Mon Sep 17 00:00:00 2001
From: ryan-crabbe-berri <ryan@berri.ai>
Date: Tue, 25 Aug 2026 15:43:26 -0700
Subject: [PATCH 065/105] test: gate the test tree on B003 so a test cannot
 swap os.environ for a plain dict

---
 ruff-tests.toml                         |  5 ++++
 tests/litellm_utils_tests/test_utils.py | 35 +++++++++----------------
 2 files changed, 17 insertions(+), 23 deletions(-)

diff --git a/ruff-tests.toml b/ruff-tests.toml
index 7d28d6d838a..e036fb4946c 100644
--- a/ruff-tests.toml
+++ b/ruff-tests.toml
@@ -51,6 +51,10 @@
 # F632    `is` against a literal. It compares identity, so it passes only where CPython
 #         happens to intern the value and stops meaning what it says the moment the
 #         value is built at runtime
+# B003    `os.environ = {...}` rebinds the mapping instead of mutating it, so `putenv`
+#         never fires and a subprocess still reads the real keys the test believes it
+#         cleared. The manual restore underneath is skipped whenever the body raises,
+#         so every later test in that worker inherits a plain dict for an environment
 #
 # No target-version here on purpose: it resolves from requires-python (>=3.10), so
 # 3.11-only builtins like BaseExceptionGroup are correctly flagged in a tree that
@@ -78,4 +82,5 @@ lint.select = [
     "B023",
     "B025",
     "F632",
+    "B003",
 ]
diff --git a/tests/litellm_utils_tests/test_utils.py b/tests/litellm_utils_tests/test_utils.py
index 67f2e1ce06d..0ccfae55290 100644
--- a/tests/litellm_utils_tests/test_utils.py
+++ b/tests/litellm_utils_tests/test_utils.py
@@ -328,34 +328,23 @@ def test_trimming_with_untokenizable_field(caplog: pytest.LogCaptureFixture) ->
 
 
 def test_aget_valid_models():
-    old_environ = os.environ
-    os.environ = {"OPENAI_API_KEY": "temp"}  # mock set only openai key in environ
+    with mock.patch.dict(os.environ, {"OPENAI_API_KEY": "temp"}, clear=True):
+        valid_models = get_valid_models()
+        print(valid_models)
 
-    valid_models = get_valid_models()
-    print(valid_models)
+        # list of openai supported llms on litellm
+        expected_models = (
+            litellm.open_ai_chat_completion_models | litellm.open_ai_text_completion_models
+        )
 
-    # list of openai supported llms on litellm
-    expected_models = (
-        litellm.open_ai_chat_completion_models | litellm.open_ai_text_completion_models
-    )
-
-    assert set(valid_models) == set(expected_models)
-
-    # reset replicate env key
-    os.environ = old_environ
+        assert set(valid_models) == set(expected_models)
 
     # GEMINI
-    expected_models = litellm.gemini_models
-    old_environ = os.environ
-    os.environ = {"GEMINI_API_KEY": "temp"}  # mock set only openai key in environ
+    with mock.patch.dict(os.environ, {"GEMINI_API_KEY": "temp"}, clear=True):
+        valid_models = get_valid_models()
 
-    valid_models = get_valid_models()
-
-    print(valid_models)
-    assert set(valid_models) == set(expected_models)
-
-    # reset replicate env key
-    os.environ = old_environ
+        print(valid_models)
+        assert set(valid_models) == set(litellm.gemini_models)
 
 
 @pytest.mark.parametrize("custom_llm_provider", ["anthropic", "xai"])

From 1ba66b1555fdbe088e8cc6ce55d67ad7dae4cd63 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 15:48:58 -0700
Subject: [PATCH 066/105] fix(together_ai): strip internal thinking fields from
 outbound messages, pin chat_template_kwargs passthrough

---
 .../llms/together_ai/chat/transformation.py   | 54 ++++++++++-
 .../test_together_ai_chat_transformation.py   | 90 +++++++++++++++++++
 2 files changed, 142 insertions(+), 2 deletions(-)

diff --git a/litellm/llms/together_ai/chat/transformation.py b/litellm/llms/together_ai/chat/transformation.py
index 3162a34f1b9..6bf53ee348b 100644
--- a/litellm/llms/together_ai/chat/transformation.py
+++ b/litellm/llms/together_ai/chat/transformation.py
@@ -4,18 +4,25 @@ Translates from OpenAI's `/v1/chat/completions` to Together AI's `/v1/chat/compl
 Docs: https://docs.together.ai/docs/chat-overview
 """
 
-from collections.abc import Container
+from collections.abc import Container, Coroutine
 from types import MappingProxyType
-from typing import Final
+from typing import (
+    Final,
+    Literal,
+    cast,  # noqa: TID251  # rebuilding a TypedDict minus keys has no checked spelling
+    overload,
+)
 
 import litellm
 from litellm._logging import verbose_logger
 from litellm.exceptions import UnsupportedParamsError
+from litellm.types.llms.openai import AllMessageValues
 from litellm.utils import supports_function_calling
 
 from ...openai.chat.gpt_transformation import OpenAIGPTConfig
 
 TOOL_CALLING_PARAMS: Final = ("tools", "tool_choice", "function_call")
+LITELLM_INTERNAL_ASSISTANT_FIELDS: Final = frozenset({"thinking_blocks", "provider_specific_fields"})
 PLAIN_TEXT_RESPONSE_FORMAT: Final = MappingProxyType({"type": "text"})
 FUNCTION_CALLING_DOCS_URL: Final = "https://docs.together.ai/docs/function-calling"
 
@@ -61,7 +68,50 @@ def _tool_params_to_drop(passed_params: Container[str], model: str, drop_params:
     )
 
 
+def _without_litellm_internal_fields(message: AllMessageValues) -> AllMessageValues:
+    if message["role"] != "assistant" or LITELLM_INTERNAL_ASSISTANT_FIELDS.isdisjoint(message):
+        return message
+    return cast(  # cast-ok: rebuilding the same TypedDict minus internal keys loses the narrowed type
+        "AllMessageValues",
+        {  # mutable-ok: TypedDict rebuild minus internal keys
+            key: value for key, value in message.items() if key not in LITELLM_INTERNAL_ASSISTANT_FIELDS
+        },
+    )
+
+
 class TogetherAIChatConfig(OpenAIGPTConfig):
+    @overload
+    def _transform_messages(
+        self,
+        messages: list[AllMessageValues],  # mutable-ok: inherited contract
+        model: str,
+        is_async: Literal[True],
+    ) -> Coroutine[object, object, list[AllMessageValues]]: ...  # mutable-ok: inherited contract
+
+    @overload
+    def _transform_messages(
+        self,
+        messages: list[AllMessageValues],  # mutable-ok: inherited contract
+        model: str,
+        is_async: Literal[False] = False,
+    ) -> list[AllMessageValues]: ...  # mutable-ok: inherited contract
+
+    def _transform_messages(
+        self,
+        messages: list[AllMessageValues],  # mutable-ok: inherited contract
+        model: str,
+        is_async: bool = False,
+    ) -> list[AllMessageValues] | Coroutine[object, object, list[AllMessageValues]]:  # mutable-ok: inherited contract
+        """Together consumes replayed assistant `reasoning_content` (preserved thinking via
+        `chat_template_kwargs: {"clear_thinking": false}`), so it must stay in the payload;
+        only litellm-internal fields are stripped before sending."""
+        stripped: Final = [  # mutable-ok: super() requires a list
+            _without_litellm_internal_fields(message) for message in messages
+        ]
+        if is_async:
+            return super()._transform_messages(stripped, model, is_async=True)
+        return super()._transform_messages(stripped, model, is_async=False)
+
     def get_supported_openai_params(self, model: str) -> list:
         supports_fc: Final = _function_calling_verdict(model)
         supported_params: Final = super().get_supported_openai_params(model)
diff --git a/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py b/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py
index 0b9fd5364f9..110d93cbc0d 100644
--- a/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py
+++ b/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py
@@ -268,6 +268,96 @@ def test_streaming_chunk_preserves_tool_call_index_and_id():
     assert continuation["function"]["arguments"] == '{"city": "San'
 
 
+REPLAYED_ASSISTANT_MESSAGE = {
+    "role": "assistant",
+    "content": "The digit sum is 11.",
+    "reasoning_content": "The secret number is 47. 4 + 7 = 11.",
+    "thinking_blocks": [{"type": "thinking", "thinking": "The secret number is 47.", "signature": ""}],
+    "provider_specific_fields": {"thinking_blocks": [{"type": "thinking", "thinking": "The secret number is 47."}]},
+}
+
+PRESERVED_THINKING_MESSAGES = [
+    {"role": "user", "content": "Pick a secret two-digit number and tell me only its digit sum."},
+    REPLAYED_ASSISTANT_MESSAGE,
+    {"role": "user", "content": "What was the secret number?"},
+]
+
+
+def _assert_internal_fields_stripped_reasoning_kept(transformed_messages: list):
+    assistant_message = transformed_messages[1]
+    assert assistant_message["reasoning_content"] == REPLAYED_ASSISTANT_MESSAGE["reasoning_content"]
+    assert "thinking_blocks" not in assistant_message
+    assert "provider_specific_fields" not in assistant_message
+    assert assistant_message["content"] == REPLAYED_ASSISTANT_MESSAGE["content"]
+    assert transformed_messages[0] == PRESERVED_THINKING_MESSAGES[0]
+    assert transformed_messages[2] == PRESERVED_THINKING_MESSAGES[2]
+
+
+def test_transform_request_keeps_reasoning_content_strips_internal_fields():
+    request = TogetherAIChatConfig().transform_request(
+        model=REASONING_MODEL,
+        messages=[dict(message) for message in PRESERVED_THINKING_MESSAGES],
+        optional_params={},
+        litellm_params={"custom_llm_provider": "together_ai"},
+        headers={},
+    )
+
+    _assert_internal_fields_stripped_reasoning_kept(request["messages"])
+
+
+async def test_async_transform_request_keeps_reasoning_content_strips_internal_fields():
+    request = await TogetherAIChatConfig().async_transform_request(
+        model=REASONING_MODEL,
+        messages=[dict(message) for message in PRESERVED_THINKING_MESSAGES],
+        optional_params={},
+        litellm_params={"custom_llm_provider": "together_ai"},
+        headers={},
+    )
+
+    _assert_internal_fields_stripped_reasoning_kept(request["messages"])
+
+
+def test_completion_sends_chat_template_kwargs_and_preserved_reasoning():
+    from litellm.llms.custom_httpx.http_handler import HTTPHandler
+
+    captured_requests = []
+
+    def respond(request: httpx.Request) -> httpx.Response:
+        captured_requests.append(request)
+        return httpx.Response(
+            200,
+            json={
+                "id": "chatcmpl-together-preserved",
+                "object": "chat.completion",
+                "created": 1234567890,
+                "model": REASONING_MODEL,
+                "choices": [
+                    {
+                        "index": 0,
+                        "message": {"role": "assistant", "content": "47"},
+                        "finish_reason": "stop",
+                    }
+                ],
+                "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
+            },
+        )
+
+    client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond)))
+
+    litellm.completion(
+        model=f"together_ai/{REASONING_MODEL}",
+        messages=[dict(message) for message in PRESERVED_THINKING_MESSAGES],
+        chat_template_kwargs={"clear_thinking": False},
+        api_key="fake-key",
+        client=client,
+    )
+
+    request_body = json.loads(captured_requests[0].content)
+    assert request_body["chat_template_kwargs"] == {"clear_thinking": False}
+    assert "extra_body" not in request_body
+    _assert_internal_fields_stripped_reasoning_kept(request_body["messages"])
+
+
 def test_together_ai_config_alias_points_at_chat_config():
     assert litellm.TogetherAIConfig is litellm.TogetherAIChatConfig
     config = litellm.TogetherAIConfig(max_tokens=10)

From bb22742025065c10baba94b7d587ab000c6174ff Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 25 Aug 2026 15:54:25 -0700
Subject: [PATCH 067/105] fix(rerank): emit latency and cost headers on /rerank
 (#35419)

* fix(rerank): emit latency and cost headers on /rerank

Thread the logging object into the rerank httpx calls and pass hidden_params through to get_custom_headers, so x-litellm-overhead-duration-ms, x-litellm-response-duration-ms, x-litellm-response-cost, x-litellm-call-id and the LITELLM_DETAILED_TIMING x-litellm-timing-* headers show up on rerank like they do on chat completions

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(rerank): keep zero response cost in the /rerank cost header

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* ci: assign the new rerank endpoint tests to the proxy-endpoints shard

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test: suppress TQ008 on the rerank header tests with reasons

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: milan <milan@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: yassin <yassin@berri.ai>
---
 .github/workflows/test-unit.yml               |   1 +
 litellm/llms/bedrock/rerank/handler.py        |   3 +
 litellm/llms/custom_httpx/llm_http_handler.py |   1 +
 litellm/proxy/rerank_endpoints/endpoints.py   |   3 +
 .../test_bedrock_rerank_header_forwarding.py  |  39 +++++-
 .../custom_httpx/test_llm_http_handler.py     |  31 +++++
 .../proxy/rerank_endpoints/__init__.py        |   0
 .../proxy/rerank_endpoints/test_endpoints.py  | 120 ++++++++++++++++++
 8 files changed, 193 insertions(+), 5 deletions(-)
 create mode 100644 tests/test_litellm/proxy/rerank_endpoints/__init__.py
 create mode 100644 tests/test_litellm/proxy/rerank_endpoints/test_endpoints.py

diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml
index 2dfca3d308f..c23678c51ae 100644
--- a/.github/workflows/test-unit.yml
+++ b/.github/workflows/test-unit.yml
@@ -164,6 +164,7 @@ jobs:
               tests/test_litellm/proxy/public_endpoints
               tests/test_litellm/proxy/prompts
               tests/test_litellm/proxy/rag_endpoints
+              tests/test_litellm/proxy/rerank_endpoints
               tests/test_litellm/proxy/realtime_endpoints
               tests/test_litellm/proxy/ui_crud_endpoints
               tests/test_litellm/proxy/config_resolvers
diff --git a/litellm/llms/bedrock/rerank/handler.py b/litellm/llms/bedrock/rerank/handler.py
index 1cc72f265eb..a2a1acec80c 100644
--- a/litellm/llms/bedrock/rerank/handler.py
+++ b/litellm/llms/bedrock/rerank/handler.py
@@ -29,6 +29,7 @@ class BedrockRerankHandler(BaseAWSLLM):
     async def arerank(
         self,
         prepared_request: BedrockPreparedRequest,
+        logging_obj: LitellmLogging,
         timeout: float | httpx.Timeout | None = None,
         client: AsyncHTTPHandler | None = None,
     ):
@@ -40,6 +41,7 @@ class BedrockRerankHandler(BaseAWSLLM):
                 headers=dict(prepared_request["prepped"].headers),
                 data=prepared_request["body"],
                 timeout=timeout,
+                logging_obj=logging_obj,
             )
             response.raise_for_status()
         except httpx.HTTPStatusError as err:
@@ -98,6 +100,7 @@ class BedrockRerankHandler(BaseAWSLLM):
         if _is_async:
             return self.arerank(
                 prepared_request,
+                logging_obj=logging_obj,
                 timeout=timeout,
                 client=client if client is not None and isinstance(client, AsyncHTTPHandler) else None,
             )
diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py
index b7ddb55ae89..bb582f677be 100644
--- a/litellm/llms/custom_httpx/llm_http_handler.py
+++ b/litellm/llms/custom_httpx/llm_http_handler.py
@@ -1203,6 +1203,7 @@ class BaseLLMHTTPHandler:
                 headers=headers,
                 data=json.dumps(request_data),
                 timeout=timeout,
+                logging_obj=logging_obj,
             )
         except Exception as e:
             raise self._handle_error(e=e, provider_config=provider_config)
diff --git a/litellm/proxy/rerank_endpoints/endpoints.py b/litellm/proxy/rerank_endpoints/endpoints.py
index 45b190c1f9d..dd5803796b7 100644
--- a/litellm/proxy/rerank_endpoints/endpoints.py
+++ b/litellm/proxy/rerank_endpoints/endpoints.py
@@ -90,12 +90,15 @@ async def rerank(
         fastapi_response.headers.update(
             ProxyBaseLLMRequestProcessing.get_custom_headers(
                 user_api_key_dict=user_api_key_dict,
+                call_id=hidden_params.get("litellm_call_id", None) or data.get("litellm_call_id", None),
                 model_id=model_id,
                 cache_key=cache_key,
                 api_base=api_base,
                 version=version,
+                response_cost=hidden_params.get("response_cost", None),
                 model_region=getattr(user_api_key_dict, "allowed_model_region", ""),
                 request_data=data,
+                hidden_params=hidden_params,
                 **additional_headers,
             )
         )
diff --git a/tests/test_litellm/llms/bedrock/rerank/test_bedrock_rerank_header_forwarding.py b/tests/test_litellm/llms/bedrock/rerank/test_bedrock_rerank_header_forwarding.py
index b2a2046b131..253edad57e9 100644
--- a/tests/test_litellm/llms/bedrock/rerank/test_bedrock_rerank_header_forwarding.py
+++ b/tests/test_litellm/llms/bedrock/rerank/test_bedrock_rerank_header_forwarding.py
@@ -77,7 +77,7 @@ def test_bedrock_rerank_header_forwarding_sync(model):
 
     with (
         patch.object(client, "post") as mock_post,
-        patch(
+        patch(  # test-quality-ok: boto credential lookup needs live AWS; the HTTP boundary is already a MockTransport
             "litellm.llms.bedrock.rerank.handler.BedrockRerankHandler._get_boto_credentials_from_optional_params",
             return_value=mock_credentials_info,
         ),
@@ -170,7 +170,7 @@ async def test_bedrock_rerank_header_forwarding_async(model):
 
     with (
         patch.object(client, "post", new_callable=AsyncMock) as mock_post,
-        patch(
+        patch(  # test-quality-ok: boto credential lookup needs live AWS; the HTTP boundary is already a MockTransport
             "litellm.llms.bedrock.rerank.handler.BedrockRerankHandler._get_boto_credentials_from_optional_params",
             return_value=mock_credentials_info,
         ),
@@ -241,7 +241,7 @@ def test_bedrock_rerank_timeout_sync():
 
     with (
         patch.object(client, "post") as mock_post,
-        patch(
+        patch(  # test-quality-ok: boto credential lookup needs live AWS; the HTTP boundary is already a MockTransport
             "litellm.llms.bedrock.rerank.handler.BedrockRerankHandler._get_boto_credentials_from_optional_params",
             return_value=mock_credentials_info,
         ),
@@ -285,7 +285,7 @@ async def test_bedrock_rerank_timeout_async():
 
     with (
         patch.object(client, "post", new_callable=AsyncMock) as mock_post,
-        patch(
+        patch(  # test-quality-ok: boto credential lookup needs live AWS; the HTTP boundary is already a MockTransport
             "litellm.llms.bedrock.rerank.handler.BedrockRerankHandler._get_boto_credentials_from_optional_params",
             return_value=mock_credentials_info,
         ),
@@ -340,7 +340,7 @@ def test_bedrock_rerank_extra_headers_and_headers_merge():
 
     with (
         patch.object(client, "post") as mock_post,
-        patch(
+        patch(  # test-quality-ok: boto credential lookup needs live AWS; the HTTP boundary is already a MockTransport
             "litellm.llms.bedrock.rerank.handler.BedrockRerankHandler._get_boto_credentials_from_optional_params",
             return_value=mock_credentials_info,
         ),
@@ -400,3 +400,32 @@ def test_bedrock_rerank_extra_headers_and_headers_merge():
 
         except Exception as e:
             pytest.fail(f"Failed to merge and forward headers: {str(e)}")
+
+
+@pytest.mark.asyncio
+async def test_bedrock_rerank_records_llm_api_duration():
+    """The bedrock rerank handler must feed httpx timing into the logging obj, so the
+    proxy can emit x-litellm-overhead-duration-ms / x-litellm-timing-* on /rerank."""
+    import httpx
+
+    def handle(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(200, json=bedrock_rerank_response)
+
+    client = AsyncHTTPHandler()
+    client.client = httpx.AsyncClient(transport=httpx.MockTransport(handle))
+
+    with patch(  # test-quality-ok: boto credential lookup needs live AWS; the HTTP boundary is already a MockTransport
+        "litellm.llms.bedrock.rerank.handler.BedrockRerankHandler._get_boto_credentials_from_optional_params",
+        return_value=create_mock_credentials(),
+    ):
+        response = await litellm.arerank(
+            model="bedrock/arn:aws:bedrock:us-east-1::foundation-model/cohere.rerank-v3-5:0",
+            query=test_query,
+            documents=test_documents,
+            top_n=3,
+            client=client,
+            aws_region_name="us-east-1",
+        )
+
+    assert response._hidden_params["litellm_overhead_time_ms"] is not None
+    assert response._hidden_params["_response_ms"] >= response._hidden_params["litellm_overhead_time_ms"]
diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py
index a93b14d45f3..694a01cda5f 100644
--- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py
+++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py
@@ -2528,6 +2528,37 @@ def test_only_callbacks_that_can_charge_a_frame_are_collected_for_ws_quota(monke
     assert _collect_ws_project_quota_callbacks() == (quota,)
 
 
+@pytest.mark.asyncio
+async def test_async_rerank_records_llm_api_duration():
+    """arerank must feed the httpx timing into the logging obj, so the proxy can emit
+    x-litellm-overhead-duration-ms / x-litellm-timing-* on /rerank."""
+
+    def handle(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(
+            200,
+            json={
+                "id": "rerank-1",
+                "results": [{"index": 0, "relevance_score": 0.9}],
+                "meta": {"api_version": {"version": "2"}, "billed_units": {"search_units": 1}},
+            },
+        )
+
+    client = AsyncHTTPHandler()
+    client.client = httpx.AsyncClient(transport=httpx.MockTransport(handle))
+
+    response = await litellm.arerank(
+        model="cohere/rerank-v3.5",
+        query="what is the capital of france",
+        documents=["paris", "berlin"],
+        top_n=1,
+        api_key="fake-key",
+        client=client,
+    )
+
+    assert response._hidden_params["litellm_overhead_time_ms"] is not None
+    assert response._hidden_params["_response_ms"] >= response._hidden_params["litellm_overhead_time_ms"]
+
+
 class _JSONBodyVideoConfig(OpenAIVideoConfig):
     def use_multipart_form_data(self) -> bool:
         return False
diff --git a/tests/test_litellm/proxy/rerank_endpoints/__init__.py b/tests/test_litellm/proxy/rerank_endpoints/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/test_litellm/proxy/rerank_endpoints/test_endpoints.py b/tests/test_litellm/proxy/rerank_endpoints/test_endpoints.py
new file mode 100644
index 00000000000..9f11ff6f20d
--- /dev/null
+++ b/tests/test_litellm/proxy/rerank_endpoints/test_endpoints.py
@@ -0,0 +1,120 @@
+"""
+Tests for rerank_endpoints/endpoints.py response headers.
+"""
+
+import json
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from fastapi import Request, Response
+
+import litellm.proxy.common_request_processing as common_request_processing_mod
+import litellm.proxy.proxy_server as proxy_server_mod
+from litellm.proxy._types import UserAPIKeyAuth
+from litellm.proxy.rerank_endpoints.endpoints import rerank
+from litellm.types.utils import RerankResponse
+
+HIDDEN_PARAMS = {
+    "model_id": "deployment-1",
+    "api_base": "https://bedrock-agent-runtime.us-east-1.amazonaws.com",
+    "response_cost": 0.002,
+    "_response_ms": 1500.5,
+    "litellm_overhead_time_ms": 12.5,
+    "callback_duration_ms": 1.25,
+    "timing_llm_api_ms": 1488.0,
+    "timing_pre_processing_ms": 10.0,
+    "timing_post_processing_ms": 2.5,
+    "timing_message_copy_ms": 0.01,
+}
+
+
+def _build_request() -> Request:
+    body = json.dumps({"model": "rerank-model", "query": "q", "documents": ["a", "b"]}).encode()
+
+    async def receive():
+        return {"type": "http.request", "body": body, "more_body": False}
+
+    return Request(
+        scope={
+            "type": "http",
+            "method": "POST",
+            "path": "/rerank",
+            "headers": [(b"content-type", b"application/json")],
+            "query_string": b"",
+        },
+        receive=receive,
+    )
+
+
+async def _call_rerank(hidden_params: dict = HIDDEN_PARAMS) -> Response:
+    response = RerankResponse(id="rerank-1", results=[{"index": 0, "relevance_score": 0.9}])
+    response._hidden_params = dict(hidden_params)
+
+    fastapi_response = Response()
+    proxy_logging_obj = MagicMock()
+    proxy_logging_obj.pre_call_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["data"])
+    proxy_logging_obj.update_request_status = AsyncMock()
+
+    async def fake_add_litellm_data_to_request(**kwargs):
+        return {**kwargs["data"], "litellm_call_id": "call-123"}
+
+    async def fake_route_request(**kwargs):
+        async def _call():
+            return response
+
+        return _call()
+
+    with (
+        patch.object(proxy_server_mod, "add_litellm_data_to_request", fake_add_litellm_data_to_request),  # test-quality-ok: the rerank route reads these proxy_server module globals; no injection seam on the FastAPI handler
+        patch.object(proxy_server_mod, "route_request", fake_route_request),  # test-quality-ok: the rerank route reads these proxy_server module globals; no injection seam on the FastAPI handler
+        patch.object(proxy_server_mod, "proxy_logging_obj", proxy_logging_obj),  # test-quality-ok: the rerank route reads these proxy_server module globals; no injection seam on the FastAPI handler
+        patch.object(proxy_server_mod, "llm_router", MagicMock()),  # test-quality-ok: the rerank route reads these proxy_server module globals; no injection seam on the FastAPI handler
+        patch.object(proxy_server_mod, "version", "1.2.3"),  # test-quality-ok: the rerank route reads these proxy_server module globals; no injection seam on the FastAPI handler
+    ):
+        await rerank(
+            request=_build_request(),
+            fastapi_response=fastapi_response,
+            user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
+        )
+
+    return fastapi_response
+
+
+@pytest.mark.asyncio
+async def test_rerank_emits_latency_and_cost_headers():
+    """/rerank must surface the same hidden_params-derived headers as /chat/completions."""
+    fastapi_response = await _call_rerank()
+
+    assert fastapi_response.headers["x-litellm-call-id"] == "call-123"
+    assert fastapi_response.headers["x-litellm-response-duration-ms"] == "1500.5"
+    assert fastapi_response.headers["x-litellm-overhead-duration-ms"] == "12.5"
+    assert fastapi_response.headers["x-litellm-callback-duration-ms"] == "1.25"
+    assert fastapi_response.headers["x-litellm-response-cost"] == "0.002"
+
+
+@pytest.mark.asyncio
+async def test_rerank_emits_detailed_timing_headers_when_enabled():
+    """LITELLM_DETAILED_TIMING must also work on /rerank, not just /chat/completions."""
+    with patch.object(common_request_processing_mod, "LITELLM_DETAILED_TIMING", True):  # test-quality-ok: LITELLM_DETAILED_TIMING is a module constant; toggling it is the behavior under test
+        fastapi_response = await _call_rerank()
+
+    assert fastapi_response.headers["x-litellm-timing-llm-api-ms"] == "1488.0"
+    assert fastapi_response.headers["x-litellm-timing-pre-processing-ms"] == "10.0"
+    assert fastapi_response.headers["x-litellm-timing-post-processing-ms"] == "2.5"
+    assert fastapi_response.headers["x-litellm-timing-message-copy-ms"] == "0.01"
+
+
+@pytest.mark.asyncio
+async def test_rerank_emits_zero_response_cost_header():
+    """A free deployment costs 0.0, which is a real cost and must not be dropped."""
+    fastapi_response = await _call_rerank({**HIDDEN_PARAMS, "response_cost": 0.0})
+
+    assert fastapi_response.headers["x-litellm-response-cost"] == "0.0"
+
+
+@pytest.mark.asyncio
+async def test_rerank_omits_detailed_timing_headers_when_disabled():
+    with patch.object(common_request_processing_mod, "LITELLM_DETAILED_TIMING", False):  # test-quality-ok: LITELLM_DETAILED_TIMING is a module constant; toggling it is the behavior under test
+        fastapi_response = await _call_rerank()
+
+    assert "x-litellm-timing-llm-api-ms" not in fastapi_response.headers

From 8a61b286d1c28792c4b1574a7acda246da156c1e Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 16:03:43 -0700
Subject: [PATCH 068/105] test(together_ai): annotate preserved-thinking test
 helper and capture list

---
 .../together_ai/chat/test_together_ai_chat_transformation.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py b/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py
index 110d93cbc0d..f6d4b73f27a 100644
--- a/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py
+++ b/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py
@@ -1,5 +1,6 @@
 import json
 import logging
+from collections.abc import Mapping, Sequence
 from unittest.mock import MagicMock
 
 import httpx
@@ -283,7 +284,7 @@ PRESERVED_THINKING_MESSAGES = [
 ]
 
 
-def _assert_internal_fields_stripped_reasoning_kept(transformed_messages: list):
+def _assert_internal_fields_stripped_reasoning_kept(transformed_messages: Sequence[Mapping[str, object]]):
     assistant_message = transformed_messages[1]
     assert assistant_message["reasoning_content"] == REPLAYED_ASSISTANT_MESSAGE["reasoning_content"]
     assert "thinking_blocks" not in assistant_message
@@ -320,7 +321,7 @@ async def test_async_transform_request_keeps_reasoning_content_strips_internal_f
 def test_completion_sends_chat_template_kwargs_and_preserved_reasoning():
     from litellm.llms.custom_httpx.http_handler import HTTPHandler
 
-    captured_requests = []
+    captured_requests: list[httpx.Request] = []
 
     def respond(request: httpx.Request) -> httpx.Response:
         captured_requests.append(request)

From 6684bb343b0b10a78835cb592a980b9bd61d18ee Mon Sep 17 00:00:00 2001
From: Deepanshu Lulla <deepanshu.lulla@gmail.com>
Date: Tue, 25 Aug 2026 19:15:47 -0400
Subject: [PATCH 069/105] perf(streaming): add shared JSONFragmentAccumulator
 for Vertex and Anthropic (#36610)

* perf(streaming): add shared JSONFragmentAccumulator for Vertex and Anthropic

Vertex's handle_accumulated_json_chunk and Anthropic's
_handle_accumulated_json_chunk each independently accumulated SSE fragments
into a JSON envelope with self.accumulated_json += fragment. Because the
attribute holds a live reference, CPython copies the whole prior buffer on
every fragment, making buffer assembly O(n^2) in total payload size.
Anthropic additionally had no completeness heuristic at all and called
json.loads on the whole buffer after every fragment, and could wedge forever
on two concatenated envelopes.

Add JSONFragmentAccumulator in litellm_core_utils/: fragments append to a
list in O(1), a could_close_json heuristic lets callers skip the join+parse
entirely until a value could plausibly be complete, and pop_next_value peels
one JSON value off the front of the buffer at a time using
json.JSONDecoder().raw_decode, keeping any unconsumed remainder instead of
failing on concatenated values. Migrate both providers onto it; Anthropic's
__next__/__anext__ end-of-stream handlers now delegate to
_handle_accumulated_json_chunk(is_final=True) instead of duplicating the
parse-and-reset logic inline.

Fixes #31861.

* test(streaming): close diff-coverage gaps in JSONFragmentAccumulator migration

Codecov flagged 11 uncovered lines in the migration: the accumulated_json
setter, __next__/__anext__'s end-of-stream drain branches in both providers,
and the pop_next_value "not found" path when a buffer's newest fragment ends
in "}" but is genuinely incomplete (an inner object closed, the outer one
didn't). Add targeted tests for each.

* fix(streaming): make JSONFragmentAccumulator's completeness heuristic O(1)

could_close_json rescanned every preceding blank fragment on each call, so a
hostile upstream that sends malformed JSON (never closing) followed by many
blank keepalive fragments could drive that scan, and the join+parse it
gates, to O(n^2) total. Track the last non-blank fragment's trailing byte
incrementally in append/pop_next_value/set instead of rescanning the buffer.

Reported by automated review on PR #36610.

* fix(streaming): make JSONFragmentAccumulator.pop_next_value O(1) per call

pop_next_value previously rebuilt the full remaining string and sliced a
new remainder on every call, so draining N concatenated JSON values
already sitting in one buffer cost O(n^2) total. Replace the rebuild-and-
slice with a materialize-once cursor: pending fragments are joined into
the buffer only when new ones have arrived since the last pop, and
consumed values are dropped by advancing an offset instead of copying
the remaining string.

* test(streaming): make JSONFragmentAccumulator drain regression test CI-stable

The 80k-value drain test used an absolute ms budget that flaked on a
busier CI runner (233.5ms vs a 150ms budget calibrated on a quiet
machine). Replace it with a doubling-ratio check: draining twice as many
concatenated values should take roughly 2x as long for O(n), not
~4x for O(n^2), and that ratio holds regardless of machine speed.

* test(streaming): suppress TQ002 on the append-laziness spy test

A test-quality gate (TQ002: don't assert only that a mock was called)
landed upstream since this branch's last rebase and now flags
test_append_never_calls_raw_decode. The test verifies append() defers
all decoding to pop_next_value, which has no caller-observable proxy
other than spying on the stdlib call it must avoid making.

* test(vertex): spy on raw_decode instead of json.loads in accumulator regression tests

Post-migration to the shared JSONFragmentAccumulator, Vertex's decode
path goes through json.JSONDecoder.raw_decode, not json.loads. The two
O(n^2)/partial-fragment regression tests still patched json.loads, which
that path never calls, so both passed unconditionally regardless of
whether the underlying implementation regressed. Verified by simulating
an eager-reparse regression: both tests now fail against it and pass
against the correct implementation.

* fix(streaming): widen JSONFragmentAccumulator's whitespace skip to match str.strip()

pop_next_value's whitespace skip only matched json.decoder.WHITESPACE's
ASCII set, narrower than str.strip() (Unicode-aware) which the O(1)-cursor
rewrite replaced. A non-ASCII separator like U+00A0 between two
concatenated JSON values on one SSE line made raw_decode fail on it, and
the buffer never advanced past that byte again, permanently stranding
everything after it for the rest of the stream. Use str.isspace() to
match str.strip()'s tolerance.

---------

Co-authored-by: Deepanshu <deepanshu.lulla@alpha-sense.com>
---
 .../json_fragment_accumulator.py              |  97 +++++++
 litellm/llms/anthropic/chat/handler.py        |  61 +++--
 .../vertex_and_google_ai_studio_gemini.py     |  39 +--
 .../test_json_fragment_accumulator.py         | 236 ++++++++++++++++++
 .../chat/test_anthropic_chat_handler.py       | 137 ++++++++++
 ...test_vertex_and_google_ai_studio_gemini.py |  41 ++-
 6 files changed, 562 insertions(+), 49 deletions(-)
 create mode 100644 litellm/litellm_core_utils/json_fragment_accumulator.py
 create mode 100644 tests/test_litellm/litellm_core_utils/test_json_fragment_accumulator.py

diff --git a/litellm/litellm_core_utils/json_fragment_accumulator.py b/litellm/litellm_core_utils/json_fragment_accumulator.py
new file mode 100644
index 00000000000..81d18dd0119
--- /dev/null
+++ b/litellm/litellm_core_utils/json_fragment_accumulator.py
@@ -0,0 +1,97 @@
+import json
+from typing import Final, cast  # noqa: TID251  # raw_decode returns tuple[Any, int]; no cast-free unpack
+
+
+class JSONFragmentAccumulator:
+    """
+    Buffers a JSON value that arrives piecemeal over a stream (SSE data split
+    across TCP packets, one shard per network read, etc) without the O(n^2)
+    cost of repeated `buffer += fragment` string concatenation, and without
+    the O(n^2) cost of re-copying the unconsumed remainder on every peeled
+    value when one payload holds many concatenated JSON values.
+
+    Fragments are appended to a list in O(1). The buffer is only rebuilt into
+    a single string, and only decoded, when a caller asks for a value via
+    `pop_next_value`, and `could_close_json` lets callers skip that rebuild
+    entirely for fragments that plainly cannot close a JSON value yet. Once
+    rebuilt, consumed values are dropped by advancing a cursor rather than
+    slicing a new string, so draining N concatenated values already sitting
+    in the buffer costs O(n) total, not O(n^2).
+    """
+
+    def __init__(self) -> None:
+        self._chunks: list[str] = []  # mutable-ok: O(1) append; string concat would copy the buffer each time
+        self._buffer: str = (
+            ""  # mutable-ok: lazily materialized join of _chunks, rebuilt only when _chunks is non-empty
+        )
+        self._offset: int = 0  # mutable-ok: cursor past already-consumed values; avoids re-slicing on every pop
+        self._could_close: bool = False  # mutable-ok: cached heuristic; rescanning past fragments was itself O(n^2)
+
+    def __bool__(self) -> bool:
+        return bool(self._chunks) or self._offset < len(self._buffer)
+
+    def append(self, fragment: str) -> None:
+        self._chunks.append(fragment)  # mutable-ok: see __init__
+        stripped: Final = fragment.rstrip()
+        if stripped:
+            self._could_close = stripped[-1] in ("}", "]")  # mutable-ok: see __init__
+
+    def could_close_json(self) -> bool:
+        """
+        Whether the buffer's logical last non-whitespace byte is "}" or "]",
+        i.e. whether a JSON value could plausibly be complete. Tracked
+        incrementally in `append` rather than rescanned here, so a run of
+        blank keepalive fragments (e.g. from a malformed upstream stream)
+        can't make this, or the join+parse it gates, cost O(n^2).
+        """
+        return self._could_close
+
+    def _materialize(self) -> None:
+        if not self._chunks:
+            return
+        unconsumed: Final = self._buffer[self._offset :]
+        self._buffer = unconsumed + "".join(self._chunks)  # mutable-ok: merge pending fragments, once per append batch
+        self._offset = 0  # mutable-ok: see __init__
+        self._chunks = []  # mutable-ok: see __init__
+
+    def pop_next_value(self) -> tuple[bool, object]:
+        """
+        Attempt to decode one complete JSON value from the front of the
+        buffer. On success, advances a cursor past that value (keeping any
+        unconsumed tail, e.g. a second concatenated value, in place rather
+        than copying it) and returns (True, value). If the buffer is empty
+        or holds no complete value yet, it is left untouched and this
+        returns (False, None).
+        """
+        self._materialize()
+        length: Final = len(self._buffer)
+        start = self._offset
+        while start < length and self._buffer[start].isspace():
+            start += 1
+        if start >= length:
+            self._offset = start  # mutable-ok: see __init__
+            return False, None
+        decoder: Final = json.JSONDecoder()
+        try:
+            raw_value: Final = decoder.raw_decode(self._buffer, start)
+        except json.JSONDecodeError:
+            return False, None
+        decoded, end_index = cast("tuple[object, int]", raw_value)  # cast-ok: raw_decode returns tuple[Any, int]
+        self._offset = end_index  # mutable-ok: see __init__
+        if self._offset >= len(self._buffer):
+            self._buffer = ""  # mutable-ok: see __init__
+            self._offset = 0  # mutable-ok: see __init__
+            self._could_close = False  # mutable-ok: buffer is empty, nothing can close
+        return True, decoded
+
+    def snapshot(self) -> str:
+        self._materialize()
+        return self._buffer[self._offset :]
+
+    def set(self, value: str) -> None:
+        """Replace the buffer's contents with a single fragment."""
+        self._chunks = []  # mutable-ok: see __init__
+        self._buffer = value  # mutable-ok: see __init__
+        self._offset = 0  # mutable-ok: see __init__
+        stripped: Final = value.rstrip()
+        self._could_close = bool(stripped) and stripped[-1] in ("}", "]")  # mutable-ok: see __init__
diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py
index d9bb0d7abff..5842cc2679d 100644
--- a/litellm/llms/anthropic/chat/handler.py
+++ b/litellm/llms/anthropic/chat/handler.py
@@ -18,6 +18,7 @@ from litellm.anthropic_beta_headers_manager import (
 )
 from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
 from litellm.litellm_core_utils.core_helpers import map_finish_reason
+from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
     HTTPHandler,
@@ -654,7 +655,7 @@ class ModelResponseIterator:
 
         # For handling partial JSON chunks from fragmentation
         # See: https://github.com/BerriAI/litellm/issues/17473
-        self.accumulated_json: str = ""
+        self._json_buffer = JSONFragmentAccumulator()
         self.chunk_type: Literal["valid_json", "accumulated_json"] = "valid_json"
 
         # Track current content block type to avoid emitting tool calls for non-tool blocks
@@ -678,6 +679,14 @@ class ModelResponseIterator:
         self._current_server_tool_id: str | None = None
         self._container_id: str | None = None
 
+    @property
+    def accumulated_json(self) -> str:
+        return self._json_buffer.snapshot()
+
+    @accumulated_json.setter
+    def accumulated_json(self, value: str) -> None:
+        self._json_buffer.set(value)
+
     def check_empty_tool_call_args(self) -> bool:
         """
         Check if the tool call block so far has been an empty string
@@ -1149,31 +1158,39 @@ class ModelResponseIterator:
         container: Final = message_delta["delta"].get("container")
         return finish_reason, usage, container
 
-    def _handle_accumulated_json_chunk(self, data_str: str) -> ModelResponseStream | None:
+    def _handle_accumulated_json_chunk(self, data_str: str, is_final: bool = False) -> ModelResponseStream | None:
         """
         Handle partial JSON chunks by accumulating them until valid JSON is received.
 
         This fixes network fragmentation issues where SSE data chunks may be split
         across TCP packets. See: https://github.com/BerriAI/litellm/issues/17473
 
+        Mid-stream, defer parsing until the buffer's last byte can close a value:
+        attempting a parse after every fragment of one large object is O(n^2) and
+        holds the GIL, freezing the event loop. At end of stream (is_final) no more
+        data is coming, so drain whatever complete values remain regardless of the
+        trailing byte.
+
         Args:
             data_str: The JSON string to parse (without "data:" prefix)
+            is_final: True when called from the end-of-stream drain, where the
+                trailing-byte heuristic no longer applies
 
         Returns:
             ModelResponseStream if JSON is complete, None if still accumulating
         """
-        # Accumulate JSON data
-        self.accumulated_json += data_str
+        self._json_buffer.append(data_str)
 
-        # Try to parse the accumulated JSON
-        try:
-            data_json: Final = json.loads(self.accumulated_json)
-            self.accumulated_json = ""  # Reset after successful parsing
-            return self.chunk_parser(chunk=data_json)
-        except json.JSONDecodeError:
-            # If it's not valid JSON yet, continue to the next chunk
+        if not is_final and not self._json_buffer.could_close_json():
             return None
 
+        while True:
+            found, decoded = self._json_buffer.pop_next_value()
+            if not found:
+                return None
+            if isinstance(decoded, dict):
+                return self.chunk_parser(chunk=decoded)
+
     def _parse_sse_data(self, str_line: str) -> ModelResponseStream | None:
         """
         Parse SSE data line, handling both complete and partial JSON chunks.
@@ -1209,13 +1226,10 @@ class ModelResponseIterator:
                 chunk = self.response_iterator.__next__()
             except StopIteration:
                 # If we have accumulated JSON when stream ends, try to parse it
-                if self.accumulated_json:
-                    try:
-                        data_json = json.loads(self.accumulated_json)
-                        self.accumulated_json = ""
-                        return self.chunk_parser(chunk=data_json)
-                    except json.JSONDecodeError:
-                        pass
+                if self._json_buffer:
+                    result = self._handle_accumulated_json_chunk(data_str="", is_final=True)
+                    if result is not None:
+                        return result
                 raise StopIteration
             except ValueError as e:
                 raise RuntimeError(f"Error receiving chunk from stream: {e}")
@@ -1258,13 +1272,10 @@ class ModelResponseIterator:
                 chunk = await self.async_response_iterator.__anext__()
             except StopAsyncIteration:
                 # If we have accumulated JSON when stream ends, try to parse it
-                if self.accumulated_json:
-                    try:
-                        data_json = json.loads(self.accumulated_json)
-                        self.accumulated_json = ""
-                        return self.chunk_parser(chunk=data_json)
-                    except json.JSONDecodeError:
-                        pass
+                if self._json_buffer:
+                    result = self._handle_accumulated_json_chunk(data_str="", is_final=True)
+                    if result is not None:
+                        return result
                 raise StopAsyncIteration
             except ValueError as e:
                 raise RuntimeError(f"Error receiving chunk from stream: {e}")
diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
index d298670aa7a..d12ba24eda4 100644
--- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
+++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py
@@ -23,6 +23,7 @@ from litellm.constants import (
     DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_FLASH_LITE,
     DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO,
 )
+from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator
 from litellm.litellm_core_utils.prompt_templates.factory import (
     _encode_tool_call_id_with_signature,
 )
@@ -3087,7 +3088,7 @@ class ModelResponseIterator:
         self.streaming_response = streaming_response
         self.response = response
         self.chunk_type: Literal["valid_json", "accumulated_json"] = "valid_json"
-        self.accumulated_json = ""
+        self._json_buffer = JSONFragmentAccumulator()
         self.sent_first_chunk = False
         self.logging_obj = logging_obj
         self.response_headers = response_headers or {}
@@ -3095,6 +3096,14 @@ class ModelResponseIterator:
         self.cumulative_tool_call_index: int = 0
         self.has_seen_tool_calls: bool = False
 
+    @property
+    def accumulated_json(self) -> str:
+        return self._json_buffer.snapshot()
+
+    @accumulated_json.setter
+    def accumulated_json(self, value: str) -> None:
+        self._json_buffer.set(value)
+
     @staticmethod
     def _check_streaming_error(chunk: dict) -> None:
         """Detect embedded errors (e.g. 429 RESOURCE_EXHAUSTED) in streaming chunks and raise VertexAIError."""
@@ -3298,8 +3307,8 @@ class ModelResponseIterator:
         return self.chunk_parser(chunk=json_chunk)
 
     def handle_accumulated_json_chunk(self, chunk: str, is_final: bool = False) -> Optional["ModelResponseStream"]:
-        message: Final = litellm.CustomStreamWrapper._strip_sse_data_from_chunk(chunk) or ""
-        self.accumulated_json = (self.accumulated_json + message.replace("\n\n", "")).strip()
+        message: Final = (litellm.CustomStreamWrapper._strip_sse_data_from_chunk(chunk) or "").replace("\n\n", "")
+        self._json_buffer.append(message)
 
         # Mid-stream, defer parsing until the buffer's last byte can close a value:
         # attempting a parse after every fragment of one large object is O(n^2) and
@@ -3307,27 +3316,23 @@ class ModelResponseIterator:
         # data is coming, so drain whatever complete values remain regardless of the
         # trailing byte, otherwise a complete leading value sitting behind a truncated
         # trailing one would be silently dropped.
-        if not is_final and (not self.accumulated_json or self.accumulated_json[-1] not in "}]"):
+        if not is_final and not self._json_buffer.could_close_json():
             return None
 
         # Peel one complete JSON value from the front of the buffer and keep the
         # unconsumed tail. Running json.loads over the whole buffer would fail
         # forever once it held more than one concatenated value ("Extra data") while
         # never resetting the buffer, so the buffer grew without bound and pinned the
-        # core. raw_decode reports where the value ended, so concatenated values drain
-        # one call at a time. A leading non-dict value (never emitted by Gemini in
-        # practice) is consumed and skipped so it cannot block the dict values behind it.
-        decoder: Final = json.JSONDecoder()
-        while self.accumulated_json:
-            try:
-                raw_value = decoder.raw_decode(self.accumulated_json)
-            except json.JSONDecodeError:
+        # core. pop_next_value reports where the value ended, so concatenated values
+        # drain one call at a time. A leading non-dict value (never emitted by Gemini
+        # in practice) is consumed and skipped so it cannot block the dict values
+        # behind it.
+        while True:
+            found, decoded = self._json_buffer.pop_next_value()
+            if not found:
                 return None
-            decoded, end_index = cast("tuple[object, int]", raw_value)  # cast-ok: raw_decode -> tuple[Any,int]
-            self.accumulated_json = self.accumulated_json[end_index:].strip()
             if isinstance(decoded, dict):
                 return self.chunk_parser(chunk=decoded)
-        return None
 
     def _common_chunk_parsing_logic(self, chunk: str) -> Optional["ModelResponseStream"]:
         try:
@@ -3351,7 +3356,7 @@ class ModelResponseIterator:
         try:
             chunk: Final = self.response_iterator.__next__()
         except StopIteration:
-            if self.chunk_type == "accumulated_json" and self.accumulated_json:
+            if self.chunk_type == "accumulated_json" and self._json_buffer:
                 result: Final = self.handle_accumulated_json_chunk(chunk="", is_final=True)
                 if result is not None:
                     return result
@@ -3375,7 +3380,7 @@ class ModelResponseIterator:
         try:
             chunk: Final = await self.async_response_iterator.__anext__()
         except StopAsyncIteration:
-            if self.chunk_type == "accumulated_json" and self.accumulated_json:
+            if self.chunk_type == "accumulated_json" and self._json_buffer:
                 result: Final = self.handle_accumulated_json_chunk(chunk="", is_final=True)
                 if result is not None:
                     return result
diff --git a/tests/test_litellm/litellm_core_utils/test_json_fragment_accumulator.py b/tests/test_litellm/litellm_core_utils/test_json_fragment_accumulator.py
new file mode 100644
index 00000000000..d16230afe3a
--- /dev/null
+++ b/tests/test_litellm/litellm_core_utils/test_json_fragment_accumulator.py
@@ -0,0 +1,236 @@
+import json
+import time
+from unittest.mock import patch
+
+from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator
+
+
+def test_initial_state_is_empty():
+    accumulator = JSONFragmentAccumulator()
+    assert not accumulator
+    assert accumulator.could_close_json() is False
+    assert accumulator.snapshot() == ""
+
+
+def test_could_close_json_true_only_when_last_fragment_closes_a_value():
+    accumulator = JSONFragmentAccumulator()
+    accumulator.append('{"a": ')
+    assert accumulator.could_close_json() is False
+
+    accumulator.append("1}")
+    assert accumulator.could_close_json() is True
+
+
+def test_could_close_json_looks_past_trailing_blank_fragments():
+    """A whitespace-only or empty fragment (e.g. the flush call at end of
+    stream) must not mask a real closing byte in an earlier fragment."""
+    accumulator = JSONFragmentAccumulator()
+    accumulator.append('{"a": 1}')
+    accumulator.append("")
+    accumulator.append("   \n")
+    assert accumulator.could_close_json() is True
+
+
+def test_pop_next_value_on_empty_buffer_returns_false_without_touching_state():
+    accumulator = JSONFragmentAccumulator()
+    found, value = accumulator.pop_next_value()
+    assert found is False
+    assert value is None
+
+
+def test_pop_next_value_on_incomplete_buffer_leaves_buffer_untouched():
+    accumulator = JSONFragmentAccumulator()
+    accumulator.append('{"candidates": [{"content":')
+
+    found, value = accumulator.pop_next_value()
+
+    assert found is False
+    assert value is None
+    assert accumulator.snapshot() == '{"candidates": [{"content":'
+
+
+def test_pop_next_value_decodes_single_complete_object_and_clears_buffer():
+    accumulator = JSONFragmentAccumulator()
+    accumulator.append('{"candidates": [{"content": {"parts": [{"text": "hi"}]}}]}')
+
+    found, value = accumulator.pop_next_value()
+
+    assert found is True
+    assert value == {"candidates": [{"content": {"parts": [{"text": "hi"}]}}]}
+    assert accumulator.snapshot() == ""
+    assert not accumulator
+
+
+def test_pop_next_value_reassembles_a_value_split_across_many_fragments():
+    obj = {"candidates": [{"content": {"parts": [{"text": "x" * 5000}]}}]}
+    blob = json.dumps(obj)
+    fragments = [blob[i : i + 37] for i in range(0, len(blob), 37)]
+    assert len(fragments) > 10, "need a genuinely multi-fragment payload"
+
+    accumulator = JSONFragmentAccumulator()
+    found = False
+    value = None
+    for fragment in fragments:
+        accumulator.append(fragment)
+        if accumulator.could_close_json():
+            found, value = accumulator.pop_next_value()
+
+    assert found is True
+    assert value == obj
+
+
+def test_pop_next_value_peels_one_value_and_keeps_remainder():
+    """Two concatenated envelopes in the buffer must both surface, one per
+    call, instead of json.loads's "Extra data" failure wedging the buffer."""
+    obj = '{"a": 1}'
+    accumulator = JSONFragmentAccumulator()
+    accumulator.append(obj + obj)
+
+    first_found, first_value = accumulator.pop_next_value()
+    assert first_found is True
+    assert first_value == {"a": 1}
+    assert accumulator.snapshot() == obj, "second value must remain buffered"
+
+    second_found, second_value = accumulator.pop_next_value()
+    assert second_found is True
+    assert second_value == {"a": 1}
+    assert not accumulator
+
+
+def test_pop_next_value_skips_non_ascii_whitespace_between_concatenated_values():
+    """A separator like U+00A0 (non-breaking space) between two concatenated
+    values must not strand the second value forever. `raw_decode` only skips
+    the narrow `json.decoder.WHITESPACE` set, so the accumulator's own
+    whitespace skip must be as tolerant as `str.strip()` was before this
+    class replaced it, not merely match `raw_decode`'s narrower set."""
+    accumulator = JSONFragmentAccumulator()
+    accumulator.append('{"a": 1}' + "\xa0" + '{"a": 2}')
+
+    first_found, first_value = accumulator.pop_next_value()
+    assert first_found is True
+    assert first_value == {"a": 1}
+
+    second_found, second_value = accumulator.pop_next_value()
+    assert second_found is True, "the second value must not be permanently stranded"
+    assert second_value == {"a": 2}
+    assert not accumulator
+
+
+def test_pop_next_value_advances_past_a_non_dict_leading_value():
+    accumulator = JSONFragmentAccumulator()
+    accumulator.append("[1, 2]" + '{"a": 1}')
+
+    first_found, first_value = accumulator.pop_next_value()
+    assert first_found is True
+    assert first_value == [1, 2]
+
+    second_found, second_value = accumulator.pop_next_value()
+    assert second_found is True
+    assert second_value == {"a": 1}
+
+
+def test_set_and_snapshot_roundtrip():
+    accumulator = JSONFragmentAccumulator()
+    accumulator.set('{"a": 1}')
+    assert accumulator.snapshot() == '{"a": 1}'
+    assert accumulator
+
+    accumulator.set("")
+    assert accumulator.snapshot() == ""
+    assert not accumulator
+
+
+def test_append_never_calls_raw_decode():  # test-quality-ok: TQ002 - laziness contract has no caller-observable proxy other than spying on the stdlib call it must defer
+    """Appending must be O(1) bookkeeping only; the O(n) join+decode is
+    deferred entirely to pop_next_value."""
+    accumulator = JSONFragmentAccumulator()
+    with patch.object(json.JSONDecoder, "raw_decode", autospec=True, side_effect=json.JSONDecoder.raw_decode) as spy:
+        for fragment in ['{"a":', " 1", "}"]:
+            accumulator.append(fragment)
+        assert spy.call_count == 0
+
+
+def test_pop_next_value_calls_raw_decode_at_most_once_per_value():
+    accumulator = JSONFragmentAccumulator()
+    accumulator.append('{"a": 1}' * 3)
+
+    with patch.object(json.JSONDecoder, "raw_decode", autospec=True, side_effect=json.JSONDecoder.raw_decode) as spy:
+        for _ in range(3):
+            found, _ = accumulator.pop_next_value()
+            assert found is True
+        assert spy.call_count == 3
+
+
+def test_accumulation_of_many_fragments_is_not_quadratic():
+    """Regression guard: appending 1000 shards must stay O(n) total, not the
+    O(n^2) cost of repeated `buffer += fragment` string concatenation."""
+    accumulator = JSONFragmentAccumulator()
+    shard = "x" * 2048
+
+    start = time.perf_counter()
+    for _ in range(1000):
+        accumulator.append(shard)
+    elapsed_ms = (time.perf_counter() - start) * 1000
+
+    assert elapsed_ms < 50, f"1000-fragment append took {elapsed_ms:.1f} ms (expected < 50 ms); O(n^2) regression?"
+
+
+def test_draining_many_concatenated_values_is_not_quadratic():
+    """
+    Regression guard: peeling N JSON values already sitting in one buffer,
+    one pop_next_value() call per value with no new fragments in between,
+    must be O(n) total. Re-copying the shrinking remainder on every pop
+    (slicing a new string instead of advancing a cursor) makes total drain
+    time scale with the square of the buffer size.
+
+    Uses a doubling ratio rather than an absolute ms budget so it isn't
+    flaky on a slower or busier CI runner: doubling the input should
+    roughly double an O(n) drain's time but roughly quadruple an O(n^2)
+    drain's time, and that ratio holds regardless of machine speed.
+    """
+
+    def drain_time_ms(n: int) -> float:
+        accumulator = JSONFragmentAccumulator()
+        accumulator.append('{"a": 1}' * n)
+        start = time.perf_counter()
+        drained = 0
+        while True:
+            found, _ = accumulator.pop_next_value()
+            if not found:
+                break
+            drained += 1
+        assert drained == n
+        return (time.perf_counter() - start) * 1000
+
+    small_ms = drain_time_ms(40_000)
+    large_ms = drain_time_ms(80_000)
+
+    ratio = large_ms / max(small_ms, 0.001)
+    assert ratio < 3.0, (
+        f"doubling drained values scaled time by {ratio:.2f}x ({small_ms:.1f} ms -> {large_ms:.1f} ms); "
+        "expected roughly 2x for O(n); O(n^2) regression?"
+    )
+
+
+def test_could_close_json_after_many_blank_fragments_is_not_quadratic():
+    """
+    Regression test: a hostile upstream can send malformed JSON that never
+    closes, followed by thousands of blank keepalive fragments. Rescanning
+    every blank fragment on each could_close_json() call would make N calls
+    cost O(n^2) total; it must be O(1) regardless of how many blank
+    fragments preceded it.
+    """
+    accumulator = JSONFragmentAccumulator()
+    accumulator.append('{"a": ')  # never closes
+
+    start = time.perf_counter()
+    for _ in range(20_000):
+        accumulator.append("")
+        accumulator.could_close_json()
+    elapsed_ms = (time.perf_counter() - start) * 1000
+
+    assert accumulator.could_close_json() is False
+    assert elapsed_ms < 300, (
+        f"20000 blank-fragment could_close_json() calls took {elapsed_ms:.1f} ms "
+        "(expected < 300 ms); quadratic rescan regression?"
+    )
diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py
index f6cd6ac6734..bd750a47f63 100644
--- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py
+++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py
@@ -1008,6 +1008,143 @@ def test_multiple_partial_chunks_accumulation():
     assert result3.choices[0].delta.content == "Hello"
 
 
+def test_accumulated_json_partial_fragment_returns_none_without_parsing():
+    """
+    Regression test: before the shared JSONFragmentAccumulator, every partial
+    fragment triggered a `json.loads` attempt over the whole growing buffer,
+    unlike Vertex which already deferred parsing until the buffer could close.
+    A fragment that can't close a JSON value must not trigger a decode attempt.
+    """
+    iterator = ModelResponseIterator(
+        streaming_response=MagicMock(), sync_stream=True, json_mode=False
+    )
+    iterator.chunk_type = "accumulated_json"
+
+    with patch.object(
+        json.JSONDecoder, "raw_decode", autospec=True, side_effect=json.JSONDecoder.raw_decode
+    ) as spy:
+        result = iterator._handle_accumulated_json_chunk(
+            '{"type":"content_block_delta","index":0,"delta":'
+        )
+        assert result is None
+        assert spy.call_count == 0, "incomplete buffer should not be parsed"
+
+
+def test_accumulated_json_does_not_reparse_every_fragment():
+    """
+    Regression test for the O(n^2) json.loads-per-fragment anti-pattern: a
+    payload split across many fragments must be parsed ~once, not once per
+    fragment.
+    """
+    text = "x" * 200_000
+    blob = json.dumps(
+        {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": text}}
+    )
+    fragments = [blob[i : i + 4096] for i in range(0, len(blob), 4096)]
+    assert len(fragments) > 10, "need a multi-fragment payload to exercise the bug"
+
+    iterator = ModelResponseIterator(
+        streaming_response=MagicMock(), sync_stream=True, json_mode=False
+    )
+    iterator.chunk_type = "accumulated_json"
+
+    parsed = None
+    with patch.object(
+        json.JSONDecoder, "raw_decode", autospec=True, side_effect=json.JSONDecoder.raw_decode
+    ) as spy:
+        for fragment in fragments:
+            out = iterator._handle_accumulated_json_chunk(fragment)
+            if out is not None:
+                parsed = out
+        parse_calls = spy.call_count
+
+    assert parsed is not None, "the reassembled chunk must still parse"
+    assert parsed.choices[0].delta.content == text
+    assert parse_calls <= 2, (
+        f"raw_decode was called {parse_calls} times for {len(fragments)} fragments; "
+        "the O(n^2) per-fragment re-parse has regressed"
+    )
+
+
+def test_accumulated_json_concatenated_envelopes_do_not_wedge():
+    """
+    Regression test: Anthropic's single `json.loads(self.accumulated_json)`
+    call raised "Extra data" on two concatenated envelopes and, since the
+    buffer was never reset on that failure, returned None forever while
+    growing without bound. The shared accumulator peels one value at a time
+    and keeps the remainder, so both values surface across two calls.
+    """
+    obj = '{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"a"}}'
+    iterator = ModelResponseIterator(
+        streaming_response=MagicMock(), sync_stream=True, json_mode=False
+    )
+    iterator.chunk_type = "accumulated_json"
+
+    first = iterator._handle_accumulated_json_chunk(obj + obj)
+    assert first is not None
+    assert first.choices[0].delta.content == "a"
+
+    second = iterator._handle_accumulated_json_chunk("")
+    assert second is not None
+    assert second.choices[0].delta.content == "a"
+
+    assert iterator.accumulated_json == ""
+
+
+def test_accumulated_json_heuristic_passes_but_value_still_incomplete():
+    """
+    A buffer whose newest fragment ends in '}' can still be genuinely
+    incomplete (an inner object closed, the outer one didn't). The
+    heuristic must let the parse attempt through, and pop_next_value
+    finding nothing must propagate as None rather than raising.
+    """
+    iterator = ModelResponseIterator(
+        streaming_response=MagicMock(), sync_stream=True, json_mode=False
+    )
+    iterator.chunk_type = "accumulated_json"
+
+    result = iterator._handle_accumulated_json_chunk('{"type": {"nested": 1}')
+    assert result is None
+
+
+def test_accumulated_json_setter_and_sync_end_of_stream_drain():
+    """
+    The accumulated_json setter and __next__'s StopIteration drain branch:
+    a buffered partial JSON must still parse and return when the
+    underlying stream ends, instead of being silently dropped.
+    """
+    obj = '{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"a"}}'
+    iterator = ModelResponseIterator(
+        streaming_response=iter([]), sync_stream=True, json_mode=False
+    )
+    iterator.chunk_type = "accumulated_json"
+    iterator.accumulated_json = obj  # exercises the setter
+
+    result = iterator.__next__()
+    assert result is not None
+    assert result.choices[0].delta.content == "a"
+
+
+def test_accumulated_json_async_end_of_stream_drain():
+    """Async twin of the sync end-of-stream drain test: __anext__'s
+    StopAsyncIteration branch must also parse a buffered value."""
+    import asyncio
+
+    obj = '{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"a"}}'
+    iterator = ModelResponseIterator(
+        streaming_response=MagicMock(), sync_stream=False, json_mode=False
+    )
+    iterator.chunk_type = "accumulated_json"
+    iterator.accumulated_json = obj
+    mock_async_iterator = MagicMock()
+    mock_async_iterator.__anext__ = AsyncMock(side_effect=StopAsyncIteration)
+    iterator.async_response_iterator = mock_async_iterator
+
+    result = asyncio.run(iterator.__anext__())
+    assert result is not None
+    assert result.choices[0].delta.content == "a"
+
+
 def test_web_search_tool_result_no_extra_tool_calls():
     """
     Test that web_search_tool_result blocks don't emit tool call chunks.
diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py
index 3d882deeb52..790a32506f2 100644
--- a/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py
+++ b/tests/test_litellm/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py
@@ -3002,8 +3002,11 @@ def test_accumulated_json_does_not_reparse_every_fragment():
 
     The buffer only becomes a complete JSON object on the final fragment, so a
     correct implementation parses it ~once, not once per fragment. We assert the
-    full chunk still parses correctly AND that json.loads is not called on every
-    fragment (which is what made it quadratic).
+    full chunk still parses correctly AND that the buffer is not decoded on
+    every fragment (which is what made it quadratic). Post-migration to the
+    shared JSONFragmentAccumulator, decoding goes through
+    `json.JSONDecoder.raw_decode`, not `json.loads` (see the equivalent
+    Anthropic tests) so the spy targets that call, not `json.loads`.
     """
     from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
         ModelResponseIterator,
@@ -3024,7 +3027,9 @@ def test_accumulated_json_does_not_reparse_every_fragment():
     assert len(fragments) > 10, "need a multi-fragment payload to exercise the bug"
 
     parsed = None
-    with patch("json.loads", wraps=json.loads) as spy:
+    with patch.object(
+        json.JSONDecoder, "raw_decode", autospec=True, side_effect=json.JSONDecoder.raw_decode
+    ) as spy:
         for fragment in fragments:
             out = iterator.handle_accumulated_json_chunk(chunk=fragment)
             if out is not None:
@@ -3035,14 +3040,16 @@ def test_accumulated_json_does_not_reparse_every_fragment():
     assert parsed.choices[0].delta.content == text, "content must be preserved intact"
 
     assert parse_calls <= 2, (
-        f"json.loads was called {parse_calls} times for {len(fragments)} "
+        f"raw_decode was called {parse_calls} times for {len(fragments)} "
         "fragments; the O(n^2) per-fragment re-parse has regressed"
     )
 
 
 def test_accumulated_json_partial_fragment_returns_none_without_parsing():
-    """A fragment that cannot complete the JSON must not trigger a json.loads
-    parse of the whole growing buffer (issue #26181)."""
+    """A fragment that cannot complete the JSON must not trigger a decode
+    attempt over the whole growing buffer (issue #26181). Decoding goes
+    through `json.JSONDecoder.raw_decode` post-JSONFragmentAccumulator
+    migration, not `json.loads`."""
     from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
         ModelResponseIterator,
     )
@@ -3054,7 +3061,9 @@ def test_accumulated_json_partial_fragment_returns_none_without_parsing():
     )
     iterator.chunk_type = "accumulated_json"
 
-    with patch("json.loads", wraps=json.loads) as spy:
+    with patch.object(
+        json.JSONDecoder, "raw_decode", autospec=True, side_effect=json.JSONDecoder.raw_decode
+    ) as spy:
         result = iterator.handle_accumulated_json_chunk(
             chunk='{"candidates": [{"content": {"parts": [{"text": "partial'
         )
@@ -5552,3 +5561,21 @@ def test_accumulated_json_skips_non_dict_leading_value():
 
     assert len(out) == 1
     assert out[0].choices[0].delta.content == "a"
+
+
+def test_accumulated_json_async_end_of_stream_drains_buffered_value():
+    """Async twin of test_accumulated_json_end_of_stream_drains_all_buffered_values:
+    __anext__'s StopAsyncIteration branch must also parse a buffered value."""
+    import asyncio
+    from unittest.mock import AsyncMock, MagicMock
+
+    obj = '{"candidates":[{"content":{"parts":[{"text":"a"}]}}],"usageMetadata":{}}'
+    iterator = _accumulating_gemini_iterator()
+    iterator.accumulated_json = obj
+    mock_async_iterator = MagicMock()
+    mock_async_iterator.__anext__ = AsyncMock(side_effect=StopAsyncIteration)
+    iterator.async_response_iterator = mock_async_iterator
+
+    result = asyncio.run(iterator.__anext__())
+    assert result is not None
+    assert result.choices[0].delta.content == "a"

From d16afd2027291c64b3d617874f0909edaf2e63b9 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 16:20:17 -0700
Subject: [PATCH 070/105] fix(memory): return 404 when the memory row vanishes
 before delete

prisma-client-py delete() returns None when the row is already gone, so the
handler reported deleted: true for a row this caller never removed. Surface
the same 404 the read path uses instead
---
 litellm/proxy/memory/memory_endpoints.py      |  4 +++-
 .../proxy/memory/test_memory_endpoints.py     | 22 +++++++++++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/litellm/proxy/memory/memory_endpoints.py b/litellm/proxy/memory/memory_endpoints.py
index 193f7e09f07..98c5fdd198c 100644
--- a/litellm/proxy/memory/memory_endpoints.py
+++ b/litellm/proxy/memory/memory_endpoints.py
@@ -535,8 +535,10 @@ async def delete_memory(
     # Visibility != write authority — see the upsert handler for the rationale.
     await _assert_write_access(prisma_client, row, user_api_key_dict)
     try:
-        await _memory_table(prisma_client).delete(where={"memory_id": row.memory_id})
+        deleted: Final = await _memory_table(prisma_client).delete(where={"memory_id": row.memory_id})
     except Exception as e:
         raise _internal_error("Error deleting memory: %s", e, "Internal error deleting memory entry.")
 
+    if deleted is None:
+        raise HTTPException(status_code=404, detail=f"Memory with key '{key}' not found")
     return MemoryDeleteResponse(key=key, deleted=True)
diff --git a/tests/test_litellm/proxy/memory/test_memory_endpoints.py b/tests/test_litellm/proxy/memory/test_memory_endpoints.py
index da23e362bae..be75d980d9d 100644
--- a/tests/test_litellm/proxy/memory/test_memory_endpoints.py
+++ b/tests/test_litellm/proxy/memory/test_memory_endpoints.py
@@ -975,6 +975,28 @@ class TestMemoryEndpoints:
             resp = client.delete("/v1/memory/notes")
         assert resp.status_code == 404
 
+    def test_delete_memory_row_deleted_mid_delete_returns_404(self):
+        table = self.prisma.db.litellm_memorytable
+        table.rows.append(
+            _make_row(memory_id="m1", key="notes", user_id="user-a", team_id="team-a")
+        )
+
+        async def vanished(*_args, **_kwargs):
+            return None
+
+        original_delete = table.delete
+        table.delete = vanished
+
+        client = _make_client(_user_auth("user-a", "team-a"))
+        try:
+            with _patch_prisma(self.prisma):
+                resp = client.delete("/v1/memory/notes")
+        finally:
+            table.delete = original_delete
+
+        assert resp.status_code == 404, resp.text
+        assert resp.json()["detail"] == "Memory with key 'notes' not found"
+
     def test_visibility_filter_unscoped_for_admin_viewer(self):
         """
         proxy_admin_viewer reads with the same unscoped filter as proxy_admin;

From 75bf9f945260d5bb152d93fbd118ef47caf7d1c2 Mon Sep 17 00:00:00 2001
From: yucheng-berri <yucheng@berri.ai>
Date: Tue, 25 Aug 2026 16:22:01 -0700
Subject: [PATCH 071/105] fix(router): persist attempted_fallbacks and
 original_model_group into spend logs metadata (#38107)

---
 litellm/proxy/_types.py                       |   2 +
 .../spend_tracking/spend_tracking_utils.py    |   2 +
 litellm/router.py                             |  15 ++
 .../router_utils/fallback_event_handlers.py   |   4 +-
 .../gcs_pub_sub_body/spend_logs_payload.json  |   2 +-
 .../test_spend_management_endpoints.py        |   6 +-
 .../test_spend_tracking_utils.py              | 197 ++++++++++++++++++
 .../test_fallback_event_handlers.py           |  48 ++++-
 tests/test_litellm/test_router.py             | 165 +++++++++++++++
 9 files changed, 435 insertions(+), 6 deletions(-)

diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index 628e569e1b8..ed35691c6ec 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -3546,6 +3546,8 @@ class SpendLogsMetadata(TypedDict):
     litellm_overhead_time_ms: float | None  # LiteLLM overhead time in milliseconds
     attempted_retries: int | None  # Number of retries attempted (0 = first attempt succeeded)
     max_retries: int | None  # Max retries configured for this request
+    attempted_fallbacks: ReadOnly[int | None]  # Number of fallbacks attempted (0 = primary model group served)
+    original_model_group: ReadOnly[str | None]  # Model group requested before any fallbacks
     cost_breakdown: CostBreakdown | None  # Detailed cost breakdown (input_cost, output_cost, margin, discount, etc.)
     compression_savings: CompressionSavingsMetadata | None
     autorouter_savings: ReadOnly[float | None]  # stamped by the logging payload; None = not auto-routed
diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py
index d931d712a92..38da38ead2b 100644
--- a/litellm/proxy/spend_tracking/spend_tracking_utils.py
+++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py
@@ -131,6 +131,8 @@ def _get_spend_logs_metadata(
             litellm_overhead_time_ms=None,
             attempted_retries=None,
             max_retries=None,
+            attempted_fallbacks=None,
+            original_model_group=None,
             cost_breakdown=None,
             compression_savings=None,
             autorouter_savings=autorouter_savings,
diff --git a/litellm/router.py b/litellm/router.py
index d07effd0d90..3d6230e0428 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -141,6 +141,7 @@ from litellm.router_utils.cooldown_handlers import (
     is_advisor_orchestration_failure,
 )
 from litellm.router_utils.fallback_event_handlers import (
+    AttemptedFallbackTargets,
     _check_non_standard_fallback_format,
     get_fallback_model_group,
     run_async_fallback,
@@ -6930,6 +6931,20 @@ class Router:
         If it fails after num_retries, fall back to another model group
         """
         model_group: Final[str | None] = kwargs.get("model")
+        if not isinstance(kwargs.get("attempted_targets"), AttemptedFallbackTargets):
+            _fallback_metadata_key: Final = _get_router_metadata_variable_name(
+                function_name=getattr(kwargs.get("original_function"), "__name__", None)
+            )
+            _sibling_metadata_key: Final = (
+                "metadata" if _fallback_metadata_key == "litellm_metadata" else "litellm_metadata"
+            )
+            if isinstance(_sibling_metadata := kwargs.get(_sibling_metadata_key), dict):
+                _sibling_metadata.pop("attempted_fallbacks", None)
+                _sibling_metadata.pop("original_model_group", None)
+            if isinstance(_fallback_metadata := kwargs.get(_fallback_metadata_key), dict):
+                _fallback_metadata["attempted_fallbacks"] = 0
+                if model_group is not None:
+                    _fallback_metadata["original_model_group"] = model_group
         include_fallback_errors: Final = kwargs.get("include_fallback_errors", False) is True
         disable_fallbacks: Final[bool | None] = kwargs.pop("disable_fallbacks", False)
         fallbacks: Final[list | None] = kwargs.get("fallbacks", self.fallbacks)
diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py
index 3c9a4097321..acdc7df5bd1 100644
--- a/litellm/router_utils/fallback_event_handlers.py
+++ b/litellm/router_utils/fallback_event_handlers.py
@@ -374,11 +374,13 @@ async def run_async_fallback(
                 kwargs["model"] = mg
             elif isinstance(mg, dict):
                 kwargs.update(mg)
+            fallback_depth = fallback_depth + 1
             kwargs[metadata_variable_name] = {
+                "original_model_group": original_model_group,
                 **(kwargs.get(metadata_variable_name) or {}),
                 "model_group": kwargs.get("model", None),
+                "attempted_fallbacks": fallback_depth,
             }
-            fallback_depth = fallback_depth + 1
             kwargs["fallback_depth"] = fallback_depth
             kwargs["max_fallbacks"] = max_fallbacks
             kwargs["attempted_targets"] = attempted
diff --git a/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json b/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json
index 2a4f3acc896..b55756b94ea 100644
--- a/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json
+++ b/tests/logging_callback_tests/gcs_pub_sub_body/spend_logs_payload.json
@@ -11,7 +11,7 @@
     "user": "",
     "team_id": "",
     "organization_id": "",
-    "metadata": "{\"applied_guardrails\": [], \"batch_models\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"guardrail_information\": null, \"compression_savings\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null}",
+    "metadata": "{\"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"guardrail_information\": null, \"compression_savings\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null}",
     "cache_key": "Cache OFF",
     "spend": 0.00022500000000000002,
     "total_tokens": 30,
diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py
index 2b062d9020d..8c15ead8983 100644
--- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py
+++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py
@@ -2629,7 +2629,7 @@ class TestSpendLogsPayload:
                     "model": "gpt-4o",
                     "user": "",
                     "team_id": "",
-                    "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
+                    "metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}',
                     "cache_key": "Cache OFF",
                     "spend": 0.00022500000000000002,
                     "total_tokens": 30,
@@ -2725,7 +2725,7 @@ class TestSpendLogsPayload:
                     "model": "claude-4-sonnet-20250514",
                     "user": "",
                     "team_id": "",
-                    "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
+                    "metadata": '{"applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
                     "cache_key": "Cache OFF",
                     "spend": 0.01383,
                     "total_tokens": 2598,
@@ -2819,7 +2819,7 @@ class TestSpendLogsPayload:
                     "model": "claude-4-sonnet-20250514",
                     "user": "",
                     "team_id": "",
-                    "metadata": '{"applied_guardrails": [], "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
+                    "metadata": '{"applied_guardrails": [], "attempted_fallbacks": 0, "original_model_group": "my-anthropic-model-group", "batch_models": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "usage_object": {"completion_tokens": 503, "prompt_tokens": 2095, "total_tokens": 2598, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "model_map_information": {"model_map_key": "claude-4-sonnet-20250514", "model_map_value": {"key": "claude-4-sonnet-20250514", "max_tokens": 128000, "max_input_tokens": 200000, "max_output_tokens": 128000, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "anthropic", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": true, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": false, "supports_reasoning": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["stream", "stop", "temperature", "top_p", "max_tokens", "max_completion_tokens", "tools", "tool_choice", "extra_headers", "parallel_tool_calls", "response_format", "user", "reasoning_effort", "thinking"]}}, "additional_usage_values": {"completion_tokens_details": {"accepted_prediction_tokens": null, "audio_tokens": null, "reasoning_tokens": null, "rejected_prediction_tokens": null, "text_tokens": 503, "image_tokens": null}, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0, "text_tokens": null, "image_tokens": null}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}',
                     "cache_key": "Cache OFF",
                     "spend": 0.01383,
                     "total_tokens": 2598,
diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
index 843e1d1296f..5b9d591ea56 100644
--- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
+++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
@@ -3642,3 +3642,200 @@ def test_caller_forged_autorouter_savings_is_discarded(bucket):
     )
     metadata = json.loads(payload["metadata"])
     assert metadata["autorouter_savings"] is None
+
+
+def test_get_logging_payload_includes_fallback_info_in_spend_logs_metadata():
+    """
+    Test that fallback info (attempted_fallbacks, original_model_group) from metadata
+    is included in the spend logs metadata JSON.
+    """
+    kwargs = {
+        "model": "gpt-3.5-turbo",
+        "litellm_params": {
+            "metadata": {
+                "user_api_key": "sk-test-key",
+                "attempted_fallbacks": 2,
+                "original_model_group": "azure-gpt-fallback",
+            }
+        },
+        "standard_logging_object": StandardLoggingPayload(
+            id="test-fallback-123",
+            call_type="completion",
+            stream=False,
+            response_cost=0.001,
+            status="success",
+            total_tokens=100,
+            prompt_tokens=50,
+            completion_tokens=50,
+            startTime=1234567890.0,
+            endTime=1234567891.0,
+            completionStartTime=None,
+            model_map_information=StandardLoggingModelInformation(
+                model_map_key="gpt-3.5-turbo", model_map_value=None
+            ),
+            model="gpt-3.5-turbo",
+            model_id="model-123",
+            model_group="openai",
+            custom_llm_provider="openai",
+            api_base="https://api.openai.com",
+            metadata=StandardLoggingMetadata(
+                user_api_key_hash="test_hash",
+                user_api_key_alias=None,
+                user_api_key_team_id=None,
+                user_api_key_org_id=None,
+                user_api_key_user_id=None,
+                user_api_key_team_alias=None,
+                spend_logs_metadata=None,
+                requester_ip_address=None,
+                requester_metadata=None,
+                user_api_key_end_user_id=None,
+            ),
+            cache_hit=False,
+            cache_key=None,
+            saved_cache_cost=0.0,
+            request_tags=[],
+            end_user=None,
+            requester_ip_address=None,
+            messages=[],
+            response={},
+            error_str=None,
+            model_parameters={},
+            hidden_params=StandardLoggingHiddenParams(
+                model_id="model-123",
+                cache_key=None,
+                api_base="https://api.openai.com",
+                response_cost="0.001",
+                litellm_overhead_time_ms=None,
+                additional_headers=None,
+                batch_models=None,
+                litellm_model_name=None,
+                usage_object=None,
+            ),
+        ),
+    }
+
+    response_obj = {
+        "id": "test-response-retry",
+        "choices": [{"message": {"content": "Hello!"}}],
+        "usage": {
+            "total_tokens": 100,
+            "prompt_tokens": 50,
+            "completion_tokens": 50,
+        },
+    }
+
+    start_time = datetime.datetime.now(timezone.utc)
+    end_time = datetime.datetime.now(timezone.utc)
+
+    payload = get_logging_payload(
+        kwargs=kwargs,
+        response_obj=response_obj,
+        start_time=start_time,
+        end_time=end_time,
+    )
+
+    metadata = json.loads(payload["metadata"])
+
+    assert (
+        metadata.get("attempted_fallbacks") == 2
+    ), f"Expected attempted_fallbacks=2, got {metadata.get('attempted_fallbacks')}"
+    assert (
+        metadata.get("original_model_group") == "azure-gpt-fallback"
+    ), f"Expected original_model_group=azure-gpt-fallback, got {metadata.get('original_model_group')}"
+
+
+def test_get_logging_payload_handles_missing_fallback_info_gracefully():
+    """
+    Test that fallback fields are None when not present in metadata (backward compatibility).
+    """
+    kwargs = {
+        "model": "gpt-3.5-turbo",
+        "litellm_params": {
+            "metadata": {
+                "user_api_key": "sk-test-key",
+            }
+        },
+        "standard_logging_object": StandardLoggingPayload(
+            id="test-no-fallback-456",
+            call_type="completion",
+            stream=False,
+            response_cost=0.001,
+            status="success",
+            total_tokens=100,
+            prompt_tokens=50,
+            completion_tokens=50,
+            startTime=1234567890.0,
+            endTime=1234567891.0,
+            completionStartTime=None,
+            model_map_information=StandardLoggingModelInformation(
+                model_map_key="gpt-3.5-turbo", model_map_value=None
+            ),
+            model="gpt-3.5-turbo",
+            model_id="model-123",
+            model_group="openai",
+            custom_llm_provider="openai",
+            api_base="https://api.openai.com",
+            metadata=StandardLoggingMetadata(
+                user_api_key_hash="test_hash",
+                user_api_key_alias=None,
+                user_api_key_team_id=None,
+                user_api_key_org_id=None,
+                user_api_key_user_id=None,
+                user_api_key_team_alias=None,
+                spend_logs_metadata=None,
+                requester_ip_address=None,
+                requester_metadata=None,
+                user_api_key_end_user_id=None,
+            ),
+            cache_hit=False,
+            cache_key=None,
+            saved_cache_cost=0.0,
+            request_tags=[],
+            end_user=None,
+            requester_ip_address=None,
+            messages=[],
+            response={},
+            error_str=None,
+            model_parameters={},
+            hidden_params=StandardLoggingHiddenParams(
+                model_id="model-123",
+                cache_key=None,
+                api_base="https://api.openai.com",
+                response_cost="0.001",
+                litellm_overhead_time_ms=None,
+                additional_headers=None,
+                batch_models=None,
+                litellm_model_name=None,
+                usage_object=None,
+            ),
+        ),
+    }
+
+    response_obj = {
+        "id": "test-response-no-fallback",
+        "choices": [{"message": {"content": "Hello!"}}],
+        "usage": {
+            "total_tokens": 100,
+            "prompt_tokens": 50,
+            "completion_tokens": 50,
+        },
+    }
+
+    start_time = datetime.datetime.now(timezone.utc)
+    end_time = datetime.datetime.now(timezone.utc)
+
+    payload = get_logging_payload(
+        kwargs=kwargs,
+        response_obj=response_obj,
+        start_time=start_time,
+        end_time=end_time,
+    )
+
+    metadata = json.loads(payload["metadata"])
+
+    assert (
+        metadata.get("attempted_fallbacks") is None
+    ), "attempted_fallbacks should be None when not provided"
+    assert (
+        metadata.get("original_model_group") is None
+    ), "original_model_group should be None when not provided"
diff --git a/tests/test_litellm/router_utils/test_fallback_event_handlers.py b/tests/test_litellm/router_utils/test_fallback_event_handlers.py
index 24477248a8a..3e02838b88f 100644
--- a/tests/test_litellm/router_utils/test_fallback_event_handlers.py
+++ b/tests/test_litellm/router_utils/test_fallback_event_handlers.py
@@ -308,7 +308,11 @@ async def test_run_async_fallback_handles_explicitly_none_metadata():
         metadata=None,
     )
 
-    assert router.received_kwargs["metadata"] == {"model_group": "azure-group"}
+    assert router.received_kwargs["metadata"] == {
+        "model_group": "azure-group",
+        "attempted_fallbacks": 1,
+        "original_model_group": "openai-group",
+    }
 
 
 @pytest.mark.asyncio
@@ -843,3 +847,45 @@ class TestRunAsyncFallbackTriggersCooldown:
                 )
 
             mock_trigger.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_run_async_fallback_stamps_fallback_info_into_metadata():
+    """Spend logs are built from the request metadata of the nested call, so the
+    fallback signal has to be stamped there before recursing."""
+    router = RecordingRouter()
+
+    await run_async_fallback(
+        litellm_router=router,
+        fallback_model_group=["fallback-model"],
+        original_model_group="primary-model",
+        original_exception=RuntimeError("original failed"),
+        max_fallbacks=3,
+        fallback_depth=0,
+    )
+
+    metadata = router.received_kwargs["metadata"]
+    assert metadata["attempted_fallbacks"] == 1
+    assert metadata["original_model_group"] == "primary-model"
+    assert metadata["model_group"] == "fallback-model"
+
+
+@pytest.mark.asyncio
+async def test_run_async_fallback_preserves_original_model_group_on_nested_fallback():
+    """A second-level fallback receives the first fallback target as its
+    original_model_group argument, so the first-stamped value must survive the hop."""
+    router = RecordingRouter()
+
+    await run_async_fallback(
+        litellm_router=router,
+        fallback_model_group=["second-fallback"],
+        original_model_group="first-fallback",
+        original_exception=RuntimeError("first fallback failed"),
+        max_fallbacks=3,
+        fallback_depth=1,
+        metadata={"attempted_fallbacks": 1, "original_model_group": "primary-model"},
+    )
+
+    metadata = router.received_kwargs["metadata"]
+    assert metadata["attempted_fallbacks"] == 2
+    assert metadata["original_model_group"] == "primary-model"
diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py
index 56fd6df446f..e59b00f53ff 100644
--- a/tests/test_litellm/test_router.py
+++ b/tests/test_litellm/test_router.py
@@ -10325,3 +10325,168 @@ async def test_factory_function_anthropic_messages_uses_streaming_fallback_dispa
         result = await wrapped(model="primary")
     assert result == "ok"
     mock_anthropic.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+async def test_async_function_with_fallbacks_stamps_zero_attempted_fallbacks():
+    """A request served by the primary model group records attempted_fallbacks=0 and
+    the requested model group in metadata, mirroring the x-litellm-attempted-fallbacks header."""
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "gpt-3.5-turbo",
+                "litellm_params": {"model": "gpt-3.5-turbo", "mock_response": "hi"},
+            }
+        ]
+    )
+    metadata = {}
+
+    await router.acompletion(
+        model="gpt-3.5-turbo",
+        messages=[{"role": "user", "content": "hey"}],
+        metadata=metadata,
+    )
+
+    assert metadata["attempted_fallbacks"] == 0
+    assert metadata["original_model_group"] == "gpt-3.5-turbo"
+
+
+@pytest.mark.asyncio
+async def test_async_function_with_fallbacks_stamps_route_bucket_not_litellm_metadata():
+    """A chat completion carrying both metadata buckets gets stamped in the route's bucket
+    (metadata), matching where run_async_fallback rewrites, so the two never diverge."""
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "gpt-3.5-turbo",
+                "litellm_params": {"model": "gpt-3.5-turbo", "mock_response": "hi"},
+            }
+        ]
+    )
+    metadata = {}
+    litellm_metadata = {"client_key": "client_value"}
+
+    await router.acompletion(
+        model="gpt-3.5-turbo",
+        messages=[{"role": "user", "content": "hey"}],
+        metadata=metadata,
+        litellm_metadata=litellm_metadata,
+    )
+
+    assert metadata["attempted_fallbacks"] == 0
+    assert metadata["original_model_group"] == "gpt-3.5-turbo"
+    assert litellm_metadata["client_key"] == "client_value"
+    assert "attempted_fallbacks" not in litellm_metadata
+    assert "original_model_group" not in litellm_metadata
+
+
+@pytest.mark.asyncio
+async def test_async_function_with_fallbacks_overrides_client_supplied_stamp_values():
+    """Client-supplied attempted_fallbacks and original_model_group are replaced on entry,
+    so a reused metadata dict or a spoofed value cannot leak stale attribution into logs."""
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "gpt-3.5-turbo",
+                "litellm_params": {"model": "gpt-3.5-turbo", "mock_response": "hi"},
+            }
+        ]
+    )
+    metadata = {"attempted_fallbacks": 99, "original_model_group": "stale-group"}
+
+    await router.acompletion(
+        model="gpt-3.5-turbo",
+        messages=[{"role": "user", "content": "hey"}],
+        metadata=metadata,
+    )
+
+    assert metadata["attempted_fallbacks"] == 0
+    assert metadata["original_model_group"] == "gpt-3.5-turbo"
+
+
+@pytest.mark.asyncio
+async def test_async_function_with_fallbacks_stamps_despite_forged_reentry_params():
+    """A client injecting fallback_depth or a JSON-shaped attempted_targets via request
+    litellm params cannot skip the entry stamp; only the router's own in-process
+    AttemptedFallbackTargets instance marks a genuine re-entrant hop."""
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "gpt-3.5-turbo",
+                "litellm_params": {"model": "gpt-3.5-turbo", "mock_response": "hi"},
+            }
+        ]
+    )
+    metadata = {"attempted_fallbacks": 99, "original_model_group": "spoofed-group"}
+
+    await router.acompletion(
+        model="gpt-3.5-turbo",
+        messages=[{"role": "user", "content": "hey"}],
+        metadata=metadata,
+        fallback_depth=3,
+        attempted_targets={"keys": ["spoofed-group"]},
+    )
+
+    assert metadata["attempted_fallbacks"] == 0
+    assert metadata["original_model_group"] == "gpt-3.5-turbo"
+
+
+@pytest.mark.asyncio
+async def test_async_function_with_fallbacks_skips_stamp_on_genuine_reentrant_hop():
+    """A re-entrant hop carrying the router's own AttemptedFallbackTargets instance keeps
+    the per-hop metadata that run_async_fallback wrote instead of resetting it to zero."""
+    from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets
+
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "gpt-3.5-turbo",
+                "litellm_params": {"model": "gpt-3.5-turbo", "mock_response": "hi"},
+            }
+        ]
+    )
+    metadata = {"attempted_fallbacks": 1, "original_model_group": "prod-chat"}
+
+    await router.acompletion(
+        model="gpt-3.5-turbo",
+        messages=[{"role": "user", "content": "hey"}],
+        metadata=metadata,
+        attempted_targets=AttemptedFallbackTargets(keys=frozenset(("prod-chat",))),
+    )
+
+    assert metadata["attempted_fallbacks"] == 1
+    assert metadata["original_model_group"] == "prod-chat"
+
+
+@pytest.mark.asyncio
+async def test_async_function_with_fallbacks_scrubs_spoofed_values_from_sibling_bucket():
+    """Spend logs read a truthy litellm_metadata dict in preference to metadata, so spoofed
+    stamp keys planted in the bucket the route does not own are removed on entry instead of
+    flowing into the spend log row."""
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "gpt-3.5-turbo",
+                "litellm_params": {"model": "gpt-3.5-turbo", "mock_response": "hi"},
+            }
+        ]
+    )
+    metadata = {}
+    litellm_metadata = {
+        "attempted_fallbacks": 99,
+        "original_model_group": "spoofed-group",
+        "client_key": "client_value",
+    }
+
+    await router.acompletion(
+        model="gpt-3.5-turbo",
+        messages=[{"role": "user", "content": "hey"}],
+        metadata=metadata,
+        litellm_metadata=litellm_metadata,
+    )
+
+    assert "attempted_fallbacks" not in litellm_metadata
+    assert "original_model_group" not in litellm_metadata
+    assert litellm_metadata["client_key"] == "client_value"
+    assert metadata["attempted_fallbacks"] == 0
+    assert metadata["original_model_group"] == "gpt-3.5-turbo"

From 1bc441ca2a70aded43e14789bceab1cb946036fa Mon Sep 17 00:00:00 2001
From: mateo <mateo@berri.ai>
Date: Tue, 25 Aug 2026 23:33:47 +0000
Subject: [PATCH 072/105] fix(proxy): detect held-back provider output through
 the anthropic streaming response wrapper

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/proxy/common_request_processing.py          | 6 +++++-
 tests/test_litellm/proxy/test_budget_reservation.py | 9 +++++++--
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py
index 33d5ec31313..e49fba086fe 100644
--- a/litellm/proxy/common_request_processing.py
+++ b/litellm/proxy/common_request_processing.py
@@ -51,6 +51,9 @@ from litellm.litellm_core_utils.streaming_handler import (
 from litellm.llms.anthropic.experimental_pass_through.messages.agentic_streaming_iterator import (
     AgenticAnthropicStreamingIterator,
 )
+from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
+    AnthropicMessagesStreamingResponse,
+)
 from litellm.proxy._types import ProxyException, UserAPIKeyAuth
 from litellm.proxy.auth.auth_checks import can_key_call_resolved_model
 from litellm.proxy.auth.auth_utils import check_response_size_is_safe
@@ -207,7 +210,8 @@ _CLIENT_DISCONNECTED_ERROR_INFORMATION: Final[StandardLoggingPayloadErrorInforma
 
 
 def _withheld_provider_output(response: object) -> bool:
-    return isinstance(response, AgenticAnthropicStreamingIterator) and response.has_buffered_provider_output
+    stream: Final = response.completion_stream if isinstance(response, AnthropicMessagesStreamingResponse) else response
+    return isinstance(stream, AgenticAnthropicStreamingIterator) and stream.has_buffered_provider_output
 
 
 def _should_return_raw_model_name(request_data: dict[str, object]) -> bool:
diff --git a/tests/test_litellm/proxy/test_budget_reservation.py b/tests/test_litellm/proxy/test_budget_reservation.py
index 0444629ba34..9fa868548fc 100644
--- a/tests/test_litellm/proxy/test_budget_reservation.py
+++ b/tests/test_litellm/proxy/test_budget_reservation.py
@@ -13,6 +13,9 @@ from litellm.constants import STREAM_SSE_KEEPALIVE_PING_BYTES
 from litellm.llms.anthropic.experimental_pass_through.messages.agentic_streaming_iterator import (
     AgenticAnthropicStreamingIterator,
 )
+from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
+    AnthropicMessagesStreamingResponse,
+)
 from litellm.proxy._types import (
     LiteLLM_BudgetTable,
     LiteLLM_EndUserTable,
@@ -2528,9 +2531,11 @@ async def test_streaming_cancel_while_holding_back_provider_output_keeps_reserva
         ping_interval_seconds=0.01,
     )
 
+    wrapped = AnthropicMessagesStreamingResponse(completion_stream=held_back, hidden_params={})
+
     async def ping_then_cancel(user_api_key_dict, response, request_data):
         yield await response.__anext__()
-        while not response.has_buffered_provider_output:
+        while not held_back.has_buffered_provider_output:
             yield await response.__anext__()
         raise asyncio.CancelledError()
 
@@ -2538,7 +2543,7 @@ async def test_streaming_cancel_while_holding_back_provider_output_keeps_reserva
     streaming_logging_obj.async_post_call_streaming_iterator_hook = ping_then_cancel
     streaming_logging_obj._arelease_max_parallel_requests_on_disconnect = AsyncMock()
     generator = ProxyBaseLLMRequestProcessing.async_streaming_data_generator(
-        response=held_back,
+        response=wrapped,
         user_api_key_dict=valid_token,
         request_data=_request_body(),
         proxy_logging_obj=streaming_logging_obj,

From ba8d8b6e149c7011e556e6ae10179de5de05942e Mon Sep 17 00:00:00 2001
From: yucheng-berri <yucheng@berri.ai>
Date: Tue, 25 Aug 2026 16:38:18 -0700
Subject: [PATCH 073/105] fix(logging): redact tool call arguments to valid
 JSON and preserve null content (#38182)

* fix(logging): redact tool call arguments to valid JSON and preserve null content

Resolves LIT-6102

* refactor(logging): centralize redacted tool-call arguments constant and satisfy test-quality gate

* fix(responses): drop Final annotations on loop-assigned locals flagged by basedpyright

* fix(responses): skip custom tool calls in redacted-arguments normalizer

* fix(logging): keep the redaction sentinel in stored tool-call arguments and preserve null output text
---
 litellm/constants.py                          |  2 +
 .../integrations/langfuse/langfuse_otel.py    |  7 +-
 litellm/litellm_core_utils/redact_messages.py | 44 +++++----
 .../session_handler.py                        | 15 ++-
 .../transformation.py                         |  4 +
 .../integrations/test_langfuse_otel.py        | 46 +++++++++
 .../test_redact_messages.py                   | 95 ++++++++++++++++++-
 .../test_litellm_completion_responses.py      | 13 +++
 .../test_session_handler.py                   | 80 ++++++++++++++++
 9 files changed, 283 insertions(+), 23 deletions(-)

diff --git a/litellm/constants.py b/litellm/constants.py
index 765bbfe1e54..57a8fc2b97c 100644
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -49,6 +49,8 @@ LITELLM_MAX_STREAMING_DURATION_SECONDS: Final = (
 # Set to 0 to disable truncation.
 MAX_BASE64_LENGTH_FOR_LOGGING: Final = int(os.getenv("MAX_BASE64_LENGTH_FOR_LOGGING", 64))
 REDACTED_BY_LITELLM: Final = "redacted-by-litellm"
+# in-memory stand-in handed to provider converters for redacted arguments; never stored
+REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER: Final = "{}"
 
 MAX_STRING_LENGTH_STDOUT_LOG: Final = get_env_int("MAX_STRING_LENGTH_STDOUT_LOG", 4096)
 
diff --git a/litellm/integrations/langfuse/langfuse_otel.py b/litellm/integrations/langfuse/langfuse_otel.py
index a93c45ef840..3d044c3ea15 100644
--- a/litellm/integrations/langfuse/langfuse_otel.py
+++ b/litellm/integrations/langfuse/langfuse_otel.py
@@ -10,6 +10,7 @@ from litellm.integrations.langfuse.langfuse_otel_attributes import (
     LangfuseLLMObsOTELAttributes,
 )
 from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
+from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
 from litellm.types.integrations.langfuse_otel import (
     LangfuseSpanAttributes,
 )
@@ -197,7 +198,11 @@ class LangfuseOtelLogger(OpenTelemetry):
                         )
                     elif item_type == "function_call":
                         arguments_str = getattr(item, "arguments", "{}")
-                        arguments_obj = json.loads(arguments_str) if isinstance(arguments_str, str) else arguments_str
+                        arguments_obj = (
+                            safe_json_loads(arguments_str, default={})
+                            if isinstance(arguments_str, str)
+                            else arguments_str
+                        )
                         langfuse_tool_call = {
                             "id": getattr(item, "id", ""),
                             "name": getattr(item, "name", ""),
diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py
index be187b091b8..064228bf8d7 100644
--- a/litellm/litellm_core_utils/redact_messages.py
+++ b/litellm/litellm_core_utils/redact_messages.py
@@ -97,16 +97,18 @@ def _redact_function_call(function_call) -> None:
 def _redact_choice_content(choice):
     """Helper to redact content in a choice (message or delta)."""
     if isinstance(choice, litellm.Choices):
-        choice.message.content = REDACTED_BY_LITELLM
-        if hasattr(choice.message, "reasoning_content"):
+        if choice.message.content is not None:
+            choice.message.content = REDACTED_BY_LITELLM
+        if getattr(choice.message, "reasoning_content", None) is not None:
             choice.message.reasoning_content = REDACTED_BY_LITELLM
         if hasattr(choice.message, "thinking_blocks"):
             choice.message.thinking_blocks = None
         _redact_tool_calls(getattr(choice.message, "tool_calls", None))
         _redact_function_call(getattr(choice.message, "function_call", None))
     elif isinstance(choice, litellm.utils.StreamingChoices):
-        choice.delta.content = REDACTED_BY_LITELLM
-        if hasattr(choice.delta, "reasoning_content"):
+        if choice.delta.content is not None:
+            choice.delta.content = REDACTED_BY_LITELLM
+        if getattr(choice.delta, "reasoning_content", None) is not None:
             choice.delta.reasoning_content = REDACTED_BY_LITELLM
         if hasattr(choice.delta, "thinking_blocks"):
             choice.delta.thinking_blocks = None
@@ -117,19 +119,19 @@ def _redact_choice_content(choice):
 def _redact_responses_api_output(output_items):
     """Helper to redact ResponsesAPIResponse output items."""
     for output_item in output_items:
-        if hasattr(output_item, "text"):
+        if getattr(output_item, "text", None) is not None:
             output_item.text = REDACTED_BY_LITELLM
 
         if hasattr(output_item, "content") and isinstance(output_item.content, list):
             for content_part in output_item.content:
-                if hasattr(content_part, "text"):
+                if getattr(content_part, "text", None) is not None:
                     content_part.text = REDACTED_BY_LITELLM
 
         # Redact reasoning items in output array
         if hasattr(output_item, "type") and output_item.type == "reasoning":
             if hasattr(output_item, "summary") and isinstance(output_item.summary, list):
                 for summary_item in output_item.summary:
-                    if hasattr(summary_item, "text"):
+                    if getattr(summary_item, "text", None) is not None:
                         summary_item.text = REDACTED_BY_LITELLM
 
         if hasattr(output_item, "type") and output_item.type == "function_call" and hasattr(output_item, "arguments"):
@@ -142,17 +144,17 @@ def _redact_responses_api_output_dict(output_items, redacted_str: str):
         if not isinstance(output_item, dict):
             continue
 
-        if "text" in output_item:
+        if output_item.get("text") is not None:
             output_item["text"] = redacted_str
 
         if isinstance(output_item.get("content"), list):
             for content_item in output_item["content"]:
-                if isinstance(content_item, dict) and "text" in content_item:
+                if isinstance(content_item, dict) and content_item.get("text") is not None:
                     content_item["text"] = redacted_str
 
         if output_item.get("type") == "reasoning" and isinstance(output_item.get("summary"), list):
             for summary_item in output_item["summary"]:
-                if isinstance(summary_item, dict) and "text" in summary_item:
+                if isinstance(summary_item, dict) and summary_item.get("text") is not None:
                     summary_item["text"] = redacted_str
 
         if output_item.get("type") == "function_call" and "arguments" in output_item:
@@ -189,40 +191,42 @@ def _redact_standard_logging_object(model_call_details: dict):
             standard_logging_object["response"] = {"text": redacted_str}
 
 
-def _redact_tool_calls_dict(message: dict, redacted_str: str) -> None:
+def _redact_tool_calls_dict(message: dict) -> None:
     """Redact tool call / function_call arguments in a dict-form message or delta."""
     tool_calls: Final = message.get("tool_calls")
     if isinstance(tool_calls, list):
         for tool_call in tool_calls:
             if isinstance(tool_call, dict) and isinstance(tool_call.get("function"), dict):
-                tool_call["function"]["arguments"] = redacted_str
+                tool_call["function"]["arguments"] = REDACTED_BY_LITELLM
 
     function_call: Final = message.get("function_call")
     if isinstance(function_call, dict) and "arguments" in function_call:
-        function_call["arguments"] = redacted_str
+        function_call["arguments"] = REDACTED_BY_LITELLM
 
 
 def _redact_model_response_dict_choices(choices, redacted_str: str):
     for choice in choices:
         if isinstance(choice, dict):
             if "message" in choice and isinstance(choice["message"], dict):
-                choice["message"]["content"] = redacted_str
-                if "reasoning_content" in choice["message"]:
+                if choice["message"].get("content") is not None:
+                    choice["message"]["content"] = redacted_str
+                if choice["message"].get("reasoning_content") is not None:
                     choice["message"]["reasoning_content"] = redacted_str
                 if "thinking_blocks" in choice["message"]:
                     choice["message"]["thinking_blocks"] = None
                 if "audio" in choice["message"]:
                     choice["message"]["audio"] = None
-                _redact_tool_calls_dict(choice["message"], redacted_str)
+                _redact_tool_calls_dict(choice["message"])
             elif "delta" in choice and isinstance(choice["delta"], dict):
-                choice["delta"]["content"] = redacted_str
-                if "reasoning_content" in choice["delta"]:
+                if choice["delta"].get("content") is not None:
+                    choice["delta"]["content"] = redacted_str
+                if choice["delta"].get("reasoning_content") is not None:
                     choice["delta"]["reasoning_content"] = redacted_str
                 if "thinking_blocks" in choice["delta"]:
                     choice["delta"]["thinking_blocks"] = None
                 if "audio" in choice["delta"]:
                     choice["delta"]["audio"] = None
-                _redact_tool_calls_dict(choice["delta"], redacted_str)
+                _redact_tool_calls_dict(choice["delta"])
         else:
             _redact_choice_content(choice)
 
@@ -263,7 +267,7 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
             isinstance(result, (litellm.ModelResponse, litellm.ResponsesAPIResponse, litellm.EmbeddingResponse))
             or (isinstance(result, dict) and ("choices" in result or "output" in result))
         ):
-            return {"text": "redacted-by-litellm"}
+            return {"text": REDACTED_BY_LITELLM}
 
         _result: Final = copy.deepcopy(result)
         if isinstance(_result, litellm.ModelResponse):
diff --git a/litellm/responses/litellm_completion_transformation/session_handler.py b/litellm/responses/litellm_completion_transformation/session_handler.py
index 1566bb1bdd7..7798dfeb6bd 100644
--- a/litellm/responses/litellm_completion_transformation/session_handler.py
+++ b/litellm/responses/litellm_completion_transformation/session_handler.py
@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final, cast
 
 import litellm
 from litellm._logging import verbose_proxy_logger
+from litellm.constants import REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER
 from litellm.proxy._types import SpendLogsPayload
 from litellm.proxy.spend_tracking.cold_storage_handler import ColdStorageHandler
 from litellm.responses.utils import ResponsesAPIRequestUtils
@@ -29,6 +30,17 @@ COLD_STORAGE_HANDLER: Final = ColdStorageHandler()
 ########################################################
 
 
+def _normalize_redacted_tool_call_arguments(message: Message) -> None:
+    """Redaction stores the bare sentinel (invalid JSON) in tool-call arguments;
+    normalize replayed history to "{}" so provider converters can parse it."""
+    for tool_call in message.tool_calls or []:
+        if (function := getattr(tool_call, "function", None)) is not None and function.arguments == REDACTED_BY_LITELLM:
+            function.arguments = REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER
+    function_call: Final = message.function_call
+    if function_call is not None and function_call.arguments == REDACTED_BY_LITELLM:
+        function_call.arguments = REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER
+
+
 class ResponsesSessionHandler:
     @staticmethod
     async def get_chat_completion_message_history_for_previous_response_id(
@@ -143,7 +155,8 @@ class ResponsesSessionHandler:
             model_response: Final = ModelResponse(**_response_output)
             for choice in model_response.choices:
                 if hasattr(choice, "message"):
-                    chat_completion_message_history.append(getattr(choice, "message"))
+                    _normalize_redacted_tool_call_arguments(message := getattr(choice, "message"))
+                    chat_completion_message_history.append(message)
         return chat_completion_message_history
 
     @staticmethod
diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py
index b8d7b726a28..7beb62d2ea3 100644
--- a/litellm/responses/litellm_completion_transformation/transformation.py
+++ b/litellm/responses/litellm_completion_transformation/transformation.py
@@ -32,6 +32,7 @@ from typing_extensions import TypedDict
 
 from litellm._logging import verbose_logger
 from litellm.caching import InMemoryCache
+from litellm.constants import REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER
 from litellm.litellm_core_utils.get_supported_openai_params import (
     get_supported_openai_params,
 )
@@ -1551,6 +1552,9 @@ class LiteLLMCompletionResponsesConfig:
         # store their payload in "input" (raw string) rather than
         # "arguments" (JSON string), so normalize to arguments here.
         raw_arguments = function_call.get("arguments")
+        if raw_arguments == REDACTED_BY_LITELLM:
+            # redaction stores the bare sentinel (invalid JSON) in arguments
+            raw_arguments = REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER
         if not raw_arguments and function_call.get("type") == "custom_tool_call":
             raw_input: Final = function_call.get("input") or ""
             raw_arguments = json.dumps({"content": raw_input}) if raw_input else ""
diff --git a/tests/test_litellm/integrations/test_langfuse_otel.py b/tests/test_litellm/integrations/test_langfuse_otel.py
index 417921c166b..89607494920 100644
--- a/tests/test_litellm/integrations/test_langfuse_otel.py
+++ b/tests/test_litellm/integrations/test_langfuse_otel.py
@@ -933,6 +933,52 @@ class TestLangfuseOtelResponsesAPI:
             assert output_data[0]["arguments"]["location"] == "San Francisco"
             assert output_data[0]["arguments"]["unit"] == "celsius"
 
+    def test_responses_api_function_call_with_redacted_arguments(self):
+        """Sentinel arguments (invalid JSON) must not kill the whole observation output."""
+        from openai.types.responses import ResponseFunctionToolCall
+
+        from litellm.types.integrations.langfuse_otel import LangfuseSpanAttributes
+
+        response_obj = ResponsesAPIResponse(
+            id="response-redacted",
+            created_at=1625247700,
+            output=[
+                ResponseFunctionToolCall(
+                    id="fc-redacted",
+                    type="function_call",
+                    name="get_weather",
+                    call_id="call-redacted",
+                    arguments="redacted-by-litellm",
+                    status="completed",
+                )
+            ],
+        )
+
+        kwargs = {
+            "call_type": "responses",
+            "messages": [{"role": "user", "content": "What's the weather?"}],
+            "model": "gpt-4o",
+            "optional_params": {},
+        }
+
+        mock_span = MagicMock()
+
+        with patch(  # test-quality-ok: the span attribute sink is the observable boundary; sibling tests in this class stub the same seam
+            "litellm.integrations.arize._utils.safe_set_attribute"
+        ) as mock_safe_set_attribute:
+            LangfuseOtelLogger._set_langfuse_specific_attributes(mock_span, kwargs, response_obj)
+
+            output_calls = [
+                call
+                for call in mock_safe_set_attribute.call_args_list
+                if call.args[1] == LangfuseSpanAttributes.OBSERVATION_OUTPUT.value
+            ]
+
+            assert len(output_calls) > 0, "observation.output should still be set"
+            output_data = json.loads(output_calls[0].args[2])
+            assert output_data[0]["name"] == "get_weather"
+            assert output_data[0]["arguments"] == {}
+
 
 if __name__ == "__main__":
     pytest.main([__file__])
diff --git a/tests/test_litellm/litellm_core_utils/test_redact_messages.py b/tests/test_litellm/litellm_core_utils/test_redact_messages.py
index 8fa6d44dd8a..3be0bae4120 100644
--- a/tests/test_litellm/litellm_core_utils/test_redact_messages.py
+++ b/tests/test_litellm/litellm_core_utils/test_redact_messages.py
@@ -350,7 +350,7 @@ class TestPerformRedaction:
         redacted = perform_redaction({}, result)
 
         message = redacted["choices"][0]["message"]
-        assert message["content"] == "redacted-by-litellm"
+        assert message["content"] is None
         tool_call = message["tool_calls"][0]
         assert tool_call["function"]["arguments"] == "redacted-by-litellm"
         assert tool_call["function"]["name"] == "get_weather"
@@ -491,6 +491,76 @@ class TestPerformRedaction:
         assert redacted["output"][0]["arguments"] == "redacted-by-litellm"
         assert redacted["output"][0]["name"] == "get_weather"
 
+    def test_redacts_every_tool_call_in_multi_element_list(self):
+        result = litellm.ModelResponse(
+            id="resp-multi",
+            choices=[
+                litellm.Choices(
+                    message=litellm.Message(
+                        content=None,
+                        role="assistant",
+                        tool_calls=[
+                            {
+                                "id": "call_1",
+                                "type": "function",
+                                "function": {"name": "get_weather", "arguments": '{"city": "a"}'},
+                            },
+                            {
+                                "id": "call_2",
+                                "type": "function",
+                                "function": {"name": "get_time", "arguments": '{"tz": "b"}'},
+                            },
+                        ],
+                    )
+                )
+            ],
+            model="gpt-4o",
+        )
+
+        redacted = perform_redaction({}, result)
+
+        tool_calls = redacted.choices[0].message.tool_calls
+        assert tool_calls[0].function.arguments == "redacted-by-litellm"
+        assert tool_calls[1].function.arguments == "redacted-by-litellm"
+
+    def test_preserves_none_content_on_tool_call_only_message(self):
+        result = litellm.ModelResponse(
+            id="resp-none",
+            choices=[
+                litellm.Choices(
+                    message=litellm.Message(
+                        content=None,
+                        role="assistant",
+                        tool_calls=[
+                            {
+                                "id": "call_1",
+                                "type": "function",
+                                "function": {"name": "get_weather", "arguments": '{"city": "a"}'},
+                            }
+                        ],
+                    )
+                )
+            ],
+            model="gpt-4o",
+        )
+
+        redacted = perform_redaction({}, result)
+
+        assert redacted.choices[0].message.content is None
+
+    def test_redacts_responses_api_function_call_arguments_object(self):
+        output_item = SimpleNamespace(
+            type="function_call",
+            name="get_weather",
+            arguments='{"city": "sensitive-city"}',
+            call_id="call_1",
+        )
+
+        _redact_responses_api_output([output_item])
+
+        assert output_item.arguments == "redacted-by-litellm"
+        assert output_item.name == "get_weather"
+
     def test_redacts_response_output_objects_with_top_level_text(self):
         output_items = [
             SimpleNamespace(text="top-level output"),
@@ -502,6 +572,29 @@ class TestPerformRedaction:
         assert output_items[0].text == "redacted-by-litellm"
         assert output_items[1] == "non-dict output item"
 
+    def test_preserves_none_text_in_responses_output(self):
+        from litellm.litellm_core_utils.redact_messages import _redact_responses_api_output_dict
+
+        none_item = SimpleNamespace(type="output_text", text=None, content=[SimpleNamespace(text=None)])
+        real_item = SimpleNamespace(type="output_text", text="real answer", content=[SimpleNamespace(text="real part")])
+
+        _redact_responses_api_output([none_item, real_item])
+
+        assert none_item.text is None
+        assert none_item.content[0].text is None
+        assert real_item.text == "redacted-by-litellm"
+        assert real_item.content[0].text == "redacted-by-litellm"
+
+        none_dict = {"type": "output_text", "text": None, "content": [{"text": None}]}
+        real_dict = {"type": "output_text", "text": "real answer", "content": [{"text": "real part"}]}
+
+        _redact_responses_api_output_dict([none_dict, real_dict], "redacted-by-litellm")
+
+        assert none_dict["text"] is None
+        assert none_dict["content"][0]["text"] is None
+        assert real_dict["text"] == "redacted-by-litellm"
+        assert real_dict["content"][0]["text"] == "redacted-by-litellm"
+
     def test_skips_non_dict_response_output_items(self):
         result = {
             "output": [
diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py
index 2273f23b1cc..b96d2eb5322 100644
--- a/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py
+++ b/tests/test_litellm/responses/litellm_completion_transformation/test_litellm_completion_responses.py
@@ -953,6 +953,19 @@ class TestFunctionCallTransformation:
         assert function.get("name") == "get_weather"
         assert function.get("arguments") == '{"location": "São Paulo, Brazil"}'
 
+    def test_function_call_transformation_normalizes_redacted_arguments(self):
+        """Redacted rows hold the bare sentinel in arguments, which is invalid JSON."""
+        result = LiteLLMCompletionResponsesConfig._transform_responses_api_function_call_to_chat_completion_message(
+            function_call={
+                "type": "function_call",
+                "name": "get_weather",
+                "arguments": "redacted-by-litellm",
+                "call_id": "call_123",
+            }
+        )
+
+        assert result[0]["tool_calls"][0]["function"]["arguments"] == "{}"
+
     def test_complete_input_transformation_with_function_calls(self):
         """Test the complete transformation with the exact input from the issue"""
         test_input = [
diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_session_handler.py b/tests/test_litellm/responses/litellm_completion_transformation/test_session_handler.py
index df477f6d01e..901fa8f57ff 100644
--- a/tests/test_litellm/responses/litellm_completion_transformation/test_session_handler.py
+++ b/tests/test_litellm/responses/litellm_completion_transformation/test_session_handler.py
@@ -9,8 +9,10 @@ import litellm
 from litellm.responses.litellm_completion_transformation import session_handler
 from litellm.responses.litellm_completion_transformation.session_handler import (
     ResponsesSessionHandler,
+    _normalize_redacted_tool_call_arguments,
 )
 from litellm.responses.utils import ResponsesAPIRequestUtils
+from litellm.types.utils import Message
 
 
 @pytest.mark.asyncio
@@ -638,3 +640,81 @@ async def test_session_lookup_does_not_retry_when_spend_logs_are_disabled(
 
     assert spend_logs == []
     assert fake_prisma_client.db.calls == [("chatcmpl-does-not-exist",)]
+
+
+def test_normalize_redacted_arguments_skips_custom_tool_calls():
+    """Custom tool calls have no .function; the normalizer must skip them, not crash (session replay path)."""
+    message = Message(
+        content=None,
+        tool_calls=[
+            {"id": "call_c", "type": "custom", "custom": {"name": "run_code", "input": "print(1)"}},
+            {"id": "call_f", "type": "function", "function": {"name": "get_weather", "arguments": "redacted-by-litellm"}},
+        ],
+    )
+
+    _normalize_redacted_tool_call_arguments(message)
+
+    assert message.tool_calls[0].custom.input == "print(1)"
+    assert message.tool_calls[1].function.arguments == "{}"
+
+
+@pytest.mark.asyncio
+async def test_message_history_normalizes_redacted_tool_call_arguments():
+    """Sessions stored with turn_off_message_logging hold the bare sentinel
+    in tool-call arguments; replay must normalize it to valid JSON."""
+    mock_spend_logs = [
+        {
+            "request_id": "chatcmpl-redacted-1",
+            "call_type": "aresponses",
+            "session_id": "sess-redacted",
+            "proxy_server_request": {
+                "input": "what is the weather in sf",
+                "model": "gpt-4o",
+            },
+            "response": {
+                "id": "chatcmpl-redacted-1",
+                "model": "gpt-4o",
+                "object": "chat.completion",
+                "choices": [
+                    {
+                        "index": 0,
+                        "message": {
+                            "role": "assistant",
+                            "content": None,
+                            "tool_calls": [
+                                {
+                                    "id": "call_1",
+                                    "type": "function",
+                                    "function": {
+                                        "name": "get_weather",
+                                        "arguments": "redacted-by-litellm",
+                                    },
+                                }
+                            ],
+                            "function_call": None,
+                        },
+                        "finish_reason": "tool_calls",
+                    }
+                ],
+                "created": 1748575031,
+                "usage": {"total_tokens": 10, "prompt_tokens": 5, "completion_tokens": 5},
+            },
+            "status": "success",
+        }
+    ]
+
+    with patch.object(  # test-quality-ok: the handler has no DI seam for the spend-log fetch; every test in this file stubs this same boundary
+        ResponsesSessionHandler,
+        "get_all_spend_logs_for_previous_response_id",
+        new_callable=AsyncMock,
+    ) as mock_get_spend_logs:
+        mock_get_spend_logs.return_value = mock_spend_logs
+
+        result = await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id(
+            "chatcmpl-redacted-1"
+        )
+
+    assistant_message = result["messages"][-1]
+    tool_call = assistant_message.tool_calls[0]
+    assert tool_call.function.arguments == "{}"
+    assert json.loads(tool_call.function.arguments) == {}

From 1b3739763354fb527e053c1585b3d85de61ee7c4 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 16:39:14 -0700
Subject: [PATCH 074/105] fix(router): forward the hold-back keepalive ping
 live and carry the withheld-output flag through the stream wrappers

The router's pre-content ping filter dropped AgenticAnthropicStreamingIterator's
hold-back keepalive, so a held-back turn sent the client nothing until the buffer
settled. A ping that no lifecycle frame precedes is now forwarded live, since a
fallback's message_start can still follow it without overlapping lifecycles

The proxy's cancel-refund guard checked isinstance against the iterator, but the
proxy only ever sees it behind FallbackAwareAnthropicMessagesStream and
AnthropicMessagesStreamingResponse, so a disconnect during hold-back refunded the
budget reservation anyway. Both wrappers now forward a duck-typed
has_buffered_provider_output flag, and the router wrapper follows a fallback
source so the flag tracks the stream actually being consumed
---
 .../messages/streaming_iterator.py            |  4 +
 litellm/proxy/common_request_processing.py    |  5 +-
 litellm/router.py                             | 29 +++++-
 .../messages/test_streaming_iterator.py       | 28 ++++++
 .../proxy/test_budget_reservation.py          | 19 +++-
 tests/test_litellm/test_router.py             | 95 +++++++++++++++++--
 6 files changed, 167 insertions(+), 13 deletions(-)

diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py
index 0a12bc3135f..8387dd8310d 100644
--- a/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py
+++ b/litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py
@@ -312,6 +312,10 @@ class AnthropicMessagesStreamingResponse:
         self.completion_stream = completion_stream
         self._hidden_params = hidden_params
 
+    @property
+    def has_buffered_provider_output(self) -> bool:
+        return getattr(self.completion_stream, "has_buffered_provider_output", False) is True
+
     def __aiter__(self) -> "AnthropicMessagesStreamingResponse":
         return self
 
diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py
index 33d5ec31313..4f56e4eb715 100644
--- a/litellm/proxy/common_request_processing.py
+++ b/litellm/proxy/common_request_processing.py
@@ -48,9 +48,6 @@ from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.litellm_core_utils.streaming_handler import (
     backfill_missing_cache_usage_fields,
 )
-from litellm.llms.anthropic.experimental_pass_through.messages.agentic_streaming_iterator import (
-    AgenticAnthropicStreamingIterator,
-)
 from litellm.proxy._types import ProxyException, UserAPIKeyAuth
 from litellm.proxy.auth.auth_checks import can_key_call_resolved_model
 from litellm.proxy.auth.auth_utils import check_response_size_is_safe
@@ -207,7 +204,7 @@ _CLIENT_DISCONNECTED_ERROR_INFORMATION: Final[StandardLoggingPayloadErrorInforma
 
 
 def _withheld_provider_output(response: object) -> bool:
-    return isinstance(response, AgenticAnthropicStreamingIterator) and response.has_buffered_provider_output
+    return getattr(response, "has_buffered_provider_output", False) is True
 
 
 def _should_return_raw_model_name(request_data: dict[str, object]) -> bool:
diff --git a/litellm/router.py b/litellm/router.py
index d07effd0d90..4cceecf50d6 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -383,6 +383,17 @@ def _anthropic_stream_should_drop_pre_content_ping(chunk: object, has_generated_
     return is_anthropic_ping_chunk(chunk)
 
 
+def _anthropic_stream_forwards_ping_live(chunk: object, has_generated_content: bool, buffered_chunk_count: int) -> bool:
+    """A `ping` that no lifecycle frame precedes reaches the client live: a fallback's own message_start can still
+    follow it without overlapping lifecycles, and AgenticAnthropicStreamingIterator's hold-back keepalive is exactly
+    such a ping."""
+    from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import is_anthropic_ping_chunk
+
+    if has_generated_content or buffered_chunk_count:
+        return False
+    return is_anthropic_ping_chunk(chunk)
+
+
 def _is_retriable_anthropic_status(status_code: int) -> bool:
     return status_code == 429 or status_code >= 500
 
@@ -426,10 +437,18 @@ class FallbackAwareAnthropicMessagesStream:
 
     def __init__(self, async_generator: AsyncGenerator[bytes, None], source_iterator: object) -> None:
         self._async_generator = async_generator
+        self._source_iterator = source_iterator
         self._hidden_params = dict(  # mutable-ok: mutated in place by merge_fallback_hidden_params
             getattr(source_iterator, "_hidden_params", None) or {}
         )
 
+    @property
+    def has_buffered_provider_output(self) -> bool:
+        return getattr(self._source_iterator, "has_buffered_provider_output", False) is True
+
+    def adopt_fallback_source(self, fallback_response: object) -> None:
+        self._source_iterator = fallback_response
+
     def __aiter__(self) -> "FallbackAwareAnthropicMessagesStream":
         return self
 
@@ -4973,7 +4992,9 @@ class Router:
             # arrives - at that point the primary attempt has committed and a
             # clean retry is no longer possible anyway - or once the primary
             # stream ends without ever producing content. A `ping` keepalive
-            # is dropped outright rather than buffered, since it can recur
+            # that nothing precedes is forwarded live (it is how a hold-back
+            # turn keeps its connection alive); one behind buffered frames is
+            # dropped outright rather than buffered, since it can recur
             # indefinitely on a slow-starting connection and carries nothing
             # worth preserving; hitting MAX_BUFFERED_PRE_CONTENT_ANTHROPIC_CHUNKS
             # forces the same early commit as real content arriving, so a
@@ -4983,6 +5004,11 @@ class Router:
             model: Final = cast(str, initial_kwargs.get("model"))  # cast-ok: kwargs always carries the model group
             try:
                 async for chunk in source_iterator:
+                    if _anthropic_stream_forwards_ping_live(
+                        chunk, has_generated_content, len(buffered_lifecycle_chunks)
+                    ):
+                        yield chunk
+                        continue
                     if _anthropic_stream_should_drop_pre_content_ping(chunk, has_generated_content):
                         continue
                     if _anthropic_stream_commits_now(chunk, has_generated_content, len(buffered_lifecycle_chunks)):
@@ -5086,6 +5112,7 @@ class Router:
             )
             fallback_hidden_params, fallback_headers = Router._prepare_fallback_hidden_params(fallback_response)
             wrapper.merge_fallback_hidden_params(fallback_hidden_params, fallback_headers)
+            wrapper.adopt_fallback_source(fallback_response)
             if hasattr(fallback_response, "__aiter__"):
                 async for fallback_item in fallback_response:
                     yield fallback_item
diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py
index 652c1f077a9..cb31280c2d5 100644
--- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py
+++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_streaming_iterator.py
@@ -8,6 +8,8 @@ import pytest
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
     INCOMPLETE_STREAM_ERROR_MESSAGE,
+    AnthropicMessagesStreamHiddenParams,
+    AnthropicMessagesStreamingResponse,
     BaseAnthropicMessagesStreamingIterator,
     _incomplete_stream_error_sse_event,
     _is_message_stop_chunk,
@@ -511,6 +513,32 @@ def test_anthropic_messages_response_as_sse_events_multiple_blocks_are_indexed()
     assert [d["delta"]["text"] for d in deltas] == ["a", "b"]
 
 
+def test_anthropic_messages_streaming_response_reports_withheld_output_of_its_stream():
+    class _HoldingBack:
+        has_buffered_provider_output = True
+
+        def __aiter__(self):
+            return self
+
+        async def __anext__(self) -> bytes:
+            raise StopAsyncIteration
+
+    async def _bare_stream():
+        yield b""
+
+    hidden_params: AnthropicMessagesStreamHiddenParams = {"additional_headers": {}}
+    assert (
+        AnthropicMessagesStreamingResponse(completion_stream=_HoldingBack(), hidden_params=hidden_params)
+        .has_buffered_provider_output
+        is True
+    )
+    assert (
+        AnthropicMessagesStreamingResponse(completion_stream=_bare_stream(), hidden_params=hidden_params)
+        .has_buffered_provider_output
+        is False
+    )
+
+
 def test_anthropic_messages_response_as_sse_events_no_content_blocks():
     response = {"id": "msg_4", "content": [], "stop_reason": "end_turn"}
     decoded = _decode_sse_events(anthropic_messages_response_as_sse_events(response))
diff --git a/tests/test_litellm/proxy/test_budget_reservation.py b/tests/test_litellm/proxy/test_budget_reservation.py
index 0444629ba34..38a346e7fb7 100644
--- a/tests/test_litellm/proxy/test_budget_reservation.py
+++ b/tests/test_litellm/proxy/test_budget_reservation.py
@@ -10,6 +10,9 @@ from fastapi import HTTPException
 import litellm
 from litellm.caching.dual_cache import DualCache
 from litellm.constants import STREAM_SSE_KEEPALIVE_PING_BYTES
+from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
+    AnthropicMessagesStreamingResponse,
+)
 from litellm.llms.anthropic.experimental_pass_through.messages.agentic_streaming_iterator import (
     AgenticAnthropicStreamingIterator,
 )
@@ -2527,6 +2530,18 @@ async def test_streaming_cancel_while_holding_back_provider_output_keeps_reserva
         server_fulfilled_tool_names=frozenset({"headroom_retrieve"}),
         ping_interval_seconds=0.01,
     )
+    router = Router(
+        model_list=[
+            {
+                "model_name": "claude-haiku-4-5",
+                "litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "sk-test"},
+            }
+        ]
+    )
+    response = await router._aanthropic_messages_streaming_iterator(
+        response=AnthropicMessagesStreamingResponse(completion_stream=held_back, hidden_params={"additional_headers": {}}),
+        initial_kwargs={"model": "claude-haiku-4-5"},
+    )
 
     async def ping_then_cancel(user_api_key_dict, response, request_data):
         yield await response.__anext__()
@@ -2538,7 +2553,7 @@ async def test_streaming_cancel_while_holding_back_provider_output_keeps_reserva
     streaming_logging_obj.async_post_call_streaming_iterator_hook = ping_then_cancel
     streaming_logging_obj._arelease_max_parallel_requests_on_disconnect = AsyncMock()
     generator = ProxyBaseLLMRequestProcessing.async_streaming_data_generator(
-        response=held_back,
+        response=response,
         user_api_key_dict=valid_token,
         request_data=_request_body(),
         proxy_logging_obj=streaming_logging_obj,
@@ -2553,7 +2568,7 @@ async def test_streaming_cancel_while_holding_back_provider_output_keeps_reserva
             received.append(chunk)
 
     with pytest.raises(asyncio.CancelledError):
-        await _drain()
+        await asyncio.wait_for(_drain(), timeout=5)
 
     assert received and received == [STREAM_SSE_KEEPALIVE_PING_BYTES] * len(received)
     assert counter_cache.in_memory_cache.get_cache(
diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py
index 56fd6df446f..4b9b91caf2c 100644
--- a/tests/test_litellm/test_router.py
+++ b/tests/test_litellm/test_router.py
@@ -20,6 +20,7 @@ from litellm.router import (
     FallbackAwareAnthropicMessagesStream,
     _anthropic_stream_commits_now,
     _anthropic_stream_should_decline_fallback,
+    _anthropic_stream_forwards_ping_live,
     _anthropic_stream_should_drop_pre_content_ping,
     _is_retriable_anthropic_status,
 )
@@ -9469,21 +9470,79 @@ async def test_anthropic_messages_content_coalesced_with_error_in_one_physical_c
 
 
 @pytest.mark.asyncio
-async def test_anthropic_messages_ping_keepalive_never_buffered_or_forwarded():
-    """Bugbot regression: a `ping` keepalive carries no content and must be
-    dropped outright before any real content arrives, rather than buffered -
+async def test_anthropic_messages_ping_behind_buffered_lifecycle_frame_is_dropped():
+    """Bugbot regression: a `ping` keepalive behind buffered lifecycle frames
+    carries no content and is dropped outright rather than buffered -
     otherwise a slow-starting connection sending many pings could grow the
     pre-content buffer without bound."""
     router = _anthropic_messages_make_router()
     source = _AnthropicMessagesFakeByteStream(
-        [_anthropic_messages_ping_chunk(), _anthropic_messages_content_chunk("hi")]
+        [
+            _anthropic_messages_message_start_chunk(),
+            _anthropic_messages_ping_chunk(),
+            _anthropic_messages_content_chunk("hi"),
+        ]
     )
 
     wrapped = await router._aanthropic_messages_streaming_iterator(response=source, initial_kwargs={"model": "primary"})
     collected = [chunk async for chunk in wrapped]
 
-    assert _anthropic_messages_ping_chunk() not in collected
-    assert collected == [_anthropic_messages_content_chunk("hi")]
+    assert collected == [_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("hi")]
+
+
+@pytest.mark.asyncio
+async def test_anthropic_messages_leading_ping_keepalive_is_forwarded_live():
+    """A `ping` that no lifecycle frame precedes is how a hold-back turn keeps
+    its connection alive (AgenticAnthropicStreamingIterator), so it must reach
+    the client at once rather than wait behind the pre-content buffer."""
+    router = _anthropic_messages_make_router()
+    content_released = asyncio.Event()
+
+    async def source():
+        yield _anthropic_messages_ping_chunk()
+        await content_released.wait()
+        yield _anthropic_messages_message_start_chunk()
+        yield _anthropic_messages_content_chunk("hi")
+
+    wrapped = await router._aanthropic_messages_streaming_iterator(response=source(), initial_kwargs={"model": "primary"})
+
+    assert await asyncio.wait_for(wrapped.__anext__(), timeout=1) == _anthropic_messages_ping_chunk()
+    content_released.set()
+    assert [chunk async for chunk in wrapped] == [
+        _anthropic_messages_message_start_chunk(),
+        _anthropic_messages_content_chunk("hi"),
+    ]
+
+
+@pytest.mark.asyncio
+async def test_anthropic_messages_leading_ping_does_not_disqualify_fallback():
+    """A live-forwarded leading `ping` commits nothing: a retriable error after
+    it still falls back, and the fallback's own lifecycle follows the ping cleanly."""
+    router = _anthropic_messages_make_router()
+    source = _AnthropicMessagesFakeByteStream(
+        [_anthropic_messages_ping_chunk(), _anthropic_messages_overloaded_error_chunk()]
+    )
+    fallback_message_start = b'event: message_start\ndata: {"type": "message_start", "message": {"id": "msg_2"}}\n\n'
+    fallback_stream = _AnthropicMessagesFallbackByteStream(
+        [fallback_message_start, _anthropic_messages_content_chunk("fallback answer")]
+    )
+
+    with patch.object(
+        router,
+        "async_function_with_fallbacks_common_utils",
+        new=AsyncMock(return_value=fallback_stream),
+    ):
+        wrapped = await router._aanthropic_messages_streaming_iterator(
+            response=source,
+            initial_kwargs={"model": "primary"},
+        )
+        collected = [chunk async for chunk in wrapped]
+
+    assert collected == [
+        _anthropic_messages_ping_chunk(),
+        fallback_message_start,
+        _anthropic_messages_content_chunk("fallback answer"),
+    ]
 
 
 @pytest.mark.asyncio
@@ -9815,6 +9874,30 @@ def test_anthropic_stream_should_drop_pre_content_ping_direct_call():
     assert _anthropic_stream_should_drop_pre_content_ping(content, has_generated_content=False) is False
 
 
+def test_anthropic_stream_forwards_ping_live_direct_call():
+    ping = _anthropic_messages_ping_chunk()
+    content = _anthropic_messages_content_chunk("hi")
+    assert _anthropic_stream_forwards_ping_live(ping, has_generated_content=False, buffered_chunk_count=0) is True
+    assert _anthropic_stream_forwards_ping_live(ping, has_generated_content=False, buffered_chunk_count=1) is False
+    assert _anthropic_stream_forwards_ping_live(ping, has_generated_content=True, buffered_chunk_count=0) is False
+    assert _anthropic_stream_forwards_ping_live(content, has_generated_content=False, buffered_chunk_count=0) is False
+
+
+def test_fallback_aware_stream_reports_withheld_output_of_its_current_source():
+    """The proxy's cancel-refund guard reads this flag off the router wrapper, so it
+    must reflect the stream actually being drained: the primary, then the fallback."""
+
+    class _HoldingBack:
+        _hidden_params = {"additional_headers": {}}
+        has_buffered_provider_output = True
+
+    wrapper = FallbackAwareAnthropicMessagesStream(_anthropic_messages_empty_generator(), _HoldingBack())
+    assert wrapper.has_buffered_provider_output is True
+
+    wrapper.adopt_fallback_source(_AnthropicMessagesFakeByteStream([]))
+    assert wrapper.has_buffered_provider_output is False
+
+
 def test_is_retriable_anthropic_status_direct_call():
     assert _is_retriable_anthropic_status(429) is True
     assert _is_retriable_anthropic_status(503) is True

From 7965cfdd2bc37861b049b35154bb823f75ae13f7 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 16:51:03 -0700
Subject: [PATCH 075/105] fix(proxy): keep listing a keyless user's own models
 when their user row is missing

Keys minted by /key/generate get no LiteLLM_UserTable row, so /v2/model/info?user_models_only=true for such a user hit the new None guard and returned 400 where the merge base returned the user's own models. Skip the team-model merge for a missing row instead of raising, since a user with no row belongs to no team
---
 litellm/proxy/proxy_server.py                 | 12 +++----
 tests/test_litellm/proxy/test_proxy_server.py | 35 +++++++++----------
 2 files changed, 22 insertions(+), 25 deletions(-)

diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index c1bf7e676e5..9ae53bebf0a 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -12227,14 +12227,12 @@ async def non_admin_all_models(
         except Exception:
             raise HTTPException(status_code=400, detail={"error": "User not found"})
 
-        if user_row is None:
-            raise HTTPException(status_code=400, detail={"error": "User not found"})
-
         # Get all models that are team models, when model team_id == user_row.teams
-        all_models += _check_if_model_is_team_model(
-            models=llm_router.get_model_list() or [],
-            user_row=user_row,
-        )
+        if user_row is not None:
+            all_models += _check_if_model_is_team_model(
+                models=llm_router.get_model_list() or [],
+                user_row=user_row,
+            )
 
     # de-duplicate models. Only return unique model ids
     unique_models: Final = _deduplicate_litellm_router_models(models=all_models)
diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py
index 9525c19f984..afc42e8db45 100644
--- a/tests/test_litellm/proxy/test_proxy_server.py
+++ b/tests/test_litellm/proxy/test_proxy_server.py
@@ -1861,36 +1861,35 @@ def test_add_team_models_to_all_models_excludes_other_teams_byok_with_shared_nam
 
 
 @pytest.mark.asyncio
-async def test_non_admin_all_models_raises_400_when_user_row_missing():
+async def test_non_admin_all_models_returns_user_models_when_user_row_missing():
     """
-    Regression test: a key whose user row no longer exists made find_unique return
-    None, and _check_if_model_is_team_model then dereferenced it
-    (`model_team_id in user_row.teams`) and raised AttributeError, surfacing as a
-    500. The miss must reuse the 400 "User not found" contract the neighbouring
-    except-branch already raises.
+    Regression test: /key/generate mints keys without a LiteLLM_UserTable row, so
+    find_unique returns None for such a user. That miss must neither raise (a 400
+    here, or the AttributeError on `user_row.teams` that used to surface as a 500)
+    nor leak team models: the user belongs to no team, so only the models they
+    added themselves come back.
     """
-    from fastapi import HTTPException
-
     from litellm.proxy.proxy_server import non_admin_all_models
 
+    user_added_model = {"model_name": "my-model", "model_info": {"id": "user-model-1"}}
     prisma_client = MagicMock()
     prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)
+    prisma_client.db.litellm_proxymodeltable.find_unique = AsyncMock(return_value=MagicMock(created_by="ghost-user"))
 
     llm_router = MagicMock()
     llm_router.get_model_list.return_value = [
-        {"model_info": {"id": "gpt-4-model-1", "team_id": "team-a"}},
+        user_added_model,
+        {"model_name": "team-model", "model_info": {"id": "team-model-1", "team_id": "team-a"}},
     ]
 
-    with pytest.raises(HTTPException) as exc_info:
-        await non_admin_all_models(
-            all_models=[],
-            llm_router=llm_router,
-            user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", user_id="deleted-user"),
-            prisma_client=prisma_client,
-        )
+    result = await non_admin_all_models(
+        all_models=[user_added_model],
+        llm_router=llm_router,
+        user_api_key_dict=UserAPIKeyAuth(api_key="sk-test", user_id="ghost-user"),
+        prisma_client=prisma_client,
+    )
 
-    assert exc_info.value.status_code == 400
-    assert exc_info.value.detail == {"error": "User not found"}
+    assert result == [user_added_model]
 
 
 @pytest.mark.asyncio

From 64d8b24c10a0eb9821390c33146ef068e36e9bfb Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 17:00:00 -0700
Subject: [PATCH 076/105] fix(router): deliver the hold-back retrieval error to
 the client instead of falling back

---
 .../messages/agentic_streaming_iterator.py    |  4 +++
 litellm/router.py                             | 11 ++++++
 .../router_code_coverage.py                   |  1 +
 tests/test_litellm/test_router.py             | 34 +++++++++++++++++++
 4 files changed, 50 insertions(+)

diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py
index 3d3d3a12b17..55a85c011d0 100644
--- a/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py
+++ b/litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py
@@ -31,6 +31,10 @@ SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES: Final = (
 )
 
 
+def is_server_fulfilled_tool_leak_error(chunk: object) -> bool:
+    return chunk == SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES
+
+
 async def _anext_or_none(iterator: AsyncIterator) -> bytes | None:
     try:
         return await iterator.__anext__()
diff --git a/litellm/router.py b/litellm/router.py
index 4cceecf50d6..b99eb2598c1 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -398,6 +398,16 @@ def _is_retriable_anthropic_status(status_code: int) -> bool:
     return status_code == 429 or status_code >= 500
 
 
+def _anthropic_stream_error_is_gateway_verdict(chunk: object) -> bool:
+    """AgenticAnthropicStreamingIterator's own retrieval-failure frame is the gateway's verdict, not a provider
+    failure: another deployment would rerun the same failed hook, so it reaches the client instead of falling back."""
+    from litellm.llms.anthropic.experimental_pass_through.messages.agentic_streaming_iterator import (
+        is_server_fulfilled_tool_leak_error,
+    )
+
+    return is_server_fulfilled_tool_leak_error(chunk)
+
+
 def _anthropic_stream_should_decline_fallback(has_generated_content: bool, error: "MidStreamFallbackError") -> bool:
     """
     A MidStreamFallbackError raised directly by the source iterator (the
@@ -5018,6 +5028,7 @@ class Router:
                         not has_generated_content
                         and error_event is not None
                         and _is_retriable_anthropic_status(error_event[2])
+                        and not _anthropic_stream_error_is_gateway_verdict(chunk)
                     )
                     if not has_generated_content and not retriable_pending_error and error_event is None:
                         buffered_lifecycle_chunks = (*buffered_lifecycle_chunks, chunk)
diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py
index 014f3a16b59..c541c035db7 100644
--- a/tests/code_coverage_tests/router_code_coverage.py
+++ b/tests/code_coverage_tests/router_code_coverage.py
@@ -80,6 +80,7 @@ ignored_function_names = [
     "_override_vector_store_methods_for_router",  # No-op placeholder, called during Router init
     "_merge_tools_from_deployment",  # Tested indirectly via _update_kwargs_with_deployment (test files lack "router" in name)
     "_invalidate_access_groups_cache",  # Tested indirectly via set_model_list, upsert_model etc. (test files lack "router" in name)
+    "has_buffered_provider_output",  # Property, so its reads in test_router.py are never an ast.Call
 ]
 
 
diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py
index 4b9b91caf2c..25c33c4a202 100644
--- a/tests/test_litellm/test_router.py
+++ b/tests/test_litellm/test_router.py
@@ -15,11 +15,15 @@ import litellm
 from litellm import Router
 from litellm.exceptions import MidStreamFallbackError
 from litellm.integrations.custom_logger import CustomLogger
+from litellm.llms.anthropic.experimental_pass_through.messages.agentic_streaming_iterator import (
+    SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES,
+)
 from litellm.router import (
     MAX_BUFFERED_PRE_CONTENT_ANTHROPIC_CHUNKS,
     FallbackAwareAnthropicMessagesStream,
     _anthropic_stream_commits_now,
     _anthropic_stream_should_decline_fallback,
+    _anthropic_stream_error_is_gateway_verdict,
     _anthropic_stream_forwards_ping_live,
     _anthropic_stream_should_drop_pre_content_ping,
     _is_retriable_anthropic_status,
@@ -9545,6 +9549,30 @@ async def test_anthropic_messages_leading_ping_does_not_disqualify_fallback():
     ]
 
 
+@pytest.mark.asyncio
+async def test_anthropic_messages_hold_back_retrieval_failure_reaches_client_without_fallback():
+    """The hold-back iterator's own retrieval-failure frame is the gateway's verdict, not a
+    provider failure: a configured fallback stays untouched and the client reads the error
+    right after the live keepalive."""
+    router = _anthropic_messages_make_router()
+    source = _AnthropicMessagesFakeByteStream(
+        [_anthropic_messages_ping_chunk(), SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES]
+    )
+    fallback = AsyncMock(
+        return_value=_AnthropicMessagesFallbackByteStream([_anthropic_messages_content_chunk("fallback answer")])
+    )
+
+    with patch.object(router, "async_function_with_fallbacks_common_utils", new=fallback):
+        wrapped = await router._aanthropic_messages_streaming_iterator(
+            response=source,
+            initial_kwargs={"model": "primary"},
+        )
+        collected = [chunk async for chunk in wrapped]
+
+    fallback.assert_not_called()
+    assert collected == [_anthropic_messages_ping_chunk(), SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES]
+
+
 @pytest.mark.asyncio
 async def test_anthropic_messages_pre_content_buffer_cap_forces_commit():
     """Bugbot regression: a hostile or pathological upstream that never emits
@@ -9883,6 +9911,12 @@ def test_anthropic_stream_forwards_ping_live_direct_call():
     assert _anthropic_stream_forwards_ping_live(content, has_generated_content=False, buffered_chunk_count=0) is False
 
 
+def test_anthropic_stream_error_is_gateway_verdict_direct_call():
+    assert _anthropic_stream_error_is_gateway_verdict(SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES) is True
+    assert _anthropic_stream_error_is_gateway_verdict(_anthropic_messages_overloaded_error_chunk()) is False
+    assert _anthropic_stream_error_is_gateway_verdict(_anthropic_messages_ping_chunk()) is False
+
+
 def test_fallback_aware_stream_reports_withheld_output_of_its_current_source():
     """The proxy's cancel-refund guard reads this flag off the router wrapper, so it
     must reflect the stream actually being drained: the primary, then the fallback."""

From 8d877980e0e7564f7394008fe621140f2758ee10 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 17:31:46 -0700
Subject: [PATCH 077/105] fix(caching): carry has_buffered_provider_output
 through the anthropic stream cache writer

---
 .../messages/response_cache.py                |  4 +++
 .../messages/test_response_cache.py           | 28 +++++++++++++++++++
 2 files changed, 32 insertions(+)

diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/response_cache.py b/litellm/llms/anthropic/experimental_pass_through/messages/response_cache.py
index 9ac5187681b..86dfe8ff451 100644
--- a/litellm/llms/anthropic/experimental_pass_through/messages/response_cache.py
+++ b/litellm/llms/anthropic/experimental_pass_through/messages/response_cache.py
@@ -46,6 +46,10 @@ class AnthropicMessagesStreamCacheWriter:
             stream._hidden_params if isinstance(stream, AnthropicMessagesStreamingResponse) else _EMPTY_MAPPING
         )
 
+    @property
+    def has_buffered_provider_output(self) -> bool:
+        return getattr(self.stream, "has_buffered_provider_output", False) is True
+
     def __aiter__(self) -> "AnthropicMessagesStreamCacheWriter":
         return self
 
diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_response_cache.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_response_cache.py
index fe0bcfa4f30..22d14614108 100644
--- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_response_cache.py
+++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_response_cache.py
@@ -4,9 +4,15 @@ from typing import Any, AsyncIterator, Dict, List
 import pytest
 
 
+import datetime
+
 import litellm
 from litellm.caching.caching import Cache, LiteLLMCacheType
+from litellm.caching.caching_handler import LLMCachingHandler
 from litellm.llms.anthropic.experimental_pass_through.messages import handler
+from litellm.llms.anthropic.experimental_pass_through.messages.response_cache import (
+    AnthropicMessagesStreamCacheWriter,
+)
 
 STREAM_EVENTS: List[bytes] = [
     b'event: message_start\ndata: {"type": "message_start", "message": {"id": "msg_stream_1", "type": "message", '
@@ -262,3 +268,25 @@ async def test_cached_stream_replay_logs_once_when_polled_after_exhaustion():
         await asyncio.sleep(0)
 
     mock_route.assert_called_once()
+class _HeldBackStream:
+    has_buffered_provider_output = True
+
+    def __aiter__(self) -> "_HeldBackStream":
+        return self
+
+    async def __anext__(self) -> bytes:
+        raise StopAsyncIteration
+
+
+def test_cache_writer_forwards_has_buffered_provider_output(request_kwargs):
+    caching_handler = LLMCachingHandler(
+        original_function=handler.anthropic_messages,
+        request_kwargs=dict(request_kwargs),
+        start_time=datetime.datetime.now(),
+    )
+    held_back = AnthropicMessagesStreamCacheWriter(stream=_HeldBackStream(), caching_handler=caching_handler)
+    assert held_back.has_buffered_provider_output is True
+    replayable = AnthropicMessagesStreamCacheWriter(
+        stream=_byte_stream(STREAM_EVENTS), caching_handler=caching_handler
+    )
+    assert replayable.has_buffered_provider_output is False

From 7c3da429c7fa56f52268383b55b01777ab2867e4 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 17:40:41 -0700
Subject: [PATCH 078/105] test(together_ai): regression suite across chat,
 responses, and messages surfaces

Adds streaming, async, /v1/responses, and /v1/messages coverage for the
Together AI overhaul (#38233, #38248, #38230, #38265, #38275), plus the
legacy api.together.xyz host and TOGETHER_AI_API_BASE through
litellm.completion. Each new test fails under a one-line mutation of the
merged code.
---
 .../test_together_ai_chat_transformation.py   | 339 +++++++++++++++++-
 1 file changed, 338 insertions(+), 1 deletion(-)

diff --git a/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py b/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py
index f6d4b73f27a..e4b90a38435 100644
--- a/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py
+++ b/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py
@@ -1,6 +1,6 @@
 import json
 import logging
-from collections.abc import Mapping, Sequence
+from collections.abc import Iterator, Mapping, Sequence
 from unittest.mock import MagicMock
 
 import httpx
@@ -9,6 +9,7 @@ import pytest
 import litellm
 from litellm.exceptions import UnsupportedParamsError
 from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from litellm.llms.openai.chat.gpt_transformation import (
     OpenAIChatCompletionStreamingHandler,
 )
@@ -33,6 +34,11 @@ def force_local_model_cost(monkeypatch):
     monkeypatch.setattr(litellm, "model_cost", get_model_cost_map(url=litellm.model_cost_map_url))
 
 
+@pytest.fixture(autouse=True)
+def isolate_together_api_base_env(monkeypatch):
+    monkeypatch.delenv("TOGETHER_AI_API_BASE", raising=False)
+
+
 @pytest.fixture
 def registry_disables_function_calling(monkeypatch):
     monkeypatch.setitem(
@@ -476,3 +482,334 @@ def test_completion_unmapped_model_sends_tools_to_together():
     tool_call = response.choices[0].message.tool_calls[0]
     assert tool_call.function.name == "get_weather"
     assert json.loads(tool_call.function.arguments) == {"city": "San Francisco"}
+
+
+TOGETHER_CHAT_URL = "https://api.together.ai/v1/chat/completions"
+
+WEATHER_AND_TIME_TOOLS = [
+    *WEATHER_TOOLS,
+    {"type": "function", "function": {"name": "get_time", "parameters": {}}},
+]
+
+ANTHROPIC_WEATHER_TOOL = {
+    "name": "get_weather",
+    "description": "Get the weather",
+    "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}},
+}
+
+
+def _chat_completion(message: Mapping[str, object], finish_reason: str = "stop") -> dict:
+    return {
+        "id": "chatcmpl-together",
+        "object": "chat.completion",
+        "created": 1234567890,
+        "model": UNMAPPED_MODEL,
+        "choices": [{"index": 0, "message": dict(message), "finish_reason": finish_reason}],
+        "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
+    }
+
+
+def _chunk(delta: Mapping[str, object], finish_reason: str | None = None) -> dict:
+    return {
+        "id": "chatcmpl-together-stream",
+        "object": "chat.completion.chunk",
+        "created": 1234567890,
+        "model": UNMAPPED_MODEL,
+        "choices": [{"index": 0, "delta": dict(delta), "finish_reason": finish_reason}],
+    }
+
+
+def _sse(*events: Mapping[str, object]) -> bytes:
+    return b"".join(f"data: {json.dumps(event)}\n\n".encode() for event in events) + b"data: [DONE]\n\n"
+
+
+def _sse_response(*events: Mapping[str, object]) -> httpx.Response:
+    return httpx.Response(200, content=_sse(*events), headers={"Content-Type": "text/event-stream"})
+
+
+def _sync_client(captured_requests: list[httpx.Request], response: httpx.Response) -> HTTPHandler:
+    def respond(request: httpx.Request) -> httpx.Response:
+        captured_requests.append(request)
+        return response
+
+    return HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond)))
+
+
+def _async_client(captured_requests: list[httpx.Request], response: httpx.Response) -> AsyncHTTPHandler:
+    def respond(request: httpx.Request) -> httpx.Response:
+        captured_requests.append(request)
+        return response
+
+    handler = AsyncHTTPHandler()
+    handler.client = httpx.AsyncClient(transport=httpx.MockTransport(respond))
+    return handler
+
+
+PARALLEL_TOOL_CALL_STREAM = (
+    _chunk({"role": "assistant", "reasoning": "Need weather "}),
+    _chunk({"reasoning": "and time."}),
+    _chunk(
+        {
+            "tool_calls": [
+                {"index": 0, "id": "call_weather", "type": "function", "function": {"name": "get_weather", "arguments": ""}}
+            ]
+        }
+    ),
+    _chunk({"tool_calls": [{"index": 0, "function": {"arguments": '{"city": "San'}}]}),
+    _chunk({"tool_calls": [{"index": 0, "function": {"arguments": ' Francisco"}'}}]}),
+    _chunk(
+        {"tool_calls": [{"index": 1, "id": "call_time", "type": "function", "function": {"name": "get_time", "arguments": ""}}]}
+    ),
+    _chunk({"tool_calls": [{"index": 1, "function": {"arguments": '{"tz": "PST"}'}}]}, finish_reason="tool_calls"),
+)
+
+
+def test_streaming_completion_rebuilds_reasoning_and_parallel_tool_calls():
+    captured_requests: list[httpx.Request] = []
+    client = _sync_client(captured_requests, _sse_response(*PARALLEL_TOOL_CALL_STREAM))
+
+    chunks = list(
+        litellm.completion(
+            model=f"together_ai/{UNMAPPED_MODEL}",
+            messages=[{"role": "user", "content": "Weather and time in San Francisco?"}],
+            tools=WEATHER_AND_TIME_TOOLS,
+            stream=True,
+            api_key="fake-key",
+            client=client,
+        )
+    )
+
+    request_body = json.loads(captured_requests[0].content)
+    assert str(captured_requests[0].url) == TOGETHER_CHAT_URL
+    assert request_body["stream"] is True
+    assert request_body["tools"] == WEATHER_AND_TIME_TOOLS
+
+    streamed_reasoning = "".join(getattr(chunk.choices[0].delta, "reasoning_content", None) or "" for chunk in chunks)
+    assert streamed_reasoning == "Need weather and time."
+
+    rebuilt = litellm.stream_chunk_builder(chunks)
+    message = rebuilt.choices[0].message
+    assert message.reasoning_content == "Need weather and time."
+    assert rebuilt.choices[0].finish_reason == "tool_calls"
+    calls = {call.id: call for call in message.tool_calls}
+    assert calls["call_weather"].function.name == "get_weather"
+    assert json.loads(calls["call_weather"].function.arguments) == {"city": "San Francisco"}
+    assert calls["call_time"].function.name == "get_time"
+    assert json.loads(calls["call_time"].function.arguments) == {"tz": "PST"}
+
+
+async def test_async_streaming_completion_strips_internal_fields_and_streams_reasoning():
+    captured_requests: list[httpx.Request] = []
+    client = _async_client(
+        captured_requests,
+        _sse_response(
+            _chunk({"role": "assistant", "reasoning": "Recalling 47."}),
+            _chunk({"content": "47"}, finish_reason="stop"),
+        ),
+    )
+
+    stream = await litellm.acompletion(
+        model=f"together_ai/{REASONING_MODEL}",
+        messages=[dict(message) for message in PRESERVED_THINKING_MESSAGES],
+        chat_template_kwargs={"clear_thinking": False},
+        stream=True,
+        api_key="fake-key",
+        client=client,
+    )
+    chunks = [chunk async for chunk in stream]
+
+    request_body = json.loads(captured_requests[0].content)
+    assert str(captured_requests[0].url) == TOGETHER_CHAT_URL
+    assert request_body["chat_template_kwargs"] == {"clear_thinking": False}
+    _assert_internal_fields_stripped_reasoning_kept(request_body["messages"])
+
+    rebuilt = litellm.stream_chunk_builder(chunks)
+    assert rebuilt.choices[0].message.reasoning_content == "Recalling 47."
+    assert rebuilt.choices[0].message.content == "47"
+
+
+@pytest.mark.parametrize("api_base", ["https://api.together.ai/v1", "https://api.together.xyz/v1"])
+def test_completion_bare_model_with_together_api_base_uses_together_config(api_base):
+    captured_requests: list[httpx.Request] = []
+    client = _sync_client(
+        captured_requests,
+        httpx.Response(200, json=_chat_completion({"role": "assistant", "content": "4", "reasoning": "2+2"})),
+    )
+
+    response = litellm.completion(
+        model=UNMAPPED_MODEL,
+        messages=[{"role": "user", "content": "What is 2+2?"}],
+        api_base=api_base,
+        api_key="fake-key",
+        client=client,
+    )
+
+    assert str(captured_requests[0].url) == f"{api_base}/chat/completions"
+    assert captured_requests[0].headers["authorization"] == "Bearer fake-key"
+    assert response._hidden_params["custom_llm_provider"] == "together_ai"
+    assert response.choices[0].message.reasoning_content == "2+2"
+
+
+def test_completion_honors_together_ai_api_base_env(monkeypatch):
+    monkeypatch.setenv("TOGETHER_AI_API_BASE", "https://together.internal.example/v1")
+    captured_requests: list[httpx.Request] = []
+    client = _sync_client(
+        captured_requests,
+        httpx.Response(200, json=_chat_completion({"role": "assistant", "content": "4"})),
+    )
+
+    litellm.completion(
+        model=f"together_ai/{REASONING_MODEL}",
+        messages=[{"role": "user", "content": "What is 2+2?"}],
+        api_key="fake-key",
+        client=client,
+    )
+
+    assert str(captured_requests[0].url) == "https://together.internal.example/v1/chat/completions"
+
+
+def test_responses_api_sends_tools_and_maps_reasoning_and_function_call():
+    captured_requests: list[httpx.Request] = []
+    client = _sync_client(
+        captured_requests,
+        httpx.Response(
+            200,
+            json=_chat_completion(
+                {
+                    "role": "assistant",
+                    "content": None,
+                    "reasoning": "Need the weather tool.",
+                    "tool_calls": [
+                        {
+                            "id": "call_abc123",
+                            "type": "function",
+                            "function": {"name": "get_weather", "arguments": '{"city": "San Francisco"}'},
+                        }
+                    ],
+                },
+                finish_reason="tool_calls",
+            ),
+        ),
+    )
+
+    response = litellm.responses(
+        model=f"together_ai/{UNMAPPED_MODEL}",
+        input="What is the weather in San Francisco?",
+        tools=[{"type": "function", "name": "get_weather", "parameters": {}}],
+        api_key="fake-key",
+        client=client,
+    )
+
+    request_body = json.loads(captured_requests[0].content)
+    assert str(captured_requests[0].url) == TOGETHER_CHAT_URL
+    assert [tool["function"]["name"] for tool in request_body["tools"]] == ["get_weather"]
+    outputs = {item.type: item for item in response.output}
+    assert outputs["reasoning"].content[0].text == "Need the weather tool."
+    assert outputs["function_call"].name == "get_weather"
+    assert json.loads(outputs["function_call"].arguments) == {"city": "San Francisco"}
+
+
+ANTHROPIC_TOOL_LOOP_MESSAGES = [
+    {"role": "user", "content": "What is the weather in San Francisco?"},
+    {
+        "role": "assistant",
+        "content": [
+            {"type": "thinking", "thinking": "I should call get_weather.", "signature": ""},
+            {"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {"city": "San Francisco"}},
+        ],
+    },
+    {
+        "role": "user",
+        "content": [{"type": "tool_result", "tool_use_id": "toolu_01", "content": "Sunny, 18C"}],
+    },
+]
+
+
+def test_anthropic_messages_replays_tool_loop_and_maps_reasoning_to_thinking_block():
+    captured_requests: list[httpx.Request] = []
+    client = _sync_client(
+        captured_requests,
+        httpx.Response(
+            200,
+            json=_chat_completion({"role": "assistant", "content": "Sunny in SF.", "reasoning": "Tool said sunny."}),
+        ),
+    )
+
+    response = litellm.anthropic.messages.create(
+        model=f"together_ai/{UNMAPPED_MODEL}",
+        max_tokens=100,
+        messages=[dict(message) for message in ANTHROPIC_TOOL_LOOP_MESSAGES],
+        tools=[ANTHROPIC_WEATHER_TOOL],
+        api_key="fake-key",
+        client=client,
+    )
+
+    request_body = json.loads(captured_requests[0].content)
+    assert str(captured_requests[0].url) == TOGETHER_CHAT_URL
+    assert [tool["function"]["name"] for tool in request_body["tools"]] == ["get_weather"]
+    assistant_turn = request_body["messages"][1]
+    assert assistant_turn["role"] == "assistant"
+    assert assistant_turn["reasoning_content"] == "I should call get_weather."
+    assert "thinking_blocks" not in assistant_turn
+    replayed_call = assistant_turn["tool_calls"][0]
+    assert replayed_call["id"] == "toolu_01"
+    assert replayed_call["function"]["name"] == "get_weather"
+    assert json.loads(replayed_call["function"]["arguments"]) == {"city": "San Francisco"}
+    tool_turn = request_body["messages"][2]
+    assert tool_turn["role"] == "tool"
+    assert tool_turn["tool_call_id"] == "toolu_01"
+    assert tool_turn["content"] == "Sunny, 18C"
+
+    blocks = {block["type"]: block for block in response["content"]}
+    assert blocks["thinking"]["thinking"] == "Tool said sunny."
+    assert blocks["text"]["text"] == "Sunny in SF."
+    assert response["stop_reason"] == "end_turn"
+
+
+def _anthropic_sse_events(stream: Iterator[bytes]) -> list[dict]:
+    return [
+        json.loads(line.removeprefix("data: "))
+        for raw in stream
+        for line in raw.decode().splitlines()
+        if line.startswith("data: ")
+    ]
+
+
+def test_anthropic_messages_streams_together_tool_call_as_input_json_delta():
+    captured_requests: list[httpx.Request] = []
+    client = _sync_client(captured_requests, _sse_response(*PARALLEL_TOOL_CALL_STREAM))
+
+    events = _anthropic_sse_events(
+        litellm.anthropic.messages.create(
+            model=f"together_ai/{UNMAPPED_MODEL}",
+            max_tokens=100,
+            messages=[{"role": "user", "content": "Weather and time in San Francisco?"}],
+            tools=[ANTHROPIC_WEATHER_TOOL, {"name": "get_time", "input_schema": {"type": "object"}}],
+            stream=True,
+            api_key="fake-key",
+            client=client,
+        )
+    )
+
+    assert json.loads(captured_requests[0].content)["stream"] is True
+    tool_starts = {
+        event["index"]: event["content_block"]
+        for event in events
+        if event["type"] == "content_block_start" and event["content_block"]["type"] == "tool_use"
+    }
+    input_json_deltas = [
+        event for event in events if event["type"] == "content_block_delta" and event["delta"]["type"] == "input_json_delta"
+    ]
+    tool_inputs = {
+        block["name"]: json.loads("".join(delta["delta"]["partial_json"] for delta in input_json_deltas if delta["index"] == index))
+        for index, block in tool_starts.items()
+    }
+    assert {block["id"] for block in tool_starts.values()} == {"call_weather", "call_time"}
+    assert tool_inputs == {"get_weather": {"city": "San Francisco"}, "get_time": {"tz": "PST"}}
+    thinking_text = "".join(
+        event["delta"]["thinking"]
+        for event in events
+        if event["type"] == "content_block_delta" and event["delta"]["type"] == "thinking_delta"
+    )
+    assert thinking_text == "Need weather and time."
+    assert [event["delta"]["stop_reason"] for event in events if event["type"] == "message_delta"] == ["tool_use"]

From 602bf1c0ac36c626832995a891affd42c93b3854 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 17:45:54 -0700
Subject: [PATCH 079/105] test(together_ai): close the injected async client
 after the streaming test

---
 .../test_together_ai_chat_transformation.py   | 26 +++++++++++--------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py b/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py
index e4b90a38435..e3a958ff167 100644
--- a/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py
+++ b/tests/test_litellm/llms/together_ai/chat/test_together_ai_chat_transformation.py
@@ -535,12 +535,13 @@ def _sync_client(captured_requests: list[httpx.Request], response: httpx.Respons
     return HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond)))
 
 
-def _async_client(captured_requests: list[httpx.Request], response: httpx.Response) -> AsyncHTTPHandler:
+async def _async_client(captured_requests: list[httpx.Request], response: httpx.Response) -> AsyncHTTPHandler:
     def respond(request: httpx.Request) -> httpx.Response:
         captured_requests.append(request)
         return response
 
     handler = AsyncHTTPHandler()
+    await handler.close()
     handler.client = httpx.AsyncClient(transport=httpx.MockTransport(respond))
     return handler
 
@@ -600,7 +601,7 @@ def test_streaming_completion_rebuilds_reasoning_and_parallel_tool_calls():
 
 async def test_async_streaming_completion_strips_internal_fields_and_streams_reasoning():
     captured_requests: list[httpx.Request] = []
-    client = _async_client(
+    client = await _async_client(
         captured_requests,
         _sse_response(
             _chunk({"role": "assistant", "reasoning": "Recalling 47."}),
@@ -608,15 +609,18 @@ async def test_async_streaming_completion_strips_internal_fields_and_streams_rea
         ),
     )
 
-    stream = await litellm.acompletion(
-        model=f"together_ai/{REASONING_MODEL}",
-        messages=[dict(message) for message in PRESERVED_THINKING_MESSAGES],
-        chat_template_kwargs={"clear_thinking": False},
-        stream=True,
-        api_key="fake-key",
-        client=client,
-    )
-    chunks = [chunk async for chunk in stream]
+    try:
+        stream = await litellm.acompletion(
+            model=f"together_ai/{REASONING_MODEL}",
+            messages=[dict(message) for message in PRESERVED_THINKING_MESSAGES],
+            chat_template_kwargs={"clear_thinking": False},
+            stream=True,
+            api_key="fake-key",
+            client=client,
+        )
+        chunks = [chunk async for chunk in stream]
+    finally:
+        await client.client.aclose()
 
     request_body = json.loads(captured_requests[0].content)
     assert str(captured_requests[0].url) == TOGETHER_CHAT_URL

From a543b0348c9db5040fd91d796429047cd2368428 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 18:12:33 -0700
Subject: [PATCH 080/105] test(e2e): cover Together AI reasoning, tool calls,
 template kwargs, and cost through a live proxy

---
 .../coverage_registry/llm_conversational.yaml |  11 +
 .../llm_translation/test_together_ai_e2e.py   | 566 ++++++++++++++++++
 tests/e2e/models.py                           |  70 ++-
 tests/e2e/proxy_client.py                     |  15 +
 4 files changed, 652 insertions(+), 10 deletions(-)
 create mode 100644 tests/e2e/llm_translation/test_together_ai_e2e.py

diff --git a/tests/e2e/coverage_registry/llm_conversational.yaml b/tests/e2e/coverage_registry/llm_conversational.yaml
index 1d4e1e028ca..fc3803e5790 100644
--- a/tests/e2e/coverage_registry/llm_conversational.yaml
+++ b/tests/e2e/coverage_registry/llm_conversational.yaml
@@ -75,3 +75,14 @@
 - {id: llm.responses.vertex.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: vertex, capability: tool_use, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Responses tool calls w/ Vertex"}
 - {id: llm.responses.azure_openai.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: azure_openai, capability: basic, streaming: nonstream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Responses w/ Azure OpenAI (smoke)"}
 - {id: llm.responses.azure_openai.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: azure_openai, capability: tool_use, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Responses tool calls w/ Azure OpenAI"}
+- {id: llm.chat_completions.together_ai.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: thinking, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together reasoning surfaces as reasoning_content (LIT-5960)"}
+- {id: llm.chat_completions.together_ai.thinking.stream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: thinking, streaming: stream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together reasoning deltas stream as reasoning_content"}
+- {id: llm.chat_completions.together_ai.thinking.nonstream.template_kwargs_forwarded, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: thinking, streaming: nonstream, assertions: [template_kwargs_forwarded], source: "llm_translation/test_together_ai_e2e.py", rationale: "chat_template_kwargs reaches Together and turns thinking off"}
+- {id: llm.chat_completions.together_ai.thinking.nonstream.replayed_reasoning_forwarded, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: thinking, streaming: nonstream, assertions: [replayed_reasoning_forwarded], source: "llm_translation/test_together_ai_e2e.py", rationale: "Replayed reasoning_content survives the Together message transform"}
+- {id: llm.chat_completions.together_ai.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together tool calls are not dropped"}
+- {id: llm.chat_completions.together_ai.tool_use.stream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: tool_use, streaming: stream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together tool calls over streaming"}
+- {id: llm.chat_completions.together_ai.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together tool result round trip"}
+- {id: llm.chat_completions.together_ai.basic.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: chat_completions, route: together_ai, capability: basic, streaming: nonstream, assertions: [cost_logged], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together cost header and spend row match the registry price"}
+- {id: llm.messages.together_ai.basic.stream.works, module: llm, tier: P1, subject_endpoint: messages, route: together_ai, capability: basic, streaming: stream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together over /v1/messages streaming"}
+- {id: llm.messages.together_ai.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: together_ai, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together tool calls over /v1/messages"}
+- {id: llm.messages.together_ai.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: together_ai, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_together_ai_e2e.py", rationale: "Together tool result round trip over /v1/messages"}
diff --git a/tests/e2e/llm_translation/test_together_ai_e2e.py b/tests/e2e/llm_translation/test_together_ai_e2e.py
new file mode 100644
index 00000000000..0a446b629de
--- /dev/null
+++ b/tests/e2e/llm_translation/test_together_ai_e2e.py
@@ -0,0 +1,566 @@
+"""Live e2e: Together AI through the gateway, driven the way its users drive it.
+
+Covers the Together overhaul (LIT-5960): reasoning surfaces as ``reasoning_content``
+non-streaming and streamed, tool calls survive the transform and round-trip a tool
+result on both ``/chat/completions`` and ``/v1/messages``, ``chat_template_kwargs``
+reaches Together, and the cost header and spend row agree with the registry price.
+
+Model selection is registry-driven so the matrix does not rot: the reasoning and
+tool-calling deployment is the cheapest live ``together_ai/`` chat model whose cost
+map row (read from the proxy's own ``/public/litellm_model_cost_map``) carries
+``supports_function_calling`` and ``supports_reasoning``. The one pinned model,
+``TEMPLATE_KWARGS_BACKEND``, is pinned on purpose: ``enable_thinking`` is a Qwen
+chat-template contract the registry has no flag for. ``REASONING_REPLAY_BACKEND`` is
+pinned for the same reason: it is the serverless Together model whose template
+renders a replayed ``reasoning_content`` back into the prompt (its prompt_tokens grow
+with the replay and it answers from it; Qwen and DeepSeek silently drop it), which is
+what makes a stripped replay observable end to end.
+
+Requires TOGETHER_API_KEY on the proxy (tests/e2e/.env). No skip gate: once the
+proxy is up, a failure here is real, per the suite's hard-fail contract.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from datetime import date
+
+import pytest
+from e2e_config import unique_marker
+from e2e_http import StreamingResponse, require_successful_call, unwrap
+from lifecycle import ResourceManager
+from models import (
+    AnthropicAssistantTurn,
+    AnthropicContentBlock,
+    AnthropicCustomTool,
+    AnthropicMessagesBody,
+    AnthropicToolResultBlock,
+    AnthropicToolResultTurn,
+    ChatAssistantTurn,
+    ChatBody,
+    ChatMessage,
+    ChatResponse,
+    ChatTool,
+    ChatToolFunction,
+    ChatToolResultTurn,
+    CostMapEntry,
+    JsonSchemaProperty,
+    LiteLLMParamsBody,
+    OutMessage,
+    SpendLogRow,
+    ToolCall,
+    ToolInputSchema,
+)
+from passthrough_client import PassthroughClient
+from pydantic import BaseModel
+
+pytestmark = pytest.mark.e2e
+
+TEMPLATE_KWARGS_BACKEND = "together_ai/Qwen/Qwen3.5-9B"
+REASONING_REPLAY_BACKEND = "together_ai/MiniMaxAI/MiniMax-M3"
+
+SECRET_PROMPT = "Remember this for later and reply with just OK."
+SECRET_REASONING = "The user told me their favorite color is chartreuse. I must remember it."
+SECRET_QUESTION = "What is my favorite color? Answer with one word."
+
+ARITHMETIC_PROMPT = "What is 17 + 26? Answer with just the number."
+WEATHER_PROMPT = "What is the weather in Paris? Use the tool."
+WEATHER_REPORT = "Paris: 22 degrees Celsius, clear skies, wind from the northwest at 9 km/h"
+COUNTING_PROMPT = "Count from 1 to 20, one number per line."
+
+WEATHER_TOOL = ChatTool(
+    function=ChatToolFunction(
+        name="get_weather",
+        description="Get the current weather for a location.",
+        parameters={
+            "type": "object",
+            "properties": {"location": {"type": "string"}},
+            "required": ["location"],
+        },
+    )
+)
+
+MESSAGES_WEATHER_TOOL = AnthropicCustomTool(
+    name="get_weather",
+    description="Get the current weather for a location.",
+    input_schema=ToolInputSchema(
+        properties={"location": JsonSchemaProperty(type="string")},
+        required=["location"],
+    ),
+)
+
+
+@dataclass(frozen=True, slots=True)
+class _Needs:
+    function_calling: bool = False
+    reasoning: bool = False
+
+
+class _WeatherArgs(BaseModel):
+    location: str
+
+
+class _StreamToolCallFunction(BaseModel):
+    name: str | None = None
+    arguments: str | None = None
+
+
+class _StreamToolCall(BaseModel):
+    function: _StreamToolCallFunction | None = None
+
+
+class _StreamDelta(BaseModel):
+    content: str | None = None
+    reasoning_content: str | None = None
+    tool_calls: list[_StreamToolCall] | None = None
+
+
+class _StreamChoice(BaseModel):
+    delta: _StreamDelta | None = None
+
+
+class _StreamChunk(BaseModel):
+    choices: list[_StreamChoice] = []
+
+
+class _MessagesEventDelta(BaseModel):
+    type: str | None = None
+    text: str = ""
+
+
+class _MessagesStreamEvent(BaseModel):
+    type: str
+    delta: _MessagesEventDelta | None = None
+
+
+def _approx_equal(actual: float, expected: float) -> bool:
+    return abs(actual - expected) <= max(1e-9, abs(expected) * 1e-2)
+
+
+def _cheapest_together_chat_model(registry: Mapping[str, CostMapEntry], needs: _Needs) -> str:
+    today = date.today().isoformat()
+
+    def qualifies(name: str, entry: CostMapEntry) -> bool:
+        return (
+            name.startswith("together_ai/")
+            and entry.litellm_provider == "together_ai"
+            and entry.mode == "chat"
+            and (entry.deprecation_date is None or entry.deprecation_date > today)
+            and (entry.input_cost_per_token or 0.0) > 0
+            and (entry.output_cost_per_token or 0.0) > 0
+            and (not needs.function_calling or bool(entry.supports_function_calling))
+            and (not needs.reasoning or bool(entry.supports_reasoning))
+        )
+
+    candidates = sorted(
+        (name for name, entry in registry.items() if qualifies(name, entry)),
+        key=lambda name: (
+            registry[name].input_cost_per_token or 0.0,
+            registry[name].output_cost_per_token or 0.0,
+            name,
+        ),
+    )
+    assert candidates, f"no live together_ai chat model in the proxy's cost map satisfies {needs}"
+    return candidates[0]
+
+
+@pytest.fixture(scope="module")
+def registry(client: PassthroughClient) -> dict[str, CostMapEntry]:
+    return client.proxy.model_cost_map()
+
+
+@pytest.fixture(scope="module")
+def reasoning_tool_backend(registry: dict[str, CostMapEntry]) -> str:
+    return _cheapest_together_chat_model(registry, _Needs(function_calling=True, reasoning=True))
+
+
+def _register(client: PassthroughClient, resources: ResourceManager, backend: str) -> tuple[str, str]:
+    model = f"e2e-together-{unique_marker()}"
+    model_id = client.proxy.create_model(
+        model, LiteLLMParamsBody(model=backend, api_key="os.environ/TOGETHER_API_KEY")
+    )
+    resources.defer(lambda: client.proxy.delete_model(model_id))
+    return model, resources.key()
+
+
+def _message(response: ChatResponse) -> OutMessage:
+    assert response.choices, f"Together returned no choices: {response}"
+    message = response.choices[0].message
+    assert message is not None, f"Together choice has no message: {response}"
+    return message
+
+
+def _deltas(result: StreamingResponse) -> list[_StreamDelta]:
+    require_successful_call(result)
+    assert result.is_streaming, f"response was not streamed: {result.headers}"
+    assert not result.stream_error, f"stream errored: {result.stream_error}"
+    assert result.stream_done, f"stream never reached [DONE]: {result.stream_events[-3:]}"
+    return [
+        choice.delta
+        for event in result.stream_events
+        for choice in _StreamChunk.model_validate_json(event).choices
+        if choice.delta is not None
+    ]
+
+
+def _single_weather_call(message: OutMessage) -> ToolCall:
+    assert message.tool_calls, f"Together dropped the tool call: {message}"
+    assert len(message.tool_calls) == 1, f"expected one tool call, got {message.tool_calls}"
+    call = message.tool_calls[0]
+    assert call.id, f"tool call carries no id, so a tool result cannot answer it: {call}"
+    assert call.function.name == "get_weather", f"wrong tool called: {call}"
+    assert call.function.arguments, f"tool call carries no arguments: {call}"
+    args = _WeatherArgs.model_validate_json(call.function.arguments)
+    assert "paris" in args.location.lower(), f"tool arguments lost the location: {args}"
+    return call
+
+
+def _weather_call(client: PassthroughClient, key: str, model: str) -> OutMessage:
+    return _message(
+        unwrap(
+            client.proxy.chat(
+                key,
+                ChatBody(
+                    model=model,
+                    messages=[ChatMessage(role="user", content=WEATHER_PROMPT)],
+                    tools=[WEATHER_TOOL],
+                    max_tokens=512,
+                ),
+            )
+        )
+    )
+
+
+class TestTogetherChatCompletions:
+    @pytest.mark.covers("llm.chat_completions.together_ai.thinking.nonstream.works")
+    def test_reasoning_surfaces_as_reasoning_content(
+        self, client: PassthroughClient, resources: ResourceManager, reasoning_tool_backend: str
+    ) -> None:
+        model, key = _register(client, resources, reasoning_tool_backend)
+
+        message = _message(
+            unwrap(
+                client.proxy.chat(
+                    key,
+                    ChatBody(
+                        model=model,
+                        messages=[ChatMessage(role="user", content=ARITHMETIC_PROMPT)],
+                        max_tokens=1024,
+                    ),
+                )
+            )
+        )
+        assert message.reasoning_content, (
+            f"{reasoning_tool_backend} reasons, but no reasoning_content came back: {message}"
+        )
+        assert message.content and "43" in message.content, f"answer lost: {message}"
+
+    @pytest.mark.covers("llm.chat_completions.together_ai.thinking.stream.works")
+    def test_reasoning_streams_as_reasoning_content_deltas(
+        self, client: PassthroughClient, resources: ResourceManager, reasoning_tool_backend: str
+    ) -> None:
+        model, key = _register(client, resources, reasoning_tool_backend)
+
+        deltas = _deltas(
+            client.proxy.chat_stream(
+                key,
+                ChatBody(
+                    model=model,
+                    messages=[ChatMessage(role="user", content=ARITHMETIC_PROMPT)],
+                    max_tokens=1024,
+                    stream=True,
+                ),
+            )
+        )
+        reasoning = "".join(delta.reasoning_content or "" for delta in deltas)
+        content = "".join(delta.content or "" for delta in deltas)
+        assert reasoning, f"stream carried no reasoning_content deltas: {deltas[:5]}"
+        assert "43" in content, f"streamed answer lost: {content!r}"
+
+    @pytest.mark.covers("llm.chat_completions.together_ai.tool_use.nonstream.works")
+    def test_tool_call_is_returned(
+        self, client: PassthroughClient, resources: ResourceManager, reasoning_tool_backend: str
+    ) -> None:
+        model, key = _register(client, resources, reasoning_tool_backend)
+        _single_weather_call(_weather_call(client, key, model))
+
+    @pytest.mark.covers("llm.chat_completions.together_ai.tool_use.stream.works")
+    def test_tool_call_is_streamed(
+        self, client: PassthroughClient, resources: ResourceManager, reasoning_tool_backend: str
+    ) -> None:
+        model, key = _register(client, resources, reasoning_tool_backend)
+
+        deltas = _deltas(
+            client.proxy.chat_stream(
+                key,
+                ChatBody(
+                    model=model,
+                    messages=[ChatMessage(role="user", content=WEATHER_PROMPT)],
+                    tools=[WEATHER_TOOL],
+                    max_tokens=512,
+                    stream=True,
+                ),
+            )
+        )
+        calls = [
+            call.function
+            for delta in deltas
+            for call in delta.tool_calls or []
+            if call.function is not None
+        ]
+        assert calls, f"stream carried no tool call deltas: {deltas[:5]}"
+        names = {call.name for call in calls if call.name}
+        assert names == {"get_weather"}, f"unexpected streamed tool names: {names}"
+        arguments = "".join(call.arguments or "" for call in calls)
+        args = _WeatherArgs.model_validate_json(arguments)
+        assert "paris" in args.location.lower(), f"streamed tool arguments lost the location: {args}"
+
+    @pytest.mark.covers("llm.chat_completions.together_ai.multi_turn.nonstream.works")
+    def test_tool_result_round_trip(
+        self, client: PassthroughClient, resources: ResourceManager, reasoning_tool_backend: str
+    ) -> None:
+        model, key = _register(client, resources, reasoning_tool_backend)
+        first = _weather_call(client, key, model)
+        call = _single_weather_call(first)
+        assert call.id is not None
+
+        answer = _message(
+            unwrap(
+                client.proxy.chat(
+                    key,
+                    ChatBody(
+                        model=model,
+                        messages=[
+                            ChatMessage(role="user", content=WEATHER_PROMPT),
+                            ChatAssistantTurn(
+                                content=first.content,
+                                reasoning_content=first.reasoning_content,
+                                tool_calls=first.tool_calls,
+                            ),
+                            ChatToolResultTurn(tool_call_id=call.id, content=WEATHER_REPORT),
+                        ],
+                        tools=[WEATHER_TOOL],
+                        max_tokens=512,
+                    ),
+                )
+            )
+        )
+        assert answer.content and "22" in answer.content, (
+            f"the model never saw the tool result: {answer}"
+        )
+
+    @pytest.mark.covers("llm.chat_completions.together_ai.thinking.nonstream.template_kwargs_forwarded")
+    def test_chat_template_kwargs_reach_together(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        """Control first: the pinned Qwen model thinks by default, so the treatment's
+        missing reasoning_content proves ``enable_thinking: false`` reached Together
+        rather than the model simply not reasoning."""
+        model, key = _register(client, resources, TEMPLATE_KWARGS_BACKEND)
+
+        def ask(chat_template_kwargs: dict[str, bool] | None) -> OutMessage:
+            return _message(
+                unwrap(
+                    client.proxy.chat(
+                        key,
+                        ChatBody(
+                            model=model,
+                            messages=[ChatMessage(role="user", content=ARITHMETIC_PROMPT)],
+                            max_tokens=1024,
+                            chat_template_kwargs=chat_template_kwargs,
+                        ),
+                    )
+                )
+            )
+
+        control = ask(None)
+        assert control.reasoning_content, (
+            f"control: {TEMPLATE_KWARGS_BACKEND} returned no reasoning_content by default, "
+            f"so the disable assertion below cannot be trusted: {control}"
+        )
+        treatment = ask({"enable_thinking": False})
+        assert not treatment.reasoning_content, (
+            "chat_template_kwargs={'enable_thinking': False} did not reach Together: "
+            f"reasoning_content is still present: {treatment}"
+        )
+        assert treatment.content and "43" in treatment.content, f"answer lost: {treatment}"
+
+    @pytest.mark.covers("llm.chat_completions.together_ai.thinking.nonstream.replayed_reasoning_forwarded")
+    def test_replayed_reasoning_content_reaches_together(
+        self, client: PassthroughClient, resources: ResourceManager
+    ) -> None:
+        """The only place the fact lives is the replayed assistant turn's
+        ``reasoning_content``, so the model can answer only if the Together transform
+        forwarded it (LIT-5960: it used to strip it with the thinking blocks)."""
+        model, key = _register(client, resources, REASONING_REPLAY_BACKEND)
+
+        answer = _message(
+            unwrap(
+                client.proxy.chat(
+                    key,
+                    ChatBody(
+                        model=model,
+                        messages=[
+                            ChatMessage(role="user", content=SECRET_PROMPT),
+                            ChatAssistantTurn(content="OK.", reasoning_content=SECRET_REASONING),
+                            ChatMessage(role="user", content=SECRET_QUESTION),
+                        ],
+                        max_tokens=512,
+                    ),
+                )
+            )
+        )
+        assert answer.content and "chartreuse" in answer.content.lower(), (
+            f"the replayed reasoning_content never reached Together: {answer}"
+        )
+
+    @pytest.mark.covers("llm.chat_completions.together_ai.basic.nonstream.cost_logged")
+    def test_cost_header_and_spend_row_match_the_registry_price(
+        self,
+        client: PassthroughClient,
+        resources: ResourceManager,
+        registry: dict[str, CostMapEntry],
+        reasoning_tool_backend: str,
+    ) -> None:
+        model, key = _register(client, resources, reasoning_tool_backend)
+
+        result = client.proxy.transport.send(
+            "/chat/completions",
+            headers=client.proxy.transport.bearer(key),
+            json=ChatBody(
+                model=model,
+                messages=[ChatMessage(role="user", content=f"{ARITHMETIC_PROMPT} {unique_marker()}")],
+                max_tokens=1024,
+            ),
+        )
+        require_successful_call(result)
+        response = ChatResponse.model_validate_json(result.body)
+        usage = response.usage
+        assert usage is not None and usage.prompt_tokens and usage.completion_tokens, (
+            f"response carries no usage, so the cost cannot be real: {result.body[:300]}"
+        )
+        header_cost = result.response_cost
+        assert header_cost is not None and header_cost > 0, (
+            f"x-litellm-response-cost header missing or non-positive: {result.headers}"
+        )
+
+        price = registry[reasoning_tool_backend]
+        assert price.input_cost_per_token and price.output_cost_per_token
+        cached = (usage.prompt_tokens_details.cached_tokens or 0) if usage.prompt_tokens_details else 0
+        expected = (
+            (usage.prompt_tokens - cached) * price.input_cost_per_token
+            + cached * (price.cache_read_input_token_cost or price.input_cost_per_token)
+            + usage.completion_tokens * price.output_cost_per_token
+        )
+        assert _approx_equal(header_cost, expected), (
+            f"header cost {header_cost} disagrees with the registry price for "
+            f"{reasoning_tool_backend} at {usage}: expected {expected}"
+        )
+
+        def _priced(rows: list[SpendLogRow]) -> bool:
+            return any(row.spend is not None and row.spend > 0 for row in rows)
+
+        rows = client.proxy.poll_logs_for_key(key, predicate=_priced)
+        priced = [row for row in rows if row.spend is not None and row.spend > 0]
+        assert priced, f"no priced spend row landed for key {key}; got {rows}"
+        row = priced[0]
+        assert row.custom_llm_provider == "together_ai", f"spend row misattributed: {row}"
+        assert row.spend is not None and _approx_equal(row.spend, header_cost), (
+            f"logged spend {row.spend} disagrees with the x-litellm-response-cost header {header_cost}"
+        )
+
+
+def _tool_use_blocks(content: list[AnthropicContentBlock] | None) -> list[AnthropicContentBlock]:
+    assert content, f"/v1/messages returned no content blocks: {content}"
+    return [block for block in content if block.type == "tool_use"]
+
+
+def _messages_weather_call(
+    client: PassthroughClient, key: str, model: str
+) -> tuple[list[AnthropicContentBlock], AnthropicContentBlock]:
+    response = unwrap(
+        client.proxy.messages(
+            key,
+            AnthropicMessagesBody(
+                model=model,
+                max_tokens=512,
+                tools=[MESSAGES_WEATHER_TOOL],
+                messages=[ChatMessage(role="user", content=WEATHER_PROMPT)],
+            ),
+        )
+    )
+    tool_uses = _tool_use_blocks(response.content)
+    assert len(tool_uses) == 1, f"expected one tool_use block, got {response.content}"
+    block = tool_uses[0]
+    assert block.name == "get_weather", f"wrong tool called: {block}"
+    assert block.id, f"tool_use block carries no id, so a tool_result cannot answer it: {block}"
+    assert response.content is not None
+    return response.content, block
+
+
+class TestTogetherMessages:
+    @pytest.mark.covers("llm.messages.together_ai.tool_use.nonstream.works")
+    def test_tool_use_block_is_returned(
+        self, client: PassthroughClient, resources: ResourceManager, reasoning_tool_backend: str
+    ) -> None:
+        model, key = _register(client, resources, reasoning_tool_backend)
+        _messages_weather_call(client, key, model)
+
+    @pytest.mark.covers("llm.messages.together_ai.multi_turn.nonstream.works")
+    def test_tool_result_round_trip(
+        self, client: PassthroughClient, resources: ResourceManager, reasoning_tool_backend: str
+    ) -> None:
+        model, key = _register(client, resources, reasoning_tool_backend)
+        first_content, block = _messages_weather_call(client, key, model)
+        assert block.id is not None
+
+        response = unwrap(
+            client.proxy.messages(
+                key,
+                AnthropicMessagesBody(
+                    model=model,
+                    max_tokens=512,
+                    tools=[MESSAGES_WEATHER_TOOL],
+                    messages=[
+                        ChatMessage(role="user", content=WEATHER_PROMPT),
+                        AnthropicAssistantTurn(content=first_content),
+                        AnthropicToolResultTurn(
+                            content=[AnthropicToolResultBlock(tool_use_id=block.id, content=WEATHER_REPORT)]
+                        ),
+                    ],
+                ),
+            )
+        )
+        assert response.content, f"/v1/messages returned no content blocks: {response}"
+        text = "".join(block.text or "" for block in response.content if block.type == "text")
+        assert "22" in text, f"the model never saw the tool result: {response.content}"
+
+    @pytest.mark.covers("llm.messages.together_ai.basic.stream.works")
+    def test_streams_text_deltas(
+        self, client: PassthroughClient, resources: ResourceManager, reasoning_tool_backend: str
+    ) -> None:
+        model, key = _register(client, resources, reasoning_tool_backend)
+
+        result = client.proxy.messages_stream(
+            key,
+            AnthropicMessagesBody(
+                model=model,
+                max_tokens=512,
+                stream=True,
+                messages=[ChatMessage(role="user", content=COUNTING_PROMPT)],
+            ),
+        )
+        require_successful_call(result)
+        assert result.is_streaming, f"response was not streamed: {result.headers}"
+        assert not result.stream_error, f"stream errored: {result.stream_error}"
+        events = [_MessagesStreamEvent.model_validate_json(event) for event in result.stream_events]
+        types = [event.type for event in events]
+        text_deltas = [
+            event.delta.text
+            for event in events
+            if event.type == "content_block_delta" and event.delta is not None and event.delta.text
+        ]
+        assert len(text_deltas) >= 2, f"stream was not incremental: {types}"
+        assert "20" in "".join(text_deltas), f"streamed text lost the answer: {text_deltas}"
+        assert "message_stop" in types, f"stream never reached message_stop: {types}"
diff --git a/tests/e2e/models.py b/tests/e2e/models.py
index 5e2cb90958e..3e8c55b1461 100644
--- a/tests/e2e/models.py
+++ b/tests/e2e/models.py
@@ -217,9 +217,45 @@ class McpChatTool(BaseModel):
     allowed_tools: list[str] | None = None
 
 
+class ToolCallFunction(BaseModel):
+    name: str | None = None
+    arguments: str | None = None
+
+
+class ToolCall(BaseModel):
+    """A tool call as the model emitted it. `id` and `type` are what a client
+    replays in the assistant turn so the tool result can be matched to it."""
+
+    id: str | None = None
+    type: str | None = None
+    function: ToolCallFunction = ToolCallFunction()
+
+
+class ChatAssistantTurn(BaseModel):
+    """An assistant turn replayed into history as the client received it: the tool
+    calls a tool result answers, or the reasoning a thinking-preserving chat
+    template consumes on the next turn."""
+
+    role: Literal["assistant"] = "assistant"
+    content: str | None = None
+    reasoning_content: str | None = None
+    tool_calls: list[ToolCall] | None = None
+
+
+class ChatToolResultTurn(BaseModel):
+    """The tool-role answer to a `ToolCall`; `tool_call_id` must be the id the model emitted."""
+
+    role: Literal["tool"] = "tool"
+    tool_call_id: str
+    content: str
+
+
+type ChatTurn = ChatMessage | ChatAssistantTurn | ChatToolResultTurn
+
+
 class ChatBody(BaseModel):
     model: str
-    messages: list[ChatMessage]
+    messages: Sequence[ChatTurn]
     stream: bool = False
     max_tokens: int | None = None
     max_completion_tokens: int | None = None
@@ -233,6 +269,7 @@ class ChatBody(BaseModel):
     tool_choice: str | None = None
     guardrails: list[str] | None = None
     response_format: dict[str, object] | None = None
+    chat_template_kwargs: dict[str, bool] | None = None
     cache: dict[str, bool] | None = {"no-cache": True}
 
 
@@ -259,15 +296,6 @@ class ReliabilityChatBody(ChatBody):
     router_settings_override: RouterSettingsOverride | None = None
 
 
-class ToolCallFunction(BaseModel):
-    name: str | None = None
-    arguments: str | None = None
-
-
-class ToolCall(BaseModel):
-    function: ToolCallFunction = ToolCallFunction()
-
-
 class McpToolFunctionRef(BaseModel):
     name: str
 
@@ -400,6 +428,7 @@ class AnthropicContentBlock(BaseModel):
     type: str | None = None
     text: str | None = None
     id: str | None = None
+    name: str | None = None
 
 
 class AnthropicToolResultBlock(BaseModel):
@@ -689,6 +718,27 @@ class ModelInfoResponse(BaseModel):
     data: list[ModelInfoEntry] = []
 
 
+class CostMapEntry(BaseModel):
+    """One `/public/litellm_model_cost_map` row: the price and capability flags the
+    proxy resolves for a `<provider>/<model>` key. Tests pick a deployment from
+    these flags instead of hardcoding a model name that rots."""
+
+    model_config = ConfigDict(extra="ignore")
+    litellm_provider: str | None = None
+    mode: str | None = None
+    deprecation_date: str | None = None
+    input_cost_per_token: float | None = None
+    output_cost_per_token: float | None = None
+    cache_read_input_token_cost: float | None = None
+    supports_function_calling: bool | None = None
+    supports_reasoning: bool | None = None
+    supports_response_schema: bool | None = None
+
+
+class CostMap(RootModel[dict[str, CostMapEntry]]):
+    pass
+
+
 class FileEntry(BaseModel):
     id: str
 
diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py
index d12364e1794..c56a0649f58 100644
--- a/tests/e2e/proxy_client.py
+++ b/tests/e2e/proxy_client.py
@@ -29,6 +29,8 @@ from models import (
     AnthropicMessagesResponse,
     ChatBody,
     ChatResponse,
+    CostMap,
+    CostMapEntry,
     CountTokensBody,
     CountTokensResponse,
     CredentialCreateBody,
@@ -251,6 +253,19 @@ class ProxyClient:
             )
         ).data
 
+    def model_cost_map(self) -> dict[str, CostMapEntry]:
+        """The registry the proxy prices with and gates capabilities on, keyed
+        `<provider>/<model>`, read from the proxy itself so a test sees exactly the
+        map the deployment under test resolves against."""
+        return unwrap(
+            self.transport.get(
+                "/public/litellm_model_cost_map",
+                headers=self.transport.master,
+                params=NoBody(),
+                response_type=CostMap,
+            )
+        ).root
+
     def list_files(self, key: str) -> Result[FileListResponse]:
         return self.transport.get(
             "/v1/files",

From cc6b3783feebd6c6c3ee6d1cb09ea758497e648e Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 18:17:28 -0700
Subject: [PATCH 081/105] test(e2e): price Together cached tokens at the
 registry cache read rate

---
 tests/e2e/llm_translation/test_together_ai_e2e.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/e2e/llm_translation/test_together_ai_e2e.py b/tests/e2e/llm_translation/test_together_ai_e2e.py
index 0a446b629de..1e257573e5f 100644
--- a/tests/e2e/llm_translation/test_together_ai_e2e.py
+++ b/tests/e2e/llm_translation/test_together_ai_e2e.py
@@ -423,6 +423,9 @@ class TestTogetherChatCompletions:
         registry: dict[str, CostMapEntry],
         reasoning_tool_backend: str,
     ) -> None:
+        """Cached prompt tokens (Together caches shared prefixes across calls) bill at
+        the registry's cache read rate, which is $0 until a row carries one (LIT-5972),
+        so the expected cost is computed from the same registry row the proxy prices with."""
         model, key = _register(client, resources, reasoning_tool_backend)
 
         result = client.proxy.transport.send(
@@ -450,7 +453,7 @@ class TestTogetherChatCompletions:
         cached = (usage.prompt_tokens_details.cached_tokens or 0) if usage.prompt_tokens_details else 0
         expected = (
             (usage.prompt_tokens - cached) * price.input_cost_per_token
-            + cached * (price.cache_read_input_token_cost or price.input_cost_per_token)
+            + cached * (price.cache_read_input_token_cost or 0.0)
             + usage.completion_tokens * price.output_cost_per_token
         )
         assert _approx_equal(header_cost, expected), (

From 55e3a9785c7369f5988c858305fd8a8cf44646e0 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 18:45:52 -0700
Subject: [PATCH 082/105] test(e2e): trim docstrings that restate the Together
 tests

---
 .../llm_translation/test_together_ai_e2e.py   | 35 ++++---------------
 tests/e2e/models.py                           |  9 -----
 2 files changed, 7 insertions(+), 37 deletions(-)

diff --git a/tests/e2e/llm_translation/test_together_ai_e2e.py b/tests/e2e/llm_translation/test_together_ai_e2e.py
index 1e257573e5f..340edaed791 100644
--- a/tests/e2e/llm_translation/test_together_ai_e2e.py
+++ b/tests/e2e/llm_translation/test_together_ai_e2e.py
@@ -1,23 +1,11 @@
-"""Live e2e: Together AI through the gateway, driven the way its users drive it.
+"""Live e2e: Together AI through the gateway on /chat/completions and /v1/messages.
 
-Covers the Together overhaul (LIT-5960): reasoning surfaces as ``reasoning_content``
-non-streaming and streamed, tool calls survive the transform and round-trip a tool
-result on both ``/chat/completions`` and ``/v1/messages``, ``chat_template_kwargs``
-reaches Together, and the cost header and spend row agree with the registry price.
-
-Model selection is registry-driven so the matrix does not rot: the reasoning and
-tool-calling deployment is the cheapest live ``together_ai/`` chat model whose cost
-map row (read from the proxy's own ``/public/litellm_model_cost_map``) carries
-``supports_function_calling`` and ``supports_reasoning``. The one pinned model,
-``TEMPLATE_KWARGS_BACKEND``, is pinned on purpose: ``enable_thinking`` is a Qwen
-chat-template contract the registry has no flag for. ``REASONING_REPLAY_BACKEND`` is
-pinned for the same reason: it is the serverless Together model whose template
-renders a replayed ``reasoning_content`` back into the prompt (its prompt_tokens grow
-with the replay and it answers from it; Qwen and DeepSeek silently drop it), which is
-what makes a stripped replay observable end to end.
-
-Requires TOGETHER_API_KEY on the proxy (tests/e2e/.env). No skip gate: once the
-proxy is up, a failure here is real, per the suite's hard-fail contract.
+The reasoning and tool-calling backend is the cheapest live ``together_ai/`` chat row
+in the proxy's own cost map that carries both capability flags. Two backends are
+pinned because the registry has no flag for what they prove: ``enable_thinking`` is a
+Qwen chat-template contract, and MiniMax-M3 is the serverless model whose template
+renders a replayed ``reasoning_content`` back into the prompt (Qwen and DeepSeek
+silently drop it). Requires TOGETHER_API_KEY on the proxy; no skip gate.
 """
 
 from __future__ import annotations
@@ -354,9 +342,6 @@ class TestTogetherChatCompletions:
     def test_chat_template_kwargs_reach_together(
         self, client: PassthroughClient, resources: ResourceManager
     ) -> None:
-        """Control first: the pinned Qwen model thinks by default, so the treatment's
-        missing reasoning_content proves ``enable_thinking: false`` reached Together
-        rather than the model simply not reasoning."""
         model, key = _register(client, resources, TEMPLATE_KWARGS_BACKEND)
 
         def ask(chat_template_kwargs: dict[str, bool] | None) -> OutMessage:
@@ -390,9 +375,6 @@ class TestTogetherChatCompletions:
     def test_replayed_reasoning_content_reaches_together(
         self, client: PassthroughClient, resources: ResourceManager
     ) -> None:
-        """The only place the fact lives is the replayed assistant turn's
-        ``reasoning_content``, so the model can answer only if the Together transform
-        forwarded it (LIT-5960: it used to strip it with the thinking blocks)."""
         model, key = _register(client, resources, REASONING_REPLAY_BACKEND)
 
         answer = _message(
@@ -423,9 +405,6 @@ class TestTogetherChatCompletions:
         registry: dict[str, CostMapEntry],
         reasoning_tool_backend: str,
     ) -> None:
-        """Cached prompt tokens (Together caches shared prefixes across calls) bill at
-        the registry's cache read rate, which is $0 until a row carries one (LIT-5972),
-        so the expected cost is computed from the same registry row the proxy prices with."""
         model, key = _register(client, resources, reasoning_tool_backend)
 
         result = client.proxy.transport.send(
diff --git a/tests/e2e/models.py b/tests/e2e/models.py
index 3e8c55b1461..086d3b03603 100644
--- a/tests/e2e/models.py
+++ b/tests/e2e/models.py
@@ -223,19 +223,12 @@ class ToolCallFunction(BaseModel):
 
 
 class ToolCall(BaseModel):
-    """A tool call as the model emitted it. `id` and `type` are what a client
-    replays in the assistant turn so the tool result can be matched to it."""
-
     id: str | None = None
     type: str | None = None
     function: ToolCallFunction = ToolCallFunction()
 
 
 class ChatAssistantTurn(BaseModel):
-    """An assistant turn replayed into history as the client received it: the tool
-    calls a tool result answers, or the reasoning a thinking-preserving chat
-    template consumes on the next turn."""
-
     role: Literal["assistant"] = "assistant"
     content: str | None = None
     reasoning_content: str | None = None
@@ -243,8 +236,6 @@ class ChatAssistantTurn(BaseModel):
 
 
 class ChatToolResultTurn(BaseModel):
-    """The tool-role answer to a `ToolCall`; `tool_call_id` must be the id the model emitted."""
-
     role: Literal["tool"] = "tool"
     tool_call_id: str
     content: str

From 893482d4acd15473542948babfcb52fe0f040a56 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 18:51:40 -0700
Subject: [PATCH 083/105] test(e2e): drop docstrings on the cost map model and
 its client accessor

---
 tests/e2e/models.py       | 4 ----
 tests/e2e/proxy_client.py | 3 ---
 2 files changed, 7 deletions(-)

diff --git a/tests/e2e/models.py b/tests/e2e/models.py
index 086d3b03603..77555a9fc95 100644
--- a/tests/e2e/models.py
+++ b/tests/e2e/models.py
@@ -710,10 +710,6 @@ class ModelInfoResponse(BaseModel):
 
 
 class CostMapEntry(BaseModel):
-    """One `/public/litellm_model_cost_map` row: the price and capability flags the
-    proxy resolves for a `<provider>/<model>` key. Tests pick a deployment from
-    these flags instead of hardcoding a model name that rots."""
-
     model_config = ConfigDict(extra="ignore")
     litellm_provider: str | None = None
     mode: str | None = None
diff --git a/tests/e2e/proxy_client.py b/tests/e2e/proxy_client.py
index c56a0649f58..2d382a610e1 100644
--- a/tests/e2e/proxy_client.py
+++ b/tests/e2e/proxy_client.py
@@ -254,9 +254,6 @@ class ProxyClient:
         ).data
 
     def model_cost_map(self) -> dict[str, CostMapEntry]:
-        """The registry the proxy prices with and gates capabilities on, keyed
-        `<provider>/<model>`, read from the proxy itself so a test sees exactly the
-        map the deployment under test resolves against."""
         return unwrap(
             self.transport.get(
                 "/public/litellm_model_cost_map",

From d16c4dad4b89a373867e1f39803014133b6339e5 Mon Sep 17 00:00:00 2001
From: Deepanshu Lulla <deepanshu.lulla@gmail.com>
Date: Tue, 25 Aug 2026 22:01:03 -0400
Subject: [PATCH 084/105] feat(logging): add
 async_post_call_failure_deployment_hook (#36657)

* feat(logging): add async_post_call_failure_deployment_hook

CustomLogger already has async_pre_call_deployment_hook and
async_post_call_success_deployment_hook, both firing once per real
deployment attempt from wrapper_async since the router re-enters that
wrapper fresh on every retry and fallback step. There was no failure-side
counterpart; the only failure signal, async_log_failure_event, fires once
per logical client request behind a dedup gate, so fallback chain attempts
2+ were invisible to callbacks needing per-deployment-attempt granularity.

Adds async_post_call_failure_deployment_hook(request_data, exception,
call_type) to CustomLogger and a matching dispatcher in utils.py, called
from wrapper_async's except block. It needs no dedup coordination since
each real attempt naturally re-enters the wrapper once. Unlike its two
siblings, the dispatcher wraps each callback call in its own try/except
since it runs on the wrapper's own exception path and a broken callback
must never mask the exception about to be re-raised to the caller.

* feat(logging): pass fallback_depth through to async_post_call_failure_deployment_hook

Router already tracks fallback_depth internally on each fallback hop
(litellm/router_utils/fallback_event_handlers.py), incrementing it once per
target tried, but nothing surfaced it to CustomLogger callbacks. Reads it
off request_data in the dispatcher and passes it through as a best-effort
int | None keyword: None on the first, pre-fallback attempt or a bare SDK
call with no router, 1 on the first fallback hop, 2 on the second, and so
on. Verified live against a real multi-hop Router fallback chain before
adding the regression tests.

* fix(logging): fire async_post_call_failure_deployment_hook on internal calls too

The failure hook was gated behind the same not _is_litellm_internal_call
check as the request-level dedup-gated failure logging, so a failed
internal sub-call (e.g. an emulated file-search step) never reached it,
even though its async_pre_call_deployment_hook and
async_post_call_success_deployment_hook siblings already fire
unconditionally for such calls.

* chore: retrigger CI (lint job hit a transient GitHub Actions infra outage on the prior push)

* chore: retrigger CI (lint job hit the same GitHub Actions infra outage again)

* fix(logging): scope async_post_call_failure_deployment_hook to the actual model call

The hook was dispatched from the wrapper's broad outer except, which also
catches BudgetExceededError (raised before any deployment attempt),
errors from async_pre_call_deployment_hook, and errors raised after a
successful model call (post_call_processing, async_post_call_success_deployment_hook,
caching). None of those are a deployment attempt failing, so the hook
misreported them as one.

Scoped the hook to a try/except around the model call itself, so it only
fires when that specific call raises, matching its own documented contract.

* test: assert the callback actually ran in the failure-hook error-isolation test

An upstream test-quality gate (TQ001) flagged this test for asserting
nothing, so it could only fail by raising. Track whether the exploding
callback actually ran and assert on it, so the test would catch a
dispatcher that silently skipped every callback instead of isolating a
raising one.

* fix(logging): harden async_post_call_failure_deployment_hook against 5 maintainer-verified issues

A maintainer's live-proxy A/B review against base found five real
problems with the failure hook, all reproduced and fixed:

- The dispatcher called overrides with fallback_depth as a required
  keyword, so an override matching this PR's own earlier 3-arg
  proof-of-fix example raised TypeError, swallowed at debug level, on
  every call. Now checks the override's signature once per class and
  omits the keyword when unsupported.
- A callback mutating the exception it receives (e.g. status_code)
  changed what the real caller got back, since it was the same live
  object about to be re-raised. Callbacks now receive a same-class
  snapshot instead.
- request_data exposed attempted_targets, the router's own live
  fallback-walk bookkeeping shared by reference across every hop, so a
  callback calling .record() on it could make the router skip a
  deployment it never actually tried. Now excluded from what the hook
  receives.
- The hook's own await sat directly in the model-call except block, so
  a caller-side cancellation landing mid-await (e.g. asyncio.wait_for)
  replaced the real deployment exception with CancelledError/
  TimeoutError. Now isolated so hook dispatch can never mask the real
  failure.
- The timestamp used for the reported failure duration was captured
  after the hook ran, so a slow callback inflated
  async_log_failure_event's duration. Now captured before the hook
  dispatches.

* fix(logging): preserve traceback/cause/context on the failure-hook exception snapshot

Bugbot found a real gap in the previous round's exception-mutation fix:
_snapshot_exception_for_hook only copied __dict__ and args, so a
callback formatting or inspecting the failure chain saw an empty
traceback and lost chained-exception context, even though the live
exception still has them. __traceback__/__cause__/__context__ aren't
stored in __dict__, so they need copying explicitly.

* fix(logging): preserve __suppress_context__ on the failure-hook exception snapshot

Setting __cause__ has a documented CPython side effect of implicitly
forcing __suppress_context__ to True. Since the previous round's
traceback fix set __cause__ before __suppress_context__, a normal
implicit-chaining exception (no `raise ... from`, __suppress_context__
naturally False) got its context wrongly suppressed on the snapshot.
Now __suppress_context__ is set explicitly, after __cause__, so it
always reflects the real exception.

* fix(logging): use MappingProxyType for the failure-hook's sanitized request_data

A LIT002 budget check (surfaced by rebasing onto a moved base) flagged
the dict comprehension building safe_request_data as mutable
construction. MappingProxyType is also a strictly better fit here: a
genuinely read-only view, not just an immutable-looking dict, matching
the intent that callbacks should never be able to mutate what they're
handed.

---------

Co-authored-by: Deepanshu <deepanshu.lulla@alpha-sense.com>
---
 litellm/integrations/custom_logger.py |  50 ++-
 litellm/utils.py                      | 112 +++++-
 tests/test_litellm/test_utils.py      | 558 ++++++++++++++++++++++++++
 3 files changed, 717 insertions(+), 3 deletions(-)

diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py
index 0339be106ea..c957fd6e61e 100644
--- a/litellm/integrations/custom_logger.py
+++ b/litellm/integrations/custom_logger.py
@@ -2,7 +2,7 @@
 #    On success, logs events to Promptlayer
 import re
 import traceback
-from collections.abc import AsyncGenerator
+from collections.abc import AsyncGenerator, Mapping
 from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional
 
 from pydantic import BaseModel
@@ -293,6 +293,54 @@ class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callbac
         Allow modifying / reviewing the response just after it's received from the deployment.
         """
 
+    async def async_post_call_failure_deployment_hook(
+        self,
+        request_data: Mapping[str, Any],
+        exception: Exception,
+        call_type: CallTypes | None,
+        fallback_depth: int | None = None,
+    ) -> None:
+        """
+        Called once per failed deployment attempt - attempt 1, every retry, and
+        every fallback chain step - because the router re-invokes the wrapped
+        function on each attempt, re-entering this hook's call site fresh
+        every time.
+
+        This is a DEPLOYMENT-LEVEL signal, distinct from the REQUEST-LEVEL
+        ``async_log_failure_event``, which fires once per logical client
+        request behind a dedup gate. ``request_data`` is mostly this
+        attempt's own kwargs, with one exception: it omits
+        ``attempted_targets``, the router's own bookkeeping of which fallback
+        targets this request has already tried, since that one object *is*
+        shared by reference across every hop of the live fallback walk.
+
+        Pairs with ``async_pre_call_deployment_hook`` and
+        ``async_post_call_success_deployment_hook`` to complete the
+        pre-call/success/failure lifecycle for a single deployment attempt.
+
+        ``fallback_depth`` is best-effort: ``None`` on the first attempt and on
+        any call made without a ``Router`` (a bare SDK call has no fallback
+        chain to be at a depth in), ``1`` on the first fallback hop, ``2`` on
+        the second, and so on. It reflects ``Router``'s own internal fallback
+        bookkeeping (``kwargs["fallback_depth"]``), not a value this hook
+        computes or guarantees the shape of across versions. It tracks
+        fallback hops only, not retries within the same model group - a
+        retry-only failure (no fallback yet) also reports ``None``. If an
+        override predates this field it's simply never passed, rather than
+        raising - safe to leave off an override written before it existed.
+
+        ``exception`` is a same-class snapshot, not the exact object about to
+        be re-raised to the real caller: read it freely, but setting an
+        attribute on it (e.g. ``status_code``) has no effect on what the
+        caller actually receives.
+
+        Default: no-op. Opt in by overriding. Keep overrides fast - this
+        runs on the request's exception path, so a slow implementation
+        delays error propagation to the caller. The reported failure
+        duration is captured before this hook runs, so a slow override
+        doesn't inflate that metric, but the caller still waits for it.
+        """
+
     async def async_post_call_streaming_deployment_hook(
         self,
         request_data: dict,
diff --git a/litellm/utils.py b/litellm/utils.py
index 59a2de609d7..802dc151428 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -35,6 +35,7 @@ from importlib import resources
 from inspect import iscoroutine
 from io import StringIO
 from os.path import abspath, dirname, join
+from types import MappingProxyType
 
 import dotenv
 import httpx
@@ -100,6 +101,43 @@ def _get_cached_custom_logger():
     return _CustomLogger
 
 
+@lru_cache(maxsize=None)
+def _accepts_fallback_depth_kwarg_for_class(cls: type) -> bool:
+    """
+    Whether cls's async_post_call_failure_deployment_hook override accepts a
+    fallback_depth keyword, cached per class so a signature the base class added after a
+    subscriber's override was written (e.g. the PR's own earlier 3-arg proof-of-fix
+    example) doesn't raise TypeError - swallowed at debug level - on every call.
+    """
+    params: Final = inspect.signature(cls.async_post_call_failure_deployment_hook).parameters
+    return "fallback_depth" in params or any(p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values())
+
+
+def _snapshot_exception_for_hook(exception: Exception) -> Exception:
+    """
+    A same-class copy of exception that skips __init__ (many litellm exceptions require
+    constructor args beyond what .args carries, so copy.copy's pickle-based reconstruction
+    fails on them). Handed to failure-hook callbacks instead of the live object so a
+    callback setting e.g. exception.status_code cannot change the status code the real
+    caller actually receives. Falls back to the live object if snapshotting fails for a
+    type this doesn't anticipate, since the real exception must still reach the callback.
+    """
+    try:
+        cls: Final = type(exception)
+        snapshot: Final = cls.__new__(cls)
+        snapshot.__dict__.update(exception.__dict__)
+        snapshot.args = exception.args
+        snapshot.__traceback__ = exception.__traceback__
+        snapshot.__cause__ = exception.__cause__
+        snapshot.__context__ = exception.__context__
+        # Setting __cause__ implicitly forces __suppress_context__ to True (CPython
+        # behavior for `raise ... from ...`), so this must be set after, not before.
+        snapshot.__suppress_context__ = exception.__suppress_context__
+        return snapshot
+    except Exception:  # noqa: BLE001  # any snapshot failure must fall back to the live object, not break the failure path
+        return exception
+
+
 def _get_cached_custom_guardrail():
     """
     Get cached CustomGuardrail class.
@@ -1253,6 +1291,59 @@ async def async_post_call_success_deployment_hook(
     return response
 
 
+async def async_post_call_failure_deployment_hook(
+    request_data: Mapping[str, Any], exception: Exception, call_type: str
+) -> None:
+    """
+    Notify CustomLogger callbacks that a deployment attempt failed.
+
+    Unlike its pre-call/post-success siblings, this wraps each callback call
+    in its own try/except: it runs on the wrapper's exception path, so a
+    broken callback must never replace the real exception that's about to be
+    re-raised to the caller.
+
+    Reads ``fallback_depth`` off ``request_data`` (set by ``Router`` on each
+    fallback hop) and passes it through to the callback; ``None`` when
+    missing or not an int, since a bare SDK call has no fallback chain.
+
+    Callbacks receive a same-class snapshot of ``exception``, not the live
+    object that's about to be re-raised, so a callback setting an attribute
+    on it (e.g. ``status_code``) cannot change what the real caller sees.
+    ``request_data`` omits ``attempted_targets``: unlike the rest of this
+    attempt's own kwargs, it's the *same* object shared by reference across
+    every hop of the live fallback walk, so a callback calling ``.record()``
+    on it would make the router skip a deployment it hasn't actually tried.
+    """
+    try:
+        typed_call_type = CallTypes(call_type)
+    except ValueError:
+        typed_call_type = None  # unknown call type
+
+    _raw_fallback_depth: Final = request_data.get("fallback_depth")
+    fallback_depth: Final = _raw_fallback_depth if isinstance(_raw_fallback_depth, int) else None
+    safe_request_data: Final = MappingProxyType({k: v for k, v in request_data.items() if k != "attempted_targets"})
+    safe_exception: Final = _snapshot_exception_for_hook(exception)
+
+    CustomLogger: Final = _get_cached_custom_logger()
+    for callback in litellm.callbacks:
+        if isinstance(callback, CustomLogger):
+            try:
+                if _accepts_fallback_depth_kwarg_for_class(type(callback)):
+                    await callback.async_post_call_failure_deployment_hook(
+                        safe_request_data, safe_exception, typed_call_type, fallback_depth=fallback_depth
+                    )
+                else:
+                    await callback.async_post_call_failure_deployment_hook(
+                        safe_request_data, safe_exception, typed_call_type
+                    )
+            except Exception as callback_error:  # noqa: BLE001  # a broken callback must not mask the real failure
+                verbose_logger.debug(
+                    "async_post_call_failure_deployment_hook error in %s: %s",
+                    type(callback).__name__,
+                    callback_error,
+                )
+
+
 def post_call_processing(
     original_response,
     model,
@@ -1670,6 +1761,9 @@ def client(original_function):
         is_completion_with_fallbacks: Final = kwargs.get("fallbacks") is not None
         kwargs.pop("_is_litellm_internal_call", None)  # discard if injected
         _is_litellm_internal_call: Final = is_internal_call.get()
+        _deployment_call_end_time: datetime.datetime | None = (
+            None  # rebind-ok: set once, from inside the except below, only if the model call itself fails
+        )
 
         try:
             if logging_obj is None:
@@ -1758,7 +1852,19 @@ def client(original_function):
                     print_verbose(f"Error while checking max token limit: {e}")
 
             # MODEL CALL
-            result = await original_function(*args, **kwargs)
+            try:
+                result = await original_function(*args, **kwargs)
+            except Exception as deployment_error:
+                _deployment_call_end_time = datetime.datetime.now()  # noqa: DTZ005  # matches the naive datetimes this whole function already times start_time/end_time with
+                try:
+                    await async_post_call_failure_deployment_hook(
+                        request_data=kwargs,
+                        exception=deployment_error,
+                        call_type=call_type,
+                    )
+                except BaseException:  # noqa: S110, BLE001  # hook dispatch - including cancellation mid-await - must never replace the real deployment failure, so there is nothing to do with what it raises
+                    pass
+                raise
             end_time = datetime.datetime.now()
 
             if _is_streaming_request(
@@ -1871,7 +1977,9 @@ def client(original_function):
             return result
         except Exception as e:
             traceback_exception: Final = traceback.format_exc()
-            end_time = datetime.datetime.now()
+            # Reuse the timestamp taken right when the deployment call itself failed, before
+            # the failure hook ran, so a slow callback doesn't inflate the reported duration.
+            end_time = _deployment_call_end_time if _deployment_call_end_time is not None else datetime.datetime.now()  # noqa: DTZ005  # matches the naive datetimes this whole function already times start_time/end_time with
             if logging_obj and not _is_litellm_internal_call:
                 try:
                     logging_obj.failure_handler(
diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py
index 77dca9322c6..e1ea6edb2d8 100644
--- a/tests/test_litellm/test_utils.py
+++ b/tests/test_litellm/test_utils.py
@@ -1,3 +1,4 @@
+import asyncio
 import json
 import logging
 import os
@@ -9,6 +10,7 @@ from jsonschema import validate
 
 
 import litellm
+from litellm._internal_context import is_internal_call
 from litellm._logging import (
     CorrelationContextFilter,
     JsonFormatter,
@@ -16,6 +18,7 @@ from litellm._logging import (
     trace_id_var,
     verbose_logger,
 )
+from litellm.integrations.custom_logger import CustomLogger
 from litellm.proxy.utils import is_valid_api_key
 from litellm.types.utils import (
     CallTypes,
@@ -34,6 +37,9 @@ from litellm.utils import (
     _check_provider_match,
     _get_potential_model_names,
     _is_streaming_request,
+    _snapshot_exception_for_hook,
+    async_post_call_failure_deployment_hook,
+    client,
     get_api_key,
     get_llm_provider,
     get_non_default_completion_params,
@@ -4977,3 +4983,555 @@ def test_completion_does_not_leak_rust_flag_into_provider_request_body():
     create_kwargs = mock_client.chat.completions.with_raw_response.create.call_args.kwargs
     assert "rust" not in create_kwargs
     assert "rust" not in (create_kwargs.get("extra_body") or {})
+
+
+class _RecordingDeploymentFailureLogger(CustomLogger):
+    def __init__(self) -> None:
+        super().__init__()
+        self.calls: list[tuple[dict, Exception, CallTypes | None, int | None]] = []
+
+    async def async_post_call_failure_deployment_hook(self, request_data, exception, call_type, fallback_depth=None):
+        self.calls.append((request_data, exception, call_type, fallback_depth))
+
+
+@pytest.mark.asyncio
+async def test_async_post_call_failure_deployment_hook_calls_custom_logger_callbacks(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """The dispatcher must call the CustomLogger hook with an equivalent exception (not
+    necessarily the same object - see test_..._snapshots_exception_so_callback_mutations_..._
+    below) and the call_type resolved to its CallTypes enum member."""
+    recorder = _RecordingDeploymentFailureLogger()
+    monkeypatch.setattr(litellm, "callbacks", [recorder])
+
+    exc = ValueError("deployment failed")
+    await async_post_call_failure_deployment_hook(
+        request_data={"model": "gpt-4o-mini"}, exception=exc, call_type="acompletion"
+    )
+
+    assert len(recorder.calls) == 1
+    request_data, received_exc, call_type, fallback_depth = recorder.calls[0]
+    assert request_data == {"model": "gpt-4o-mini"}
+    assert isinstance(received_exc, ValueError)
+    assert str(received_exc) == str(exc)
+    assert call_type == CallTypes.acompletion
+    assert fallback_depth is None
+
+
+@pytest.mark.asyncio
+async def test_async_post_call_failure_deployment_hook_falls_back_to_none_call_type(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """An unrecognized call_type string must resolve to None rather than raising."""
+    recorder = _RecordingDeploymentFailureLogger()
+    monkeypatch.setattr(litellm, "callbacks", [recorder])
+
+    await async_post_call_failure_deployment_hook(
+        request_data={}, exception=ValueError("x"), call_type="not_a_real_call_type"
+    )
+
+    assert recorder.calls[0][2] is None
+
+
+@pytest.mark.asyncio
+async def test_async_post_call_failure_deployment_hook_passes_through_fallback_depth(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """fallback_depth on request_data (set by Router on each fallback hop) must reach the
+    callback unchanged, so a subscriber can tell which fallback hop this failure is from."""
+    recorder = _RecordingDeploymentFailureLogger()
+    monkeypatch.setattr(litellm, "callbacks", [recorder])
+
+    await async_post_call_failure_deployment_hook(
+        request_data={"fallback_depth": 2}, exception=ValueError("x"), call_type="acompletion"
+    )
+
+    assert recorder.calls[0][3] == 2
+
+
+@pytest.mark.asyncio
+async def test_async_post_call_failure_deployment_hook_fallback_depth_defaults_to_none(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """fallback_depth must be None, not raise or pass through garbage, when request_data has
+    no fallback_depth at all (first attempt, or a bare SDK call with no Router) or a
+    non-int value there."""
+    recorder = _RecordingDeploymentFailureLogger()
+    monkeypatch.setattr(litellm, "callbacks", [recorder])
+
+    await async_post_call_failure_deployment_hook(request_data={}, exception=ValueError("x"), call_type="acompletion")
+    await async_post_call_failure_deployment_hook(
+        request_data={"fallback_depth": "not-an-int"}, exception=ValueError("y"), call_type="acompletion"
+    )
+
+    assert recorder.calls[0][3] is None
+    assert recorder.calls[1][3] is None
+
+
+@pytest.mark.asyncio
+async def test_async_post_call_failure_deployment_hook_swallows_callback_errors(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """A callback that raises inside the hook must not propagate out of the dispatcher."""
+
+    class ExplodingLogger(CustomLogger):
+        def __init__(self) -> None:
+            super().__init__()
+            self.called = False
+
+        async def async_post_call_failure_deployment_hook(self, request_data, exception, call_type, fallback_depth=None):
+            self.called = True
+            raise RuntimeError("hook exploded")
+
+    exploding_logger = ExplodingLogger()
+    monkeypatch.setattr(litellm, "callbacks", [exploding_logger])
+
+    await async_post_call_failure_deployment_hook(request_data={}, exception=ValueError("x"), call_type="acompletion")
+
+    assert exploding_logger.called
+
+
+@pytest.mark.asyncio
+async def test_async_post_call_failure_deployment_hook_skips_non_custom_logger_callbacks(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Callable (function-based) callbacks are not CustomLogger instances and must be skipped."""
+    called: list[bool] = []
+
+    async def fn_callback(*args: object, **kwargs: object) -> None:
+        called.append(True)
+
+    monkeypatch.setattr(litellm, "callbacks", [fn_callback])
+
+    await async_post_call_failure_deployment_hook(request_data={}, exception=ValueError("x"), call_type="acompletion")
+
+    assert called == []
+
+
+@pytest.mark.asyncio
+async def test_wrapper_async_fires_post_call_failure_deployment_hook_once_per_failed_attempt(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Regression: a failed deployment call must reach async_post_call_failure_deployment_hook
+    exactly once, sourced from wrapper_async's own except block rather than the dedup-gated
+    async_log_failure_event path, which would miss retries/fallback chain attempts 2+."""
+    recorder = _RecordingDeploymentFailureLogger()
+    monkeypatch.setattr(litellm, "callbacks", [recorder])
+
+    with pytest.raises(litellm.AuthenticationError):
+        await litellm.acompletion(
+            model="gpt-4o-mini",
+            messages=[{"role": "user", "content": "hi"}],
+            mock_response=litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-4o-mini"),
+        )
+
+    assert len(recorder.calls) == 1
+    _, received_exc, call_type, fallback_depth = recorder.calls[0]
+    assert isinstance(received_exc, litellm.AuthenticationError)
+    assert call_type == CallTypes.acompletion
+    assert fallback_depth is None
+
+
+@pytest.mark.asyncio
+async def test_wrapper_async_raises_original_exception_even_if_hook_callback_errors(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """A broken async_post_call_failure_deployment_hook override must never shadow the real
+    exception the caller is waiting on."""
+
+    class ExplodingLogger(CustomLogger):
+        async def async_post_call_failure_deployment_hook(self, request_data, exception, call_type, fallback_depth=None):
+            raise RuntimeError("hook exploded")
+
+    monkeypatch.setattr(litellm, "callbacks", [ExplodingLogger()])
+
+    with pytest.raises(litellm.AuthenticationError, match="bad key"):
+        await litellm.acompletion(
+            model="gpt-4o-mini",
+            messages=[{"role": "user", "content": "hi"}],
+            mock_response=litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-4o-mini"),
+        )
+
+
+@pytest.mark.asyncio
+async def test_router_fallback_chain_reports_increasing_fallback_depth(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Regression: a real Router fallback chain must report fallback_depth=None on the
+    first, pre-fallback attempt and fallback_depth=1 on the first fallback hop - the
+    concrete scenario async_post_call_failure_deployment_hook exists to make visible."""
+    recorder = _RecordingDeploymentFailureLogger()
+    monkeypatch.setattr(litellm, "callbacks", [recorder])
+
+    router = litellm.Router(
+        model_list=[
+            {"model_name": "bad-group", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "bad-a"}},
+            {"model_name": "good-group", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "bad-b"}},
+        ],
+        num_retries=0,
+        fallbacks=[{"bad-group": ["good-group"]}],
+    )
+
+    with pytest.raises(litellm.AuthenticationError):
+        await router.acompletion(
+            model="bad-group",
+            messages=[{"role": "user", "content": "hi"}],
+            mock_response=litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-4o-mini"),
+        )
+
+    assert len(recorder.calls) == 2
+    assert recorder.calls[0][3] is None
+    assert recorder.calls[1][3] == 1
+
+
+@pytest.mark.asyncio
+async def test_router_multi_hop_fallback_chain_reports_depth_per_hop(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Regression: fallback_depth must keep incrementing across more than one fallback
+    hop (group-a -> group-b -> group-c, all failing), not just report 1 for every
+    fallback attempt regardless of how deep the chain has gone."""
+    recorder = _RecordingDeploymentFailureLogger()
+    monkeypatch.setattr(litellm, "callbacks", [recorder])
+
+    router = litellm.Router(
+        model_list=[
+            {"model_name": "group-a", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "bad-a"}},
+            {"model_name": "group-b", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "bad-b"}},
+            {"model_name": "group-c", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "bad-c"}},
+        ],
+        num_retries=0,
+        fallbacks=[{"group-a": ["group-b", "group-c"]}],
+    )
+
+    with pytest.raises(litellm.AuthenticationError):
+        await router.acompletion(
+            model="group-a",
+            messages=[{"role": "user", "content": "hi"}],
+            mock_response=litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-4o-mini"),
+        )
+
+    assert len(recorder.calls) == 3
+    assert [call[3] for call in recorder.calls] == [None, 1, 2]
+
+
+@pytest.mark.asyncio
+async def test_wrapper_async_fires_post_call_failure_deployment_hook_on_internal_calls(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Regression: a failed attempt made while is_internal_call is set (e.g. an emulated
+    file-search step) must still reach async_post_call_failure_deployment_hook, matching
+    async_pre_call_deployment_hook, which already fires unconditionally for such calls."""
+    recorder = _RecordingDeploymentFailureLogger()
+    monkeypatch.setattr(litellm, "callbacks", [recorder])
+
+    token = is_internal_call.set(True)
+    try:
+        with pytest.raises(litellm.AuthenticationError):
+            await litellm.acompletion(
+                model="gpt-4o-mini",
+                messages=[{"role": "user", "content": "hi"}],
+                mock_response=litellm.AuthenticationError(
+                    message="bad key", llm_provider="openai", model="gpt-4o-mini"
+                ),
+            )
+    finally:
+        is_internal_call.reset(token)
+
+    assert len(recorder.calls) == 1
+    assert isinstance(recorder.calls[0][1], litellm.AuthenticationError)
+
+
+@pytest.mark.asyncio
+async def test_wrapper_async_does_not_fire_failure_hook_for_pre_call_budget_error(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Regression: a BudgetExceededError raised before any deployment call is attempted
+    (the [OPTIONAL] CHECK BUDGET gate) is not a deployment attempt failure and must not
+    reach async_post_call_failure_deployment_hook."""
+    recorder = _RecordingDeploymentFailureLogger()
+    monkeypatch.setattr(litellm, "callbacks", [recorder])
+    monkeypatch.setattr(litellm, "max_budget", 0.0001)
+    monkeypatch.setattr(litellm, "_current_cost", 100.0)
+
+    with pytest.raises(litellm.BudgetExceededError):
+        await litellm.acompletion(
+            model="gpt-4o-mini",
+            messages=[{"role": "user", "content": "hi"}],
+            mock_response="should never be reached",
+        )
+
+    assert recorder.calls == []
+
+
+@pytest.mark.asyncio
+async def test_wrapper_async_does_not_fire_failure_hook_for_post_success_error(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Regression: an error raised after the deployment call already succeeded (e.g. inside
+    async_post_call_success_deployment_hook or post_call_processing) is not a deployment
+    attempt failure and must not reach async_post_call_failure_deployment_hook."""
+
+    class ExplodingSuccessLogger(CustomLogger):
+        def __init__(self) -> None:
+            super().__init__()
+            self.failure_calls: list[Exception] = []
+
+        async def async_post_call_success_deployment_hook(self, request_data, response, call_type):
+            raise RuntimeError("boom in success hook, model call itself succeeded")
+
+        async def async_post_call_failure_deployment_hook(self, request_data, exception, call_type, fallback_depth=None):
+            self.failure_calls.append(exception)
+
+    exploding_logger = ExplodingSuccessLogger()
+    monkeypatch.setattr(litellm, "callbacks", [exploding_logger])
+
+    with pytest.raises(RuntimeError, match="boom in success hook"):
+        await litellm.acompletion(
+            model="gpt-4o-mini",
+            messages=[{"role": "user", "content": "hi"}],
+            mock_response="this call succeeds",
+        )
+
+    assert exploding_logger.failure_calls == []
+
+
+@pytest.mark.asyncio
+async def test_wrapper_async_calls_hook_override_missing_fallback_depth_param(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Regression: an override written before fallback_depth existed (this PR's own earlier
+    proof-of-fix example used exactly this 3-arg signature) must still fire, not raise a
+    TypeError on the fallback_depth keyword that gets swallowed at debug level."""
+
+    class ThreeArgLogger(CustomLogger):
+        def __init__(self) -> None:
+            super().__init__()
+            self.calls: list[tuple[dict, Exception, CallTypes | None]] = []
+
+        async def async_post_call_failure_deployment_hook(self, request_data, exception, call_type):
+            self.calls.append((request_data, exception, call_type))
+
+    three_arg_logger = ThreeArgLogger()
+    monkeypatch.setattr(litellm, "callbacks", [three_arg_logger])
+
+    with pytest.raises(litellm.AuthenticationError):
+        await litellm.acompletion(
+            model="gpt-4o-mini",
+            messages=[{"role": "user", "content": "hi"}],
+            mock_response=litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-4o-mini"),
+        )
+
+    assert len(three_arg_logger.calls) == 1
+    assert isinstance(three_arg_logger.calls[0][1], litellm.AuthenticationError)
+
+
+@pytest.mark.asyncio
+async def test_wrapper_async_failure_hook_exception_mutation_does_not_change_raised_exception(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Regression: a callback setting an attribute on the exception it receives (e.g.
+    status_code, as a real caller would read to determine the HTTP response) must not
+    change what the actual caller ends up with - the hook must not have write access to
+    the real exception about to be re-raised."""
+
+    class StatusCodeMutatingLogger(CustomLogger):
+        async def async_post_call_failure_deployment_hook(self, request_data, exception, call_type, fallback_depth=None):
+            exception.status_code = 429
+
+    monkeypatch.setattr(litellm, "callbacks", [StatusCodeMutatingLogger()])
+
+    with pytest.raises(litellm.AuthenticationError) as exc_info:
+        await litellm.acompletion(
+            model="gpt-4o-mini",
+            messages=[{"role": "user", "content": "hi"}],
+            mock_response=litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-4o-mini"),
+        )
+
+    assert exc_info.value.status_code == 401
+
+
+@pytest.mark.asyncio
+async def test_async_post_call_failure_deployment_hook_omits_attempted_targets_from_request_data(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Regression: attempted_targets is the router's own live fallback-walk bookkeeping,
+    shared by reference across every hop of a single request - unlike the rest of
+    request_data, it is not this attempt's own isolated copy. A callback calling .record()
+    on it would make the router skip a deployment it hasn't actually tried, so the
+    dispatcher must never hand it to a callback."""
+    recorder = _RecordingDeploymentFailureLogger()
+    monkeypatch.setattr(litellm, "callbacks", [recorder])
+
+    sentinel_targets = object()
+    await async_post_call_failure_deployment_hook(
+        request_data={"model": "gpt-4o-mini", "attempted_targets": sentinel_targets},
+        exception=ValueError("x"),
+        call_type="acompletion",
+    )
+
+    assert recorder.calls[0][0].get("attempted_targets") is None
+
+
+@pytest.mark.asyncio
+async def test_router_fallback_not_skipped_when_failure_hook_callback_touches_attempted_targets(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Regression: even a callback that tries to record a target on whatever it's handed as
+    attempted_targets must not affect the live Router fallback walk - the healthy fallback
+    deployment must still be reachable, not silently skipped as already-attempted.
+
+    attempted_targets is only present in kwargs starting from the second hop onward (the
+    first deployment's own failure predates the router's own fallback bookkeeping), so this
+    needs a 3-deployment chain: mid-group's failure is where the callback sees
+    attempted_targets and can prematurely mark good-group as tried. Uses per-deployment
+    mock_timeout/mock_response, not a request-level mock_response, which Router carries
+    into every hop's kwargs and would mask this test's real signal."""
+
+    class RecordingAttemptLogger(CustomLogger):
+        async def async_post_call_failure_deployment_hook(self, request_data, exception, call_type, fallback_depth=None):
+            attempted = request_data.get("attempted_targets")
+            if attempted is not None:
+                attempted.record("good-group")
+
+    monkeypatch.setattr(litellm, "callbacks", [RecordingAttemptLogger()])
+
+    def _mock_timeout_deployment(model_name: str) -> dict:
+        return {
+            "model_name": model_name,
+            "litellm_params": {
+                "model": "openai/gpt-4o-mini",
+                "api_key": "fake",
+                "mock_timeout": True,
+                "timeout": 0.001,
+                "num_retries": 0,
+            },
+        }
+
+    router = litellm.Router(
+        model_list=[
+            _mock_timeout_deployment("bad-group"),
+            _mock_timeout_deployment("mid-group"),
+            {
+                "model_name": "good-group",
+                "litellm_params": {
+                    "model": "openai/gpt-4o-mini",
+                    "api_key": "fake",
+                    "mock_response": "fallback worked",
+                    "num_retries": 0,
+                },
+            },
+        ],
+        num_retries=0,
+        fallbacks=[{"bad-group": ["mid-group", "good-group"]}],
+    )
+
+    response = await router.acompletion(
+        model="bad-group",
+        messages=[{"role": "user", "content": "hi"}],
+    )
+
+    assert response.choices[0].message.content == "fallback worked"
+
+
+@pytest.mark.asyncio
+async def test_wrapper_async_preserves_original_exception_when_hook_await_is_cancelled(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Regression: if the caller's own timeout (e.g. asyncio.wait_for) fires while the
+    failure hook is still being awaited, the real deployment exception must still reach
+    the caller - not get replaced by CancelledError/TimeoutError from the hook's own
+    await getting cancelled."""
+
+    class SlowLogger(CustomLogger):
+        async def async_post_call_failure_deployment_hook(self, request_data, exception, call_type, fallback_depth=None):
+            await asyncio.sleep(5)
+
+    monkeypatch.setattr(litellm, "callbacks", [SlowLogger()])
+
+    with pytest.raises(litellm.AuthenticationError):
+        await asyncio.wait_for(
+            litellm.acompletion(
+                model="gpt-4o-mini",
+                messages=[{"role": "user", "content": "hi"}],
+                mock_response=litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-4o-mini"),
+            ),
+            timeout=0.2,
+        )
+
+
+@pytest.mark.asyncio
+async def test_wrapper_async_failure_hook_latency_does_not_inflate_reported_duration(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Regression: a slow failure-hook callback must not inflate the duration reported to
+    async_log_failure_event - that's real observability data (e.g. latency dashboards),
+    and the hook's own runtime is not part of how long the deployment call itself took."""
+    reported_durations: list[float] = []
+
+    class SlowLoggerWithDurationCapture(CustomLogger):
+        async def async_post_call_failure_deployment_hook(self, request_data, exception, call_type, fallback_depth=None):
+            await asyncio.sleep(1)
+
+        async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
+            reported_durations.append((end_time - start_time).total_seconds())
+
+    monkeypatch.setattr(litellm, "callbacks", [SlowLoggerWithDurationCapture()])
+
+    with pytest.raises(litellm.AuthenticationError):
+        await litellm.acompletion(
+            model="gpt-4o-mini",
+            messages=[{"role": "user", "content": "hi"}],
+            mock_response=litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-4o-mini"),
+        )
+    await asyncio.sleep(0.1)
+
+    assert len(reported_durations) == 1
+    assert reported_durations[0] < 0.5
+
+
+@pytest.mark.asyncio
+async def test_wrapper_async_failure_hook_exception_snapshot_preserves_traceback(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """Regression: the exception snapshot handed to failure-hook callbacks (see
+    test_..._exception_mutation_does_not_change_raised_exception above) must still carry
+    __traceback__/__cause__/__context__, not just __dict__/args - a callback formatting or
+    inspecting the failure chain needs the real traceback, not an empty one."""
+    received: list[Exception] = []
+
+    class TracebackCapturingLogger(CustomLogger):
+        async def async_post_call_failure_deployment_hook(self, request_data, exception, call_type, fallback_depth=None):
+            received.append(exception)
+
+    monkeypatch.setattr(litellm, "callbacks", [TracebackCapturingLogger()])
+
+    with pytest.raises(litellm.AuthenticationError):
+        await litellm.acompletion(
+            model="gpt-4o-mini",
+            messages=[{"role": "user", "content": "hi"}],
+            mock_response=litellm.AuthenticationError(message="bad key", llm_provider="openai", model="gpt-4o-mini"),
+        )
+
+    assert len(received) == 1
+    assert received[0].__traceback__ is not None
+
+
+def test_snapshot_exception_for_hook_preserves_suppress_context_flag() -> None:
+    """Regression: setting __cause__ has a documented CPython side effect of implicitly
+    forcing __suppress_context__ to True, even when the real exception's own
+    __suppress_context__ is False (the common case: no `raise ... from`, just an
+    exception raised while handling another one, which chains __context__ but does not
+    suppress it). Snapshotting __cause__ before __suppress_context__ would silently flip
+    a real exception's __suppress_context__=False to True on the snapshot, hiding a
+    chained context a callback formatting it should still see."""
+    def _raise_chained_without_from() -> None:
+        try:
+            raise ValueError("inner cause")
+        except ValueError:
+            raise RuntimeError("outer error")  # no `from` clause: implicit chaining, not suppressed
+
+    with pytest.raises(RuntimeError) as exc_info:
+        _raise_chained_without_from()
+
+    e = exc_info.value
+    assert e.__suppress_context__ is False  # sanity check on the real exception itself
+    snapshot = _snapshot_exception_for_hook(e)
+    assert snapshot.__suppress_context__ is False
+    assert snapshot.__context__ is e.__context__

From 72b8b47ee8c3f35ce367f243c84e918695bed0e9 Mon Sep 17 00:00:00 2001
From: yuneng-jiang <yuneng@berri.ai>
Date: Tue, 25 Aug 2026 19:44:45 -0700
Subject: [PATCH 085/105] bump: litellm-enterprise 0.1.59 -> 0.1.60, litellm
 1.99.0 -> 1.100.0 (#38243)

---
 enterprise/pyproject.toml | 4 ++--
 pyproject.toml            | 6 +++---
 uv.lock                   | 6 +++---
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml
index ccfe7eda5e2..b7a62e52cf9 100644
--- a/enterprise/pyproject.toml
+++ b/enterprise/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.59"
+version = "0.1.60"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.59"
+version = "0.1.60"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",
diff --git a/pyproject.toml b/pyproject.toml
index fca5c7da1e2..37b00373f8a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "litellm"
-version = "1.99.0"
+version = "1.100.0"
 description = "Library to easily interface with LLM API providers"
 readme = "README.md"
 requires-python = ">=3.10, <3.15"
@@ -68,7 +68,7 @@ proxy = [
     "azure-storage-blob>=12.28.0,<13.0",
     "mcp>=1.28.1,<2.0",
     "litellm-proxy-extras==0.4.89",
-    "litellm-enterprise==0.1.59",
+    "litellm-enterprise==0.1.60",
     "RestrictedPython>=8.1,<9.0",
     "rich>=13.9.4,<14.0",
     "InquirerPy>=0.3.4,<1.0",
@@ -310,7 +310,7 @@ members = ["enterprise", "litellm-proxy-extras"]
 profile = "black"
 
 [tool.commitizen]
-version = "1.99.0"
+version = "1.100.0"
 version_files = [
     "pyproject.toml:^version",
 ]
diff --git a/uv.lock b/uv.lock
index 628483c0117..eeb1ced69e2 100644
--- a/uv.lock
+++ b/uv.lock
@@ -10,7 +10,7 @@ resolution-markers = [
 ]
 
 [options]
-exclude-newer = "2026-08-19T15:53:37.294198Z"
+exclude-newer = "2026-08-23T02:27:57.028643Z"
 exclude-newer-span = "P3D"
 
 [manifest]
@@ -4266,7 +4266,7 @@ wheels = [
 
 [[package]]
 name = "litellm"
-version = "1.99.0"
+version = "1.100.0"
 source = { editable = "." }
 dependencies = [
     { name = "aiohttp" },
@@ -4661,7 +4661,7 @@ proxy-dev = [
 
 [[package]]
 name = "litellm-enterprise"
-version = "0.1.59"
+version = "0.1.60"
 source = { editable = "enterprise" }
 
 [[package]]

From 68f5cbd52296baae115e20f7893fe5428ecf9ade Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 20:09:31 -0700
Subject: [PATCH 086/105] fix(logging): keep tracebacks for provider-originated
 4xx errors

is_expected_client_error treated every HTTP 4xx as a rejection the proxy
issued itself, so a 401 or 429 the provider returned lost its traceback in
the standard logging payload and the OTel error span dropped
litellm.provider.error.stack_trace. An exception carrying llm_provider is
an upstream or deployment problem and keeps its traceback; the proxy's own
pre-call rejections still skip it
---
 litellm/litellm_core_utils/core_helpers.py    |  8 ++++++-
 .../integrations/otel/test_otel_v2_logger.py  | 24 +++++++++++++++++++
 .../litellm_core_utils/test_core_helpers.py   | 22 +++++++++++++++++
 .../test_litellm_logging.py                   | 18 ++++++++++++++
 4 files changed, 71 insertions(+), 1 deletion(-)

diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py
index b71ef4c8e06..b669ab94350 100644
--- a/litellm/litellm_core_utils/core_helpers.py
+++ b/litellm/litellm_core_utils/core_helpers.py
@@ -60,13 +60,19 @@ def safe_divide(
 
 def is_expected_client_error(exception: BaseException | None) -> bool:
     """
-    True when the exception maps to an HTTP 4xx status.
+    True when the proxy itself rejected the request with an HTTP 4xx before any
+    provider call (bad key, budget, unknown model, guardrail). A 4xx returned by
+    a provider (the exception carries ``llm_provider``) is an upstream or
+    deployment problem, so it is never an expected client error and keeps its
+    traceback.
 
     ProxyException stores the status on .code (as a str), HTTPException and
     litellm exceptions on .status_code.
     """
     if exception is None:
         return False
+    if getattr(exception, "llm_provider", None):
+        return False
     code: Final[object] = getattr(exception, "code", None)
     status_code: Final[object] = code if code is not None else getattr(exception, "status_code", None)
     if status_code is None or isinstance(status_code, bool):
diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py
index 704a1d3a7bb..1bc02bab89a 100644
--- a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py
+++ b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py
@@ -330,6 +330,30 @@ def test_real_llm_failure_still_emitted():
     assert span.status.status_code is StatusCode.ERROR
 
 
+def test_provider_auth_failure_span_carries_stack_trace():
+    """Regression for LIT-6163: a 401 the provider returned is not an expected
+    client error, so the error span built from the real failure payload keeps
+    ``litellm.provider.error.stack_trace`` alongside code and llm_provider."""
+    from litellm.exceptions import AuthenticationError
+    from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
+
+    try:
+        raise AuthenticationError(
+            message="AnthropicException - API key is invalid.", llm_provider="anthropic", model="claude-haiku-4-5"
+        )
+    except AuthenticationError as caught:
+        error_information = StandardLoggingPayloadSetup.get_error_information(caught)
+    logger, exporter = _logger()
+    payload = _payload(status="failure", custom_llm_provider="anthropic", error_information=error_information)
+    _emit_llm(logger, _kwargs(payload=payload), fail=True)
+    (span,) = exporter.get_finished_spans()
+    assert span.status.status_code is StatusCode.ERROR
+    assert span.attributes["error.type"] == "AuthenticationError"
+    assert span.attributes["litellm.provider.error.code"] == "401"
+    assert span.attributes["litellm.provider.error.llm_provider"] == "anthropic"
+    assert "test_otel_v2_logger" in span.attributes["litellm.provider.error.stack_trace"]
+
+
 def test_idempotent_on_repeat_callback():
     """The carrier is the dedup: once the async callback closes the span and
     clears the carrier, a second callback firing emits nothing."""
diff --git a/tests/test_litellm/litellm_core_utils/test_core_helpers.py b/tests/test_litellm/litellm_core_utils/test_core_helpers.py
index 33a33711698..a3c9aadd4d9 100644
--- a/tests/test_litellm/litellm_core_utils/test_core_helpers.py
+++ b/tests/test_litellm/litellm_core_utils/test_core_helpers.py
@@ -278,3 +278,25 @@ class TestIsExpectedClientError:
         assert is_expected_client_error(WithCode("invalid_request_error")) is False
         assert is_expected_client_error(Exception("no status")) is False
         assert is_expected_client_error(None) is False
+
+    def test_provider_originated_4xx_is_not_expected(self):
+        """Regression for LIT-6163: a 4xx the provider returned is an upstream or
+        deployment problem, so it keeps its traceback; only the proxy's own
+        pre-call rejections (no llm_provider) are expected client errors."""
+        from litellm.exceptions import AuthenticationError, RateLimitError
+        from litellm.litellm_core_utils.core_helpers import is_expected_client_error
+
+        provider_auth_failure = AuthenticationError(
+            message="AnthropicException - API key is invalid.", llm_provider="anthropic", model="claude-haiku-4-5"
+        )
+        assert is_expected_client_error(provider_auth_failure) is False
+
+        provider_rate_limit = RateLimitError(message="rate limited upstream", llm_provider="openai", model="gpt-4o")
+        assert is_expected_client_error(provider_rate_limit) is False
+
+        class RouterRejection(Exception):
+            def __init__(self):
+                self.status_code = 429
+                self.llm_provider = ""
+
+        assert is_expected_client_error(RouterRejection()) is True
diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
index f93daa61570..812c91290dd 100644
--- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
+++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
@@ -5714,6 +5714,24 @@ def test_get_error_information_skips_traceback_for_expected_4xx(monkeypatch):
     assert "test_litellm_logging" in result["traceback"]
 
 
+def test_get_error_information_keeps_traceback_for_provider_4xx():
+    """Regression for LIT-6163: a 4xx the provider returned (invalid deployment
+    key, upstream validation) is an operator problem, so its traceback must
+    survive the expected-client-error gate and reach every payload consumer."""
+    from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
+
+    assert litellm.log_client_error_tracebacks is False
+    provider_exc = _raise_and_catch(
+        litellm.AuthenticationError(
+            message="AnthropicException - API key is invalid.", llm_provider="anthropic", model="claude-haiku-4-5"
+        )
+    )
+    result = StandardLoggingPayloadSetup.get_error_information(provider_exc)
+    assert result["error_code"] == "401"
+    assert result["llm_provider"] == "anthropic"
+    assert "test_litellm_logging" in result["traceback"]
+
+
 def test_failure_handler_helper_fn_builds_payload_once_per_exception():
     """Regression for LIT-6043: async and sync failure handlers both call
     _failure_handler_helper_fn for the same failed request; the standardized

From ac6deec5292a288110a688a921ac6efa18d8c7b2 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 20:25:38 -0700
Subject: [PATCH 087/105] fix(logging): keep tracebacks for unmapped provider
 exceptions too

The /v1/messages route logs the provider's raw BaseLLMException, which carries
no llm_provider, so its 4xx still counted as an expected client error and lost
its traceback. Treat BaseLLMException as provider-originated as well.
---
 litellm/litellm_core_utils/core_helpers.py    | 17 ++++++++++++----
 .../integrations/otel/test_otel_v2_logger.py  | 20 +++++++++++++++++++
 .../litellm_core_utils/test_core_helpers.py   |  4 ++++
 .../test_litellm_logging.py                   | 14 +++++++++++++
 4 files changed, 51 insertions(+), 4 deletions(-)

diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py
index b669ab94350..1089499a64f 100644
--- a/litellm/litellm_core_utils/core_helpers.py
+++ b/litellm/litellm_core_utils/core_helpers.py
@@ -58,20 +58,29 @@ def safe_divide(
     return numerator / denominator
 
 
+def _is_provider_originated(exception: BaseException) -> bool:
+    if getattr(exception, "llm_provider", None):
+        return True
+    from litellm.llms.base_llm.chat.transformation import BaseLLMException
+
+    return isinstance(exception, BaseLLMException)
+
+
 def is_expected_client_error(exception: BaseException | None) -> bool:
     """
     True when the proxy itself rejected the request with an HTTP 4xx before any
     provider call (bad key, budget, unknown model, guardrail). A 4xx returned by
-    a provider (the exception carries ``llm_provider``) is an upstream or
-    deployment problem, so it is never an expected client error and keeps its
-    traceback.
+    a provider is an upstream or deployment problem, so it is never an expected
+    client error and keeps its traceback: a mapped litellm exception carries
+    ``llm_provider``, and the raw ``BaseLLMException`` that provider handlers
+    raise before mapping (the /v1/messages route surfaces it as-is) is one too.
 
     ProxyException stores the status on .code (as a str), HTTPException and
     litellm exceptions on .status_code.
     """
     if exception is None:
         return False
-    if getattr(exception, "llm_provider", None):
+    if _is_provider_originated(exception):
         return False
     code: Final[object] = getattr(exception, "code", None)
     status_code: Final[object] = code if code is not None else getattr(exception, "status_code", None)
diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py
index 1bc02bab89a..455d84c764f 100644
--- a/tests/test_litellm/integrations/otel/test_otel_v2_logger.py
+++ b/tests/test_litellm/integrations/otel/test_otel_v2_logger.py
@@ -354,6 +354,26 @@ def test_provider_auth_failure_span_carries_stack_trace():
     assert "test_otel_v2_logger" in span.attributes["litellm.provider.error.stack_trace"]
 
 
+def test_unmapped_provider_auth_failure_span_carries_stack_trace():
+    """Regression for LIT-6163 on /v1/messages: that route logs the provider's
+    raw exception (no llm_provider), and its error span keeps the stack trace."""
+    from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
+    from litellm.llms.anthropic.common_utils import AnthropicError
+
+    try:
+        raise AnthropicError(status_code=401, message='{"type":"authentication_error","message":"API key is invalid."}')
+    except AnthropicError as caught:
+        error_information = StandardLoggingPayloadSetup.get_error_information(caught)
+    logger, exporter = _logger()
+    payload = _payload(status="failure", custom_llm_provider="anthropic", error_information=error_information)
+    _emit_llm(logger, _kwargs(payload=payload), fail=True)
+    (span,) = exporter.get_finished_spans()
+    assert span.status.status_code is StatusCode.ERROR
+    assert span.attributes["error.type"] == "AnthropicError"
+    assert span.attributes["litellm.provider.error.code"] == "401"
+    assert "test_otel_v2_logger" in span.attributes["litellm.provider.error.stack_trace"]
+
+
 def test_idempotent_on_repeat_callback():
     """The carrier is the dedup: once the async callback closes the span and
     clears the carrier, a second callback firing emits nothing."""
diff --git a/tests/test_litellm/litellm_core_utils/test_core_helpers.py b/tests/test_litellm/litellm_core_utils/test_core_helpers.py
index a3c9aadd4d9..efbf7192a0e 100644
--- a/tests/test_litellm/litellm_core_utils/test_core_helpers.py
+++ b/tests/test_litellm/litellm_core_utils/test_core_helpers.py
@@ -285,6 +285,7 @@ class TestIsExpectedClientError:
         pre-call rejections (no llm_provider) are expected client errors."""
         from litellm.exceptions import AuthenticationError, RateLimitError
         from litellm.litellm_core_utils.core_helpers import is_expected_client_error
+        from litellm.llms.anthropic.common_utils import AnthropicError
 
         provider_auth_failure = AuthenticationError(
             message="AnthropicException - API key is invalid.", llm_provider="anthropic", model="claude-haiku-4-5"
@@ -294,6 +295,9 @@ class TestIsExpectedClientError:
         provider_rate_limit = RateLimitError(message="rate limited upstream", llm_provider="openai", model="gpt-4o")
         assert is_expected_client_error(provider_rate_limit) is False
 
+        unmapped_provider_failure = AnthropicError(status_code=401, message='{"type":"authentication_error"}')
+        assert is_expected_client_error(unmapped_provider_failure) is False
+
         class RouterRejection(Exception):
             def __init__(self):
                 self.status_code = 429
diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
index 812c91290dd..5feaeff4408 100644
--- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
+++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
@@ -5732,6 +5732,20 @@ def test_get_error_information_keeps_traceback_for_provider_4xx():
     assert "test_litellm_logging" in result["traceback"]
 
 
+def test_get_error_information_keeps_traceback_for_unmapped_provider_4xx():
+    """Regression for LIT-6163 on /v1/messages: that route logs the provider's
+    raw BaseLLMException (no llm_provider), which still keeps its traceback."""
+    from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
+    from litellm.llms.anthropic.common_utils import AnthropicError
+
+    assert litellm.log_client_error_tracebacks is False
+    raw_provider_exc = _raise_and_catch(AnthropicError(status_code=401, message='{"type":"authentication_error"}'))
+    result = StandardLoggingPayloadSetup.get_error_information(raw_provider_exc)
+    assert result["error_code"] == "401"
+    assert result["error_class"] == "AnthropicError"
+    assert "test_litellm_logging" in result["traceback"]
+
+
 def test_failure_handler_helper_fn_builds_payload_once_per_exception():
     """Regression for LIT-6043: async and sync failure handlers both call
     _failure_handler_helper_fn for the same failed request; the standardized

From a6e1708e6c243a5e3434b135abb08132575cec68 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 20:30:44 -0700
Subject: [PATCH 088/105] fix(logging): keep proxy-side rate limits as expected
 client errors

ProxyRateLimitError derives from HTTPException but carries an llm_provider,
so it read as provider-originated and regained its traceback. Any
HTTPException is a proxy rejection regardless of llm_provider.
---
 litellm/litellm_core_utils/core_helpers.py           | 12 ++++++++++++
 .../litellm_core_utils/test_core_helpers.py          |  7 +++++++
 2 files changed, 19 insertions(+)

diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py
index 1089499a64f..9d5b0c86da6 100644
--- a/litellm/litellm_core_utils/core_helpers.py
+++ b/litellm/litellm_core_utils/core_helpers.py
@@ -58,7 +58,17 @@ def safe_divide(
     return numerator / denominator
 
 
+def _is_proxy_rejection(exception: BaseException) -> bool:
+    try:
+        from starlette.exceptions import HTTPException
+    except ImportError:
+        return False
+    return isinstance(exception, HTTPException)
+
+
 def _is_provider_originated(exception: BaseException) -> bool:
+    if _is_proxy_rejection(exception):
+        return False
     if getattr(exception, "llm_provider", None):
         return True
     from litellm.llms.base_llm.chat.transformation import BaseLLMException
@@ -74,6 +84,8 @@ def is_expected_client_error(exception: BaseException | None) -> bool:
     client error and keeps its traceback: a mapped litellm exception carries
     ``llm_provider``, and the raw ``BaseLLMException`` that provider handlers
     raise before mapping (the /v1/messages route surfaces it as-is) is one too.
+    The proxy's own limiters raise ``HTTPException`` subclasses that also carry
+    an ``llm_provider``, so any ``HTTPException`` stays a proxy rejection.
 
     ProxyException stores the status on .code (as a str), HTTPException and
     litellm exceptions on .status_code.
diff --git a/tests/test_litellm/litellm_core_utils/test_core_helpers.py b/tests/test_litellm/litellm_core_utils/test_core_helpers.py
index efbf7192a0e..a1f3f5eac70 100644
--- a/tests/test_litellm/litellm_core_utils/test_core_helpers.py
+++ b/tests/test_litellm/litellm_core_utils/test_core_helpers.py
@@ -286,6 +286,7 @@ class TestIsExpectedClientError:
         from litellm.exceptions import AuthenticationError, RateLimitError
         from litellm.litellm_core_utils.core_helpers import is_expected_client_error
         from litellm.llms.anthropic.common_utils import AnthropicError
+        from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
 
         provider_auth_failure = AuthenticationError(
             message="AnthropicException - API key is invalid.", llm_provider="anthropic", model="claude-haiku-4-5"
@@ -298,6 +299,12 @@ class TestIsExpectedClientError:
         unmapped_provider_failure = AnthropicError(status_code=401, message='{"type":"authentication_error"}')
         assert is_expected_client_error(unmapped_provider_failure) is False
 
+        proxy_rate_limit = ProxyRateLimitError(
+            detail={"error": "Max parallel requests reached"}, model="claude-haiku-4-5", llm_provider="anthropic"
+        )
+        assert proxy_rate_limit.llm_provider == "anthropic"
+        assert is_expected_client_error(proxy_rate_limit) is True
+
         class RouterRejection(Exception):
             def __init__(self):
                 self.status_code = 429

From 514cae1b3dc1ff72fdf5b10aa440e75b523c207d Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 20:51:55 -0700
Subject: [PATCH 089/105] fix(logging): keep the proxy's own budget rejection
 an expected client error

The auth handler stamps the requested model's provider onto BudgetExceededError before logging it, which made a key-over-budget 429 look provider-originated and regain its traceback (and an OTel stack_trace) after the provider 4xx carve-out. Any exception whose unified rate-limit category names litellm's own limiter is now a proxy rejection, matching the HTTPException rule.
---
 litellm/litellm_core_utils/core_helpers.py    | 16 ++++++++++++-
 .../litellm_core_utils/test_core_helpers.py   | 23 +++++++++++++++++++
 .../test_litellm_logging.py                   | 13 +++++++++++
 3 files changed, 51 insertions(+), 1 deletion(-)

diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py
index 9d5b0c86da6..33eef9d3ac3 100644
--- a/litellm/litellm_core_utils/core_helpers.py
+++ b/litellm/litellm_core_utils/core_helpers.py
@@ -58,7 +58,18 @@ def safe_divide(
     return numerator / denominator
 
 
+def _is_litellm_limit_rejection(exception: BaseException) -> bool:
+    from litellm.exceptions import RateLimitErrorCategory
+
+    litellm_limit_categories: Final = frozenset(
+        (RateLimitErrorCategory.LITELLM_RATE_LIMIT.value, RateLimitErrorCategory.LITELLM_BATCH_RATE_LIMIT.value)
+    )
+    return getattr(exception, "category", None) in litellm_limit_categories
+
+
 def _is_proxy_rejection(exception: BaseException) -> bool:
+    if _is_litellm_limit_rejection(exception):
+        return True
     try:
         from starlette.exceptions import HTTPException
     except ImportError:
@@ -85,7 +96,10 @@ def is_expected_client_error(exception: BaseException | None) -> bool:
     ``llm_provider``, and the raw ``BaseLLMException`` that provider handlers
     raise before mapping (the /v1/messages route surfaces it as-is) is one too.
     The proxy's own limiters raise ``HTTPException`` subclasses that also carry
-    an ``llm_provider``, so any ``HTTPException`` stays a proxy rejection.
+    an ``llm_provider``, so any ``HTTPException`` stays a proxy rejection, and
+    so does any exception whose unified rate-limit ``category`` names litellm's
+    own limiter (``BudgetExceededError`` is a plain ``Exception`` that the auth
+    handler decorates with the requested model's provider).
 
     ProxyException stores the status on .code (as a str), HTTPException and
     litellm exceptions on .status_code.
diff --git a/tests/test_litellm/litellm_core_utils/test_core_helpers.py b/tests/test_litellm/litellm_core_utils/test_core_helpers.py
index a1f3f5eac70..93cb01e1969 100644
--- a/tests/test_litellm/litellm_core_utils/test_core_helpers.py
+++ b/tests/test_litellm/litellm_core_utils/test_core_helpers.py
@@ -311,3 +311,26 @@ class TestIsExpectedClientError:
                 self.llm_provider = ""
 
         assert is_expected_client_error(RouterRejection()) is True
+
+    def test_budget_rejection_decorated_with_provider_is_expected(self):
+        """The auth handler stamps the requested model's provider onto the proxy's
+        own BudgetExceededError before logging it, which must not turn a key-over-budget
+        429 into a provider error that keeps its traceback."""
+        from litellm.exceptions import BudgetExceededError, RateLimitError, RateLimitErrorCategory
+        from litellm.litellm_core_utils.core_helpers import is_expected_client_error
+
+        over_budget = BudgetExceededError(current_cost=0.01, max_budget=0.0, llm_provider="anthropic")
+        assert over_budget.llm_provider == "anthropic"
+        assert is_expected_client_error(over_budget) is True
+
+        litellm_limit = RateLimitError(
+            message="key over rpm", llm_provider="anthropic", model="claude-haiku-4-5",
+            category=RateLimitErrorCategory.LITELLM_RATE_LIMIT,
+        )
+        assert is_expected_client_error(litellm_limit) is True
+
+        vendor_limit = RateLimitError(
+            message="rate limited upstream", llm_provider="anthropic", model="claude-haiku-4-5",
+            category=RateLimitErrorCategory.VENDOR_RATE_LIMIT,
+        )
+        assert is_expected_client_error(vendor_limit) is False
diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
index 5feaeff4408..29b283ec009 100644
--- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
+++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
@@ -5746,6 +5746,19 @@ def test_get_error_information_keeps_traceback_for_unmapped_provider_4xx():
     assert "test_litellm_logging" in result["traceback"]
 
 
+def test_get_error_information_skips_traceback_for_budget_rejection_with_provider():
+    """A key-over-budget 429 is the proxy's own rejection even after the auth
+    handler stamps the requested model's provider onto it, so it stays cheap."""
+    from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
+
+    assert litellm.log_client_error_tracebacks is False
+    over_budget = _raise_and_catch(litellm.BudgetExceededError(current_cost=0.01, max_budget=0.0, llm_provider="anthropic"))
+    result = StandardLoggingPayloadSetup.get_error_information(over_budget)
+    assert result["error_code"] == "429"
+    assert result["llm_provider"] == "anthropic"
+    assert result["traceback"] == ""
+
+
 def test_failure_handler_helper_fn_builds_payload_once_per_exception():
     """Regression for LIT-6043: async and sync failure handlers both call
     _failure_handler_helper_fn for the same failed request; the standardized

From a5b6f31585cebb717e7f62967602a48c89b60210 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Tue, 25 Aug 2026 21:24:07 -0700
Subject: [PATCH 090/105] refactor(ui): install the shadcn alert primitive

components/shared/Alert.tsx was base-vega's own alert.tsx copied in by
hand, carrying the same four exports and the same class strings, so
npx shadcn add could never reach it and it would drift from every
upstream fix silently. It also still wrapped each part in forwardRef,
which React 19 no longer needs.

Install the primitive into components/ui/ where the CLI can update it,
and reduce the shared file to a wrapper that adds the four status
variants (info, success, warning, error) the dashboard actually uses on
top of upstream's default and destructive.

Rendered output is unchanged: every variant produces byte-identical
classes, role and data-variant, so all 45 call sites look the same.
---
 ui/litellm-dashboard/eslint-suppressions.json |  5 ++
 .../src/components/shared/Alert.tsx           | 85 +++++--------------
 .../src/components/ui/alert.tsx               | 56 ++++++++++++
 3 files changed, 82 insertions(+), 64 deletions(-)
 create mode 100644 ui/litellm-dashboard/src/components/ui/alert.tsx

diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json
index a647eedc354..e22812df465 100644
--- a/ui/litellm-dashboard/eslint-suppressions.json
+++ b/ui/litellm-dashboard/eslint-suppressions.json
@@ -2133,6 +2133,11 @@
       "count": 1
     }
   },
+  "src/components/ui/alert.tsx": {
+    "local/filename-pascal-case": {
+      "count": 1
+    }
+  },
   "src/components/ui/avatar.tsx": {
     "local/filename-pascal-case": {
       "count": 1
diff --git a/ui/litellm-dashboard/src/components/shared/Alert.tsx b/ui/litellm-dashboard/src/components/shared/Alert.tsx
index 490fcba6ef5..1a895c204de 100644
--- a/ui/litellm-dashboard/src/components/shared/Alert.tsx
+++ b/ui/litellm-dashboard/src/components/shared/Alert.tsx
@@ -1,75 +1,32 @@
-import * as React from "react";
-
-import { cva, type VariantProps } from "class-variance-authority";
+import type * as React from "react";
 
+import { Alert as AlertPrimitive, AlertAction, AlertDescription, AlertTitle } from "@/components/ui/alert";
 import { cn } from "@/lib/cva.config";
 
-const alertVariants = cva(
-  "group/alert relative grid w-full gap-0.5 rounded-lg border px-4 py-3 text-left text-sm has-data-[slot=alert-action]:relative has-data-[slot=alert-action]:pr-18 has-[>svg]:grid-cols-[auto_1fr] has-[>svg]:gap-x-2.5 *:[svg]:row-span-2 *:[svg]:translate-y-0.5 *:[svg]:text-current *:[svg:not([class*='size-'])]:size-4",
-  {
-    variants: {
-      variant: {
-        default: "bg-card text-card-foreground",
-        destructive:
-          "bg-card text-destructive *:data-[slot=alert-description]:text-destructive/90 *:[svg]:text-current",
-        info: "border-info/20 bg-info/5 text-info *:[svg]:text-current",
-        success: "border-success/20 bg-success/5 text-success *:[svg]:text-current",
-        warning: "border-warning/20 bg-warning/5 text-warning *:[svg]:text-current",
-        error:
-          "border-destructive/20 bg-destructive/10 text-destructive *:data-[slot=alert-description]:text-destructive/90 *:[svg]:text-destructive",
-      },
-    },
-    defaultVariants: {
-      variant: "default",
-    },
-  },
-);
+const STATUS_VARIANT_CLASSES = {
+  info: "border-info/20 bg-info/5 text-info *:[svg]:text-current",
+  success: "border-success/20 bg-success/5 text-success *:[svg]:text-current",
+  warning: "border-warning/20 bg-warning/5 text-warning *:[svg]:text-current",
+  error:
+    "border-destructive/20 bg-destructive/10 text-destructive *:data-[slot=alert-description]:text-destructive/90 *:[svg]:text-destructive",
+} as const;
 
-type AlertProps = React.ComponentProps<"div"> & VariantProps<typeof alertVariants>;
+type StatusVariant = keyof typeof STATUS_VARIANT_CLASSES;
+type AlertVariant = NonNullable<React.ComponentProps<typeof AlertPrimitive>["variant"]> | StatusVariant;
 
-const Alert = React.forwardRef<HTMLDivElement, AlertProps>(({ className, variant = "default", ...props }, ref) => (
-  <div
-    ref={ref}
-    data-slot="alert"
+type AlertProps = Omit<React.ComponentProps<typeof AlertPrimitive>, "variant"> & {
+  variant?: AlertVariant;
+};
+
+const isStatusVariant = (variant: AlertVariant): variant is StatusVariant => variant in STATUS_VARIANT_CLASSES;
+
+const Alert = ({ variant = "default", className, ...props }: AlertProps) => (
+  <AlertPrimitive
     data-variant={variant}
-    role="alert"
-    className={cn(alertVariants({ variant }), className)}
+    variant={variant === "destructive" ? "destructive" : "default"}
+    className={cn(isStatusVariant(variant) ? STATUS_VARIANT_CLASSES[variant] : undefined, className)}
     {...props}
   />
-));
-Alert.displayName = "Alert";
-
-const AlertTitle = React.forwardRef<HTMLDivElement, React.ComponentProps<"div">>(({ className, ...props }, ref) => (
-  <div
-    ref={ref}
-    data-slot="alert-title"
-    className={cn(
-      "font-medium group-has-[>svg]/alert:col-start-2 [&_a]:underline [&_a]:underline-offset-3 [&_a]:hover:text-foreground",
-      className,
-    )}
-    {...props}
-  />
-));
-AlertTitle.displayName = "AlertTitle";
-
-const AlertDescription = React.forwardRef<HTMLDivElement, React.ComponentProps<"div">>(
-  ({ className, ...props }, ref) => (
-    <div
-      ref={ref}
-      data-slot="alert-description"
-      className={cn(
-        "text-sm text-balance text-muted-foreground md:text-pretty [&_a]:underline [&_a]:underline-offset-3 [&_a]:hover:text-foreground [&_p:not(:last-child)]:mb-4",
-        className,
-      )}
-      {...props}
-    />
-  ),
 );
-AlertDescription.displayName = "AlertDescription";
-
-const AlertAction = React.forwardRef<HTMLDivElement, React.ComponentProps<"div">>(({ className, ...props }, ref) => (
-  <div ref={ref} data-slot="alert-action" className={cn("absolute top-2.5 right-3", className)} {...props} />
-));
-AlertAction.displayName = "AlertAction";
 
 export { Alert, AlertTitle, AlertDescription, AlertAction };
diff --git a/ui/litellm-dashboard/src/components/ui/alert.tsx b/ui/litellm-dashboard/src/components/ui/alert.tsx
new file mode 100644
index 00000000000..01d86e0961a
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/ui/alert.tsx
@@ -0,0 +1,56 @@
+import * as React from "react";
+import { cva, type VariantProps } from "class-variance-authority";
+
+import { cn } from "@/lib/cva.config";
+
+const alertVariants = cva(
+  "group/alert relative grid w-full gap-0.5 rounded-lg border px-4 py-3 text-left text-sm has-data-[slot=alert-action]:relative has-data-[slot=alert-action]:pr-18 has-[>svg]:grid-cols-[auto_1fr] has-[>svg]:gap-x-2.5 *:[svg]:row-span-2 *:[svg]:translate-y-0.5 *:[svg]:text-current *:[svg:not([class*='size-'])]:size-4",
+  {
+    variants: {
+      variant: {
+        default: "bg-card text-card-foreground",
+        destructive:
+          "bg-card text-destructive *:data-[slot=alert-description]:text-destructive/90 *:[svg]:text-current",
+      },
+    },
+    defaultVariants: {
+      variant: "default",
+    },
+  },
+);
+
+function Alert({ className, variant, ...props }: React.ComponentProps<"div"> & VariantProps<typeof alertVariants>) {
+  return <div data-slot="alert" role="alert" className={cn(alertVariants({ variant }), className)} {...props} />;
+}
+
+function AlertTitle({ className, ...props }: React.ComponentProps<"div">) {
+  return (
+    <div
+      data-slot="alert-title"
+      className={cn(
+        "font-medium group-has-[>svg]/alert:col-start-2 [&_a]:underline [&_a]:underline-offset-3 [&_a]:hover:text-foreground",
+        className,
+      )}
+      {...props}
+    />
+  );
+}
+
+function AlertDescription({ className, ...props }: React.ComponentProps<"div">) {
+  return (
+    <div
+      data-slot="alert-description"
+      className={cn(
+        "text-sm text-balance text-muted-foreground md:text-pretty [&_a]:underline [&_a]:underline-offset-3 [&_a]:hover:text-foreground [&_p:not(:last-child)]:mb-4",
+        className,
+      )}
+      {...props}
+    />
+  );
+}
+
+function AlertAction({ className, ...props }: React.ComponentProps<"div">) {
+  return <div data-slot="alert-action" className={cn("absolute top-2.5 right-3", className)} {...props} />;
+}
+
+export { Alert, AlertTitle, AlertDescription, AlertAction };

From d2aea2d4e7eff9aaa720a6d6b5ec3884daf94052 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Tue, 25 Aug 2026 22:56:59 -0700
Subject: [PATCH 091/105] refactor(ui): re-pull label, textarea, separator and
 skeleton from the registry

These four primitives still wrapped their body in React.forwardRef, which
the dashboard has not needed since it moved to React 19: a function
component receives ref as an ordinary prop and the existing {...props}
spread already hands it to the DOM node.

Re-pulling each from base-vega drops the wrapper and its displayName.
These four were picked because the ref plumbing is their only divergence
from current upstream, so the class strings, data-slot values and exports
are untouched and nothing renders differently. The other seven primitives
that still carry forwardRef have also drifted on their class strings, so
re-pulling them would ship a visual change alongside the cleanup and they
are left alone here.

Textarea is the one with real ref call sites, roughly seventeen of them
through react-hook-form's field.ref, and ref-forwarding.test.tsx did not
cover it. Add that case next to the Label, Separator and Skeleton ones
already there.
---
 ui/litellm-dashboard/src/components/ui/label.tsx      | 10 ++++------
 .../src/components/ui/ref-forwarding.test.tsx         |  7 +++++++
 ui/litellm-dashboard/src/components/ui/separator.tsx  | 11 ++++-------
 ui/litellm-dashboard/src/components/ui/skeleton.tsx   | 11 +++--------
 ui/litellm-dashboard/src/components/ui/textarea.tsx   | 10 ++++------
 5 files changed, 22 insertions(+), 27 deletions(-)

diff --git a/ui/litellm-dashboard/src/components/ui/label.tsx b/ui/litellm-dashboard/src/components/ui/label.tsx
index 1ac4eed0d4e..ded2dfc1a7b 100644
--- a/ui/litellm-dashboard/src/components/ui/label.tsx
+++ b/ui/litellm-dashboard/src/components/ui/label.tsx
@@ -4,10 +4,9 @@ import * as React from "react";
 
 import { cn } from "@/lib/cva.config";
 
-const Label = React.forwardRef<HTMLLabelElement, React.ComponentPropsWithoutRef<"label">>(
-  ({ className, ...props }, ref) => (
+function Label({ className, ...props }: React.ComponentProps<"label">) {
+  return (
     <label
-      ref={ref}
       data-slot="label"
       className={cn(
         "flex items-center gap-2 text-sm leading-none font-medium select-none group-data-[disabled=true]:pointer-events-none group-data-[disabled=true]:opacity-50 peer-disabled:cursor-not-allowed peer-disabled:opacity-50",
@@ -15,8 +14,7 @@ const Label = React.forwardRef<HTMLLabelElement, React.ComponentPropsWithoutRef<
       )}
       {...props}
     />
-  ),
-);
-Label.displayName = "Label";
+  );
+}
 
 export { Label };
diff --git a/ui/litellm-dashboard/src/components/ui/ref-forwarding.test.tsx b/ui/litellm-dashboard/src/components/ui/ref-forwarding.test.tsx
index 66f7bd27138..ead00bd00ef 100644
--- a/ui/litellm-dashboard/src/components/ui/ref-forwarding.test.tsx
+++ b/ui/litellm-dashboard/src/components/ui/ref-forwarding.test.tsx
@@ -11,6 +11,7 @@ import { Label } from "./label";
 import { Separator } from "./separator";
 import { Skeleton } from "./skeleton";
 import { Table, TableBody, TableCaption, TableCell, TableFooter, TableHead, TableHeader, TableRow } from "./table";
+import { Textarea } from "./textarea";
 import { UiLoadingSpinner } from "./ui-loading-spinner";
 
 describe("ui primitives forward refs to their DOM node", () => {
@@ -50,6 +51,12 @@ describe("ui primitives forward refs to their DOM node", () => {
     expect(ref.current).toBeInstanceOf(HTMLDivElement);
   });
 
+  it("Textarea", () => {
+    const ref = React.createRef<HTMLTextAreaElement>();
+    render(<Textarea ref={ref} />);
+    expect(ref.current).toBeInstanceOf(HTMLTextAreaElement);
+  });
+
   it("UiLoadingSpinner", () => {
     const ref = React.createRef<SVGSVGElement>();
     render(<UiLoadingSpinner ref={ref} />);
diff --git a/ui/litellm-dashboard/src/components/ui/separator.tsx b/ui/litellm-dashboard/src/components/ui/separator.tsx
index a8a8d9cf5c2..443f8e905f9 100644
--- a/ui/litellm-dashboard/src/components/ui/separator.tsx
+++ b/ui/litellm-dashboard/src/components/ui/separator.tsx
@@ -1,14 +1,12 @@
 "use client";
 
 import { Separator as SeparatorPrimitive } from "@base-ui/react/separator";
-import * as React from "react";
 
 import { cn } from "@/lib/cva.config";
 
-const Separator = React.forwardRef<React.ComponentRef<typeof SeparatorPrimitive>, SeparatorPrimitive.Props>(
-  ({ className, orientation = "horizontal", ...props }, ref) => (
+function Separator({ className, orientation = "horizontal", ...props }: SeparatorPrimitive.Props) {
+  return (
     <SeparatorPrimitive
-      ref={ref}
       data-slot="separator"
       orientation={orientation}
       className={cn(
@@ -17,8 +15,7 @@ const Separator = React.forwardRef<React.ComponentRef<typeof SeparatorPrimitive>
       )}
       {...props}
     />
-  ),
-);
-Separator.displayName = "Separator";
+  );
+}
 
 export { Separator };
diff --git a/ui/litellm-dashboard/src/components/ui/skeleton.tsx b/ui/litellm-dashboard/src/components/ui/skeleton.tsx
index 1104379dde6..f1c5f5585cb 100644
--- a/ui/litellm-dashboard/src/components/ui/skeleton.tsx
+++ b/ui/litellm-dashboard/src/components/ui/skeleton.tsx
@@ -1,12 +1,7 @@
-import * as React from "react";
-
 import { cn } from "@/lib/cva.config";
 
-const Skeleton = React.forwardRef<HTMLDivElement, React.ComponentPropsWithoutRef<"div">>(
-  ({ className, ...props }, ref) => (
-    <div ref={ref} data-slot="skeleton" className={cn("animate-pulse rounded-md bg-muted", className)} {...props} />
-  ),
-);
-Skeleton.displayName = "Skeleton";
+function Skeleton({ className, ...props }: React.ComponentProps<"div">) {
+  return <div data-slot="skeleton" className={cn("animate-pulse rounded-md bg-muted", className)} {...props} />;
+}
 
 export { Skeleton };
diff --git a/ui/litellm-dashboard/src/components/ui/textarea.tsx b/ui/litellm-dashboard/src/components/ui/textarea.tsx
index debeffdaa37..591f8a0e63b 100644
--- a/ui/litellm-dashboard/src/components/ui/textarea.tsx
+++ b/ui/litellm-dashboard/src/components/ui/textarea.tsx
@@ -2,10 +2,9 @@ import * as React from "react";
 
 import { cn } from "@/lib/cva.config";
 
-const Textarea = React.forwardRef<HTMLTextAreaElement, React.ComponentProps<"textarea">>(
-  ({ className, ...props }, ref) => (
+function Textarea({ className, ...props }: React.ComponentProps<"textarea">) {
+  return (
     <textarea
-      ref={ref}
       data-slot="textarea"
       className={cn(
         "flex field-sizing-content min-h-16 w-full rounded-md border border-input bg-transparent px-2.5 py-2 text-base shadow-xs transition-[color,box-shadow] outline-none placeholder:text-muted-foreground focus-visible:border-ring focus-visible:ring-3 focus-visible:ring-ring/50 disabled:cursor-not-allowed disabled:opacity-50 aria-invalid:border-destructive aria-invalid:ring-3 aria-invalid:ring-destructive/20 md:text-sm dark:bg-input/30 dark:aria-invalid:border-destructive/50 dark:aria-invalid:ring-destructive/40",
@@ -13,8 +12,7 @@ const Textarea = React.forwardRef<HTMLTextAreaElement, React.ComponentProps<"tex
       )}
       {...props}
     />
-  ),
-);
-Textarea.displayName = "Textarea";
+  );
+}
 
 export { Textarea };

From e52f05566d902d77c5a4cf15e86e1c87ac807293 Mon Sep 17 00:00:00 2001
From: mphilippnv <mphilipp@nvidia.com>
Date: Tue, 25 Aug 2026 23:06:02 -0700
Subject: [PATCH 092/105] feat(prometheus): configure deployment caller
 identity (#38221)

* feat(prometheus): configure deployment caller identity

* test(prometheus): satisfy strict caller identity lint

* fix(prometheus): align caller identity on latency metrics

* fix(prometheus): validate caller identity mode before collectors register

Fail config load on an invalid prometheus_deployment_and_latency_caller_identity
value (including null) and on include_labels entries the selected mode removes
from a target metric, instead of booting green with an empty /metrics.
Validate the mode at the top of PrometheusLogger.__init__ so an invalid value
raises before any collector lands in the process-global registry, keeping
retries free of duplicated-timeseries errors. Label-validation errors now name
the mode setting alongside the rejected label.

---------

Co-authored-by: Mark Philipp <mphilipp622@gmail.com>
Co-authored-by: Yucheng Zhu <yucheng@berri.ai>
---
 litellm/__init__.py                           |   5 +
 litellm/integrations/prometheus.py            |  21 +
 litellm/proxy/proxy_server.py                 |   8 +
 litellm/types/integrations/prometheus.py      | 110 ++++-
 .../test_prometheus_caller_identity.py        | 411 ++++++++++++++++++
 5 files changed, 552 insertions(+), 3 deletions(-)
 create mode 100644 tests/test_litellm/integrations/test_prometheus_caller_identity.py

diff --git a/litellm/__init__.py b/litellm/__init__.py
index a520d284906..eebd2dad91e 100644
--- a/litellm/__init__.py
+++ b/litellm/__init__.py
@@ -464,6 +464,11 @@ prometheus_metrics_config: Optional[List] = None
 prometheus_exclude_metrics: Optional[List[str]] = None
 prometheus_exclude_labels: Optional[List[str]] = None
 prometheus_emit_stream_label: bool = False
+prometheus_deployment_and_latency_caller_identity: Literal[
+    "api_key_alias",
+    "user_email",
+    "both",
+] = "api_key_alias"
 # Opt-in: emit `rate_limit_category` and `rate_limit_type` labels on
 # `litellm_proxy_failed_requests_metric`. Off by default to preserve the
 # pre-unification label set so existing dashboards / recording rules keyed on
diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py
index beda1a29075..e185283d427 100644
--- a/litellm/integrations/prometheus.py
+++ b/litellm/integrations/prometheus.py
@@ -49,6 +49,7 @@ from litellm.types.integrations.prometheus import *
 from litellm.types.integrations.prometheus import (
     _sanitize_prometheus_label_name,
     _sanitize_prometheus_label_value,
+    validate_prometheus_deployment_and_latency_caller_identity,
 )
 from litellm.types.utils import (
     StandardLoggingGuardrailInformation,
@@ -175,6 +176,11 @@ class PrometheusLogger(CustomLogger):
         try:
             from prometheus_client import Counter, Gauge, Histogram
 
+            # Validate the caller-identity mode before any collector registers so an
+            # invalid value cannot leave partially-registered metrics behind in the
+            # process-global registry.
+            validate_prometheus_deployment_and_latency_caller_identity()
+
             # Always initialize label_filters, even for non-premium users
             self.label_filters = self._parse_prometheus_config()
 
@@ -2465,6 +2471,7 @@ class PrometheusLogger(CustomLogger):
         else:
             _metadata = {
                 "user_api_key_alias": getattr(_metadata_raw, "user_api_key_alias", None),
+                "user_api_key_user_email": getattr(_metadata_raw, "user_api_key_user_email", None),
                 "user_api_key_team_id": getattr(_metadata_raw, "user_api_key_team_id", None),
                 "user_api_key_team_alias": getattr(_metadata_raw, "user_api_key_team_alias", None),
                 "user_api_key_hash": getattr(_metadata_raw, "user_api_key_hash", None),
@@ -2487,6 +2494,17 @@ class PrometheusLogger(CustomLogger):
                 return getattr(user_api_key_auth, "key_alias", None)
             return None
 
+        def _get_user_email() -> str | None:
+            val = _metadata.get("user_api_key_user_email")
+            if val is not None:
+                return val
+            val = _litellm_params_metadata.get("user_api_key_user_email")
+            if val is not None:
+                return val
+            if user_api_key_auth is not None:
+                return self._safe_get(user_api_key_auth, "user_email")
+            return None
+
         def _get_team_id() -> str | None:
             val = _metadata.get("user_api_key_team_id")
             if val is not None:
@@ -2522,6 +2540,7 @@ class PrometheusLogger(CustomLogger):
 
         return {
             "api_key_alias": _get_api_key_alias(),
+            "user_email": _get_user_email(),
             "team": _get_team_id(),
             "team_alias": _get_team_alias(),
             "hashed_api_key": _get_hashed_api_key(),
@@ -2579,6 +2598,7 @@ class PrometheusLogger(CustomLogger):
             _metadata: Final = standard_logging_payload.get("metadata", {}) or {}
             hashed_api_key: Final = fallback_values.get("hashed_api_key") or _metadata.get("user_api_key_hash")
             api_key_alias: Final = fallback_values.get("api_key_alias") or _metadata.get("user_api_key_alias")
+            user_email: Final = fallback_values.get("user_email")
             team: Final = fallback_values.get("team") or _metadata.get("user_api_key_team_id")
             team_alias: Final = fallback_values.get("team_alias") or _metadata.get("user_api_key_team_alias")
             client_ip: Final = fallback_values.get("client_ip") or _metadata.get("requester_ip_address")
@@ -2619,6 +2639,7 @@ class PrometheusLogger(CustomLogger):
                 requested_model=label_requested_model,
                 hashed_api_key=hashed_api_key,
                 api_key_alias=api_key_alias,
+                user_email=user_email,
                 team=team,
                 team_alias=team_alias,
                 tags=standard_logging_payload.get("request_tags", []),
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 9ae53bebf0a..9b5d1b9bfea 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -4869,6 +4869,14 @@ class ProxyConfig:
         if litellm_settings is None:
             litellm_settings = {}
         if litellm_settings:
+            # Prometheus collectors have fixed label schemas. Load and validate this
+            # setting before processing callbacks so YAML key order cannot construct
+            # the collectors with the default caller-identity mode, and so an invalid
+            # value fails the boot instead of being swallowed by callback init.
+            from litellm.types.integrations.prometheus import validate_caller_identity_settings
+
+            validate_caller_identity_settings(litellm_settings)
+
             # ANSI escape code for blue text
             blue_color_code: Final = "\033[94m"
             reset_color_code: Final = "\033[0m"
diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py
index ebec5df55fa..f6c8a011b86 100644
--- a/litellm/types/integrations/prometheus.py
+++ b/litellm/types/integrations/prometheus.py
@@ -1,5 +1,5 @@
 import re
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
 from dataclasses import MISSING, dataclass, field, fields
 from enum import Enum
 from types import MappingProxyType
@@ -92,7 +92,20 @@ class LabelValidationError:
 
     @property
     def message(self) -> str:
-        return f"Invalid labels for metric '{self.metric_name}': {self.invalid_labels}"
+        base_message: Final = f"Invalid labels for metric '{self.metric_name}': {self.invalid_labels}"
+        if self.metric_name in PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_METRICS and any(
+            label in ("api_key_alias", "user_email") for label in self.invalid_labels
+        ):
+            mode: Final[object] = getattr(
+                litellm,
+                "prometheus_deployment_and_latency_caller_identity",
+                "api_key_alias",
+            )
+            return (
+                f"{base_message} (the caller-identity label on this metric is set by "
+                f"prometheus_deployment_and_latency_caller_identity={mode!r})"
+            )
+        return base_message
 
 
 @dataclass
@@ -276,6 +289,94 @@ DEFINED_PROMETHEUS_METRICS = Literal[
 ]
 
 
+PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_METRICS: Final[frozenset[str]] = frozenset(
+    {
+        "litellm_deployment_total_requests",
+        "litellm_deployment_success_responses",
+        "litellm_deployment_failure_responses",
+        "litellm_request_total_latency_metric",
+        "litellm_llm_api_latency_metric",
+        "litellm_llm_api_time_to_first_token_metric",
+        "litellm_request_queue_time_seconds",
+        "litellm_overhead_latency_metric",
+        "litellm_deployment_latency_per_output_token",
+    }
+)
+
+PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_VALUES: Final[tuple[str, ...]] = (
+    "api_key_alias",
+    "user_email",
+    "both",
+)
+
+
+def validate_prometheus_deployment_and_latency_caller_identity() -> str:
+    """Return the configured caller-identity mode, raising on an invalid value."""
+    caller_identity: Final[object] = getattr(
+        litellm,
+        "prometheus_deployment_and_latency_caller_identity",
+        "api_key_alias",
+    )
+    if isinstance(caller_identity, str) and caller_identity in PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_VALUES:
+        return caller_identity
+    accepted_values: Final = ", ".join(PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_VALUES)
+    raise ValueError(
+        "Invalid prometheus_deployment_and_latency_caller_identity="
+        f"{caller_identity!r}. Accepted values: {accepted_values}."
+    )
+
+
+def validate_caller_identity_settings(litellm_settings: Mapping[str, Any]) -> None:
+    """Store the caller-identity mode from litellm_settings and validate it together
+    with prometheus_metrics_config, raising on an invalid value or on include_labels
+    that request a label the selected mode removes."""
+    if "prometheus_deployment_and_latency_caller_identity" not in litellm_settings:
+        return
+    litellm.prometheus_deployment_and_latency_caller_identity = litellm_settings[
+        "prometheus_deployment_and_latency_caller_identity"
+    ]
+    caller_identity_mode: Final = validate_prometheus_deployment_and_latency_caller_identity()
+    if caller_identity_mode != "user_email":
+        return
+    conflicting_metrics: Final = tuple(
+        metric_name
+        for metric_config in (litellm_settings.get("prometheus_metrics_config") or ())
+        if isinstance(metric_config, dict) and "api_key_alias" in (metric_config.get("include_labels") or ())
+        for metric_name in (metric_config.get("metrics") or ())
+        if metric_name in PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_METRICS
+    )
+    if conflicting_metrics:
+        conflicting_names: Final = ", ".join(conflicting_metrics)
+        raise ValueError(
+            "prometheus_metrics_config include_labels contains 'api_key_alias' for "
+            f"{conflicting_names}, but prometheus_deployment_and_latency_caller_identity="
+            "'user_email' replaces that label on these metrics. Use 'user_email' in "
+            "include_labels or change the mode."
+        )
+
+
+def _resolve_deployment_and_latency_caller_identity_labels(
+    metric_name: str,
+    labels: Sequence[object],
+) -> list[str]:  # mutable-ok: every caller must receive an independently mutable label list
+    """Return a fresh label list with the configured caller identity schema."""
+    if not all(isinstance(label, str) for label in labels):
+        raise TypeError(f"Prometheus labels for {metric_name} must be strings")
+    resolved_labels: Final = [label for label in labels if isinstance(label, str)]
+    if metric_name not in PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_METRICS:
+        return resolved_labels
+
+    caller_identity: Final = validate_prometheus_deployment_and_latency_caller_identity()
+
+    alias_index: Final = resolved_labels.index(UserAPIKeyLabelNames.API_KEY_ALIAS.value)
+    if caller_identity == "user_email":
+        resolved_labels[alias_index] = UserAPIKeyLabelNames.USER_EMAIL.value
+    elif caller_identity == "both":
+        resolved_labels.insert(alias_index + 1, UserAPIKeyLabelNames.USER_EMAIL.value)
+
+    return resolved_labels
+
+
 class PrometheusMetricLabels:
     litellm_llm_api_latency_metric = [
         UserAPIKeyLabelNames.v1_LITELLM_MODEL_NAME.value,
@@ -781,7 +882,10 @@ class PrometheusMetricLabels:
 
     @staticmethod
     def get_labels(label_name: DEFINED_PROMETHEUS_METRICS) -> list[str]:
-        default_labels: Final = getattr(PrometheusMetricLabels, label_name)
+        default_labels: Final = _resolve_deployment_and_latency_caller_identity_labels(
+            metric_name=label_name,
+            labels=getattr(PrometheusMetricLabels, label_name),
+        )
         custom_labels: Final = []
 
         # Add custom metadata labels
diff --git a/tests/test_litellm/integrations/test_prometheus_caller_identity.py b/tests/test_litellm/integrations/test_prometheus_caller_identity.py
new file mode 100644
index 00000000000..30c1a2e2476
--- /dev/null
+++ b/tests/test_litellm/integrations/test_prometheus_caller_identity.py
@@ -0,0 +1,411 @@
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+from pathlib import Path
+from typing import Final, cast
+from unittest.mock import patch
+
+import pytest
+import yaml
+from prometheus_client import REGISTRY, generate_latest
+from prometheus_client.parser import text_string_to_metric_families
+
+import litellm
+from litellm.integrations.prometheus import PrometheusLogger
+from litellm.proxy._types import UserAPIKeyAuth
+from litellm.types.integrations.prometheus import (
+    DEFINED_PROMETHEUS_METRICS,
+    PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_METRICS,
+    PrometheusMetricLabels,
+    UserAPIKeyLabelNames,
+    UserAPIKeyLabelValues,
+)
+from litellm.types.utils import StandardLoggingPayload
+
+TARGET_METRICS: Final[tuple[DEFINED_PROMETHEUS_METRICS, ...]] = cast(
+    tuple[DEFINED_PROMETHEUS_METRICS, ...],
+    tuple(sorted(PROMETHEUS_DEPLOYMENT_AND_LATENCY_CALLER_IDENTITY_METRICS)),
+)
+IDENTITY_MODES: Final = ("api_key_alias", "user_email", "both")
+
+
+def _clear_prometheus_registry() -> None:
+    for collector in list(REGISTRY._collector_to_names):  # pyright: ignore[reportPrivateUsage]
+        REGISTRY.unregister(collector)
+
+
+@pytest.fixture(autouse=True)
+def reset_prometheus_settings(monkeypatch: pytest.MonkeyPatch):
+    _clear_prometheus_registry()
+    monkeypatch.setattr(litellm, "prometheus_deployment_and_latency_caller_identity", "api_key_alias")
+    monkeypatch.setattr(litellm, "prometheus_metrics_config", None)
+    monkeypatch.setattr(litellm, "prometheus_exclude_metrics", None)
+    monkeypatch.setattr(litellm, "prometheus_exclude_labels", None)
+    monkeypatch.setattr(litellm, "custom_prometheus_metadata_labels", [])
+    monkeypatch.setattr(litellm, "custom_prometheus_tags", [])
+    yield
+    _clear_prometheus_registry()
+
+
+def _expected_identity_labels(baseline: list[str], mode: str) -> list[str]:
+    expected = list(baseline)
+    alias_index = expected.index(UserAPIKeyLabelNames.API_KEY_ALIAS.value)
+    if mode == "user_email":
+        expected[alias_index] = UserAPIKeyLabelNames.USER_EMAIL.value
+    elif mode == "both":
+        expected.insert(alias_index + 1, UserAPIKeyLabelNames.USER_EMAIL.value)
+    return expected
+
+
+def _set_caller_identity(monkeypatch: pytest.MonkeyPatch, mode: str) -> None:
+    monkeypatch.setattr(litellm, "prometheus_deployment_and_latency_caller_identity", mode)
+
+
+@pytest.mark.parametrize("metric_name", TARGET_METRICS)
+@pytest.mark.parametrize("mode", IDENTITY_MODES)
+def test_target_metric_label_schema_for_each_caller_identity_mode(
+    monkeypatch: pytest.MonkeyPatch,
+    metric_name: DEFINED_PROMETHEUS_METRICS,
+    mode: str,
+):
+    _set_caller_identity(monkeypatch, "api_key_alias")
+    baseline = PrometheusMetricLabels.get_labels(metric_name)
+
+    _set_caller_identity(monkeypatch, mode)
+    actual = PrometheusMetricLabels.get_labels(metric_name)
+
+    assert actual == _expected_identity_labels(baseline, mode)
+
+
+def test_repeated_label_resolution_does_not_mutate_class_level_or_shared_lists(
+    monkeypatch: pytest.MonkeyPatch,
+):
+    total_request_labels = PrometheusMetricLabels.litellm_deployment_total_requests
+    success_labels = PrometheusMetricLabels.litellm_deployment_success_responses
+    original = tuple(total_request_labels)
+
+    assert success_labels is total_request_labels
+    for mode in (*IDENTITY_MODES, *reversed(IDENTITY_MODES)):
+        _set_caller_identity(monkeypatch, mode)
+        for metric_name in TARGET_METRICS:
+            resolved = PrometheusMetricLabels.get_labels(metric_name)
+            assert resolved is not getattr(PrometheusMetricLabels, metric_name)
+
+    assert PrometheusMetricLabels.litellm_deployment_total_requests is total_request_labels
+    assert PrometheusMetricLabels.litellm_deployment_success_responses is success_labels
+    assert success_labels is total_request_labels
+    assert tuple(total_request_labels) == original
+
+
+def test_invalid_caller_identity_mode_fails_during_prometheus_initialization(
+    monkeypatch: pytest.MonkeyPatch,
+):
+    _set_caller_identity(monkeypatch, "invalid")
+
+    with pytest.raises(
+        ValueError,
+        match="prometheus_deployment_and_latency_caller_identity",
+    ) as exc_info:
+        PrometheusLogger()
+
+    message = str(exc_info.value)
+    assert "prometheus_deployment_and_latency_caller_identity" in message
+    for accepted_value in IDENTITY_MODES:
+        assert accepted_value in message
+
+
+def test_label_resolution_rejects_non_string_class_labels(monkeypatch: pytest.MonkeyPatch):
+    monkeypatch.setattr(
+        PrometheusMetricLabels,
+        "litellm_deployment_total_requests",
+        ["api_key_alias", 1],
+    )
+
+    with pytest.raises(TypeError, match=r"Prometheus labels .* must be strings"):
+        PrometheusMetricLabels.get_labels("litellm_deployment_total_requests")
+
+
+@pytest.mark.parametrize(
+    ("mode", "include_labels", "is_valid"),
+    (
+        ("api_key_alias", ["api_key_alias"], True),
+        ("api_key_alias", ["user_email"], False),
+        ("user_email", ["user_email"], True),
+        ("user_email", ["api_key_alias"], False),
+        ("both", ["api_key_alias"], True),
+        ("both", ["user_email"], True),
+        ("both", ["api_key_alias", "user_email"], True),
+    ),
+)
+def test_include_labels_validation_matches_caller_identity_mode(
+    monkeypatch: pytest.MonkeyPatch,
+    mode: str,
+    include_labels: list[str],
+    is_valid: bool,
+):
+    _set_caller_identity(monkeypatch, mode)
+    monkeypatch.setattr(
+        litellm,
+        "prometheus_metrics_config",
+        [
+            {
+                "group": "caller_identity",
+                "metrics": ["litellm_deployment_total_requests"],
+                "include_labels": include_labels,
+            }
+        ],
+    )
+
+    if not is_valid:
+        with pytest.raises(ValueError, match="Configuration validation failed"):
+            PrometheusLogger()
+        return
+
+    logger = PrometheusLogger()
+    assert logger.get_labels_for_metric("litellm_deployment_total_requests") == include_labels
+
+
+@pytest.mark.parametrize(
+    ("mode", "exclude_labels", "remaining_identity_labels"),
+    (
+        ("api_key_alias", ["api_key_alias"], set[str]()),
+        ("api_key_alias", ["user_email"], {"api_key_alias"}),
+        ("user_email", ["user_email"], set[str]()),
+        ("user_email", ["api_key_alias"], {"user_email"}),
+        ("both", ["api_key_alias"], {"user_email"}),
+        ("both", ["user_email"], {"api_key_alias"}),
+        ("both", ["api_key_alias", "user_email"], set[str]()),
+    ),
+)
+def test_exclude_labels_can_remove_supported_identity_labels(
+    monkeypatch: pytest.MonkeyPatch,
+    mode: str,
+    exclude_labels: list[str],
+    remaining_identity_labels: set[str],
+):
+    _set_caller_identity(monkeypatch, mode)
+    monkeypatch.setattr(litellm, "prometheus_exclude_labels", exclude_labels)
+
+    logger = PrometheusLogger()
+    labels = logger.get_labels_for_metric("litellm_deployment_total_requests")
+
+    assert set(labels) & {"api_key_alias", "user_email"} == remaining_identity_labels
+
+
+@pytest.mark.parametrize("mode", IDENTITY_MODES)
+def test_non_target_metric_label_schema_is_unchanged(monkeypatch: pytest.MonkeyPatch, mode: str):
+    baseline = list(PrometheusMetricLabels.litellm_overhead_with_guardrails_latency_metric)
+    _set_caller_identity(monkeypatch, mode)
+
+    actual = PrometheusMetricLabels.get_labels("litellm_overhead_with_guardrails_latency_metric")
+
+    assert actual == baseline
+    assert "api_key_alias" in actual
+    assert "user_email" not in actual
+
+
+def _standard_logging_payload(user_email: str | None = "alice@example.com") -> StandardLoggingPayload:
+    return cast(
+        StandardLoggingPayload,
+        {
+            "api_base": "https://api.example.com",
+            "model_group": "requested-model",
+            "model_id": "deployment-id",
+            "request_tags": [],
+            "metadata": {
+                "user_api_key_hash": "hashed-key",
+                "user_api_key_alias": "alias-a",
+                "user_api_key_user_email": user_email,
+                "user_api_key_team_id": "team-id",
+                "user_api_key_team_alias": "team-alias",
+                "requester_ip_address": "192.0.2.10",
+                "user_agent": "caller-identity-test",
+            },
+            "hidden_params": {
+                "additional_headers": None,
+                "litellm_overhead_time_ms": 125,
+            },
+        },
+    )
+
+
+def _sample_labels(scrape: str, sample_name: str) -> list[dict[str, str]]:
+    return [
+        sample.labels
+        for family in text_string_to_metric_families(scrape)
+        for sample in family.samples
+        if sample.name == sample_name
+    ]
+
+
+@pytest.mark.parametrize("mode", IDENTITY_MODES)
+def test_successful_request_emits_configured_identity_on_real_counter_and_histogram_samples(
+    monkeypatch: pytest.MonkeyPatch,
+    mode: str,
+):
+    _set_caller_identity(monkeypatch, mode)
+    logger = PrometheusLogger()
+    payload = _standard_logging_payload()
+    enum_values = UserAPIKeyLabelValues(
+        end_user="end-user",
+        user="user-id",
+        user_email="alice@example.com",
+        hashed_api_key="hashed-key",
+        api_key_alias="alias-a",
+        requested_model="requested-model",
+        model_group="requested-model",
+        team="team-id",
+        team_alias="team-alias",
+        model="provider-model",
+        litellm_model_name="deployment-model",
+        model_id="deployment-id",
+        api_base="https://api.example.com",
+        api_provider="openai",
+        client_ip="192.0.2.10",
+        user_agent="caller-identity-test",
+    )
+    start_time = datetime.now()
+    api_call_start_time = start_time + timedelta(milliseconds=100)
+    completion_start_time = api_call_start_time + timedelta(milliseconds=200)
+    end_time = start_time + timedelta(seconds=1)
+    request_kwargs = {
+        "model": "deployment-model",
+        "stream": True,
+        "start_time": start_time,
+        "api_call_start_time": api_call_start_time,
+        "completion_start_time": completion_start_time,
+        "end_time": end_time,
+        "litellm_params": {
+            "custom_llm_provider": "openai",
+            "metadata": {
+                "model_info": {"id": "deployment-id"},
+                "queue_time_seconds": 0.05,
+            },
+        },
+        "standard_logging_object": payload,
+    }
+
+    logger._set_latency_metrics(  # pyright: ignore[reportPrivateUsage, reportUnknownMemberType]
+        kwargs=request_kwargs,
+        model="deployment-model",
+        user_api_key="hashed-key",
+        user_api_key_alias="alias-a",
+        user_api_team="team-id",
+        user_api_team_alias="team-alias",
+        enum_values=enum_values,
+    )
+    logger.set_llm_deployment_success_metrics(  # pyright: ignore[reportUnknownMemberType]
+        request_kwargs=request_kwargs,
+        start_time=start_time,
+        end_time=end_time,
+        enum_values=enum_values,
+        output_tokens=10,
+    )
+
+    scrape = generate_latest(REGISTRY).decode()
+    sample_names = (
+        "litellm_deployment_total_requests_total",
+        "litellm_deployment_success_responses_total",
+        "litellm_request_total_latency_metric_count",
+        "litellm_llm_api_latency_metric_count",
+        "litellm_llm_api_time_to_first_token_metric_count",
+        "litellm_request_queue_time_seconds_count",
+        "litellm_overhead_latency_metric_count",
+        "litellm_deployment_latency_per_output_token_count",
+    )
+    for sample_name in sample_names:
+        samples = _sample_labels(scrape, sample_name)
+        assert len(samples) == 1, sample_name
+        labels = samples[0]
+        if mode == "api_key_alias":
+            assert labels["api_key_alias"] == "alias-a"
+            assert "user_email" not in labels
+        elif mode == "user_email":
+            assert labels["user_email"] == "alice@example.com"
+            assert "api_key_alias" not in labels
+        else:
+            assert labels["api_key_alias"] == "alias-a"
+            assert labels["user_email"] == "alice@example.com"
+
+
+@pytest.mark.parametrize(
+    ("standard_email", "metadata_email", "auth_email", "expected_email"),
+    (
+        ("standard@example.com", "metadata@example.com", "auth@example.com", "standard@example.com"),
+        (None, "metadata@example.com", "auth@example.com", "metadata@example.com"),
+        (None, None, "auth@example.com", "auth@example.com"),
+        (None, None, None, "None"),
+    ),
+)
+def test_deployment_failure_email_fallbacks_reach_both_real_counters(
+    monkeypatch: pytest.MonkeyPatch,
+    standard_email: str | None,
+    metadata_email: str | None,
+    auth_email: str | None,
+    expected_email: str,
+):
+    _set_caller_identity(monkeypatch, "both")
+    logger = PrometheusLogger()
+    payload = _standard_logging_payload(user_email=standard_email)
+    metadata = {
+        "model_info": {"id": "deployment-id"},
+        "user_api_key_user_email": metadata_email,
+        "user_api_key_auth": UserAPIKeyAuth(user_email=auth_email),
+    }
+    request_kwargs = {
+        "model": "deployment-model",
+        "litellm_params": {
+            "custom_llm_provider": "openai",
+            "metadata": metadata,
+        },
+        "standard_logging_object": payload,
+        "exception": RuntimeError("provider failed"),
+    }
+
+    logger.set_llm_deployment_failure_metrics(request_kwargs)  # pyright: ignore[reportUnknownMemberType]
+
+    scrape = generate_latest(REGISTRY).decode()
+    for sample_name in (
+        "litellm_deployment_failure_responses_total",
+        "litellm_deployment_total_requests_total",
+    ):
+        samples = _sample_labels(scrape, sample_name)
+        assert len(samples) == 1, sample_name
+        assert samples[0]["api_key_alias"] == "alias-a"
+        assert samples[0]["user_email"] == expected_email
+
+
+@pytest.mark.asyncio
+async def test_proxy_config_loads_caller_identity_before_initializing_callbacks(tmp_path: Path):
+    from litellm.proxy.proxy_server import ProxyConfig
+
+    config_path = tmp_path / "config.yaml"
+    config_path.write_text(
+        yaml.safe_dump(
+            {
+                "model_list": [
+                    {
+                        "model_name": "test-model",
+                        "litellm_params": {"model": "openai/gpt-4", "api_key": "test-key"},
+                    }
+                ],
+                "litellm_settings": {
+                    "callbacks": ["prometheus"],
+                    "prometheus_deployment_and_latency_caller_identity": "both",
+                },
+            },
+            sort_keys=False,
+        )
+    )
+    observed_modes: list[str] = []
+
+    def capture_mode(*args: object, **kwargs: object) -> None:
+        observed_modes.append(litellm.prometheus_deployment_and_latency_caller_identity)
+
+    with patch(  # test-quality-ok: callback interception verifies schema selection before construction
+        "litellm.proxy.proxy_server.initialize_callbacks_on_proxy", side_effect=capture_mode
+    ):
+        await ProxyConfig().load_config(router=None, config_file_path=str(config_path))
+
+    assert observed_modes == ["both"]
+    assert litellm.prometheus_deployment_and_latency_caller_identity == "both"

From 666648d58c82b2d8a9d394f9eca850f5caa12ff4 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 23:31:05 -0700
Subject: [PATCH 093/105] fix(otel): map /v1/messages provider errors before
 failure logging

---
 .../exception_mapping_utils.py                | 12 ++++
 .../messages/handler.py                       | 20 ++++---
 tests/e2e/logging/test_otel_trace_e2e.py      | 58 +++++++++++++++++++
 .../test_exception_mapping_utils.py           | 27 +++++++++
 ...erimental_pass_through_messages_handler.py | 53 +++++++++++++++++
 5 files changed, 163 insertions(+), 7 deletions(-)

diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py
index 4ee726b67de..b76c97ad2de 100644
--- a/litellm/litellm_core_utils/exception_mapping_utils.py
+++ b/litellm/litellm_core_utils/exception_mapping_utils.py
@@ -2486,6 +2486,18 @@ def exception_type(
                     exception_provider=exception_provider,
                     extra_information=extra_information,
                 )
+            from litellm.llms.base_llm.chat.transformation import BaseLLMException
+
+            if custom_llm_provider and isinstance(original_exception, BaseLLMException):
+                _map_openai_like_exception(
+                    model=model,
+                    original_exception=mappable_exception,
+                    custom_llm_provider=custom_llm_provider,
+                    error_str=error_str,
+                    exception_type=exception_type,
+                    exception_provider=exception_provider,
+                    extra_information=extra_information,
+                )
         if "BadRequestError.__init__() missing 1 required positional argument: 'param'" in str(
             original_exception
         ):  # deal with edge-case invalid request error bug in openai-python sdk
diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py
index f4d24bb933c..7459d1b2da5 100644
--- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py
+++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py
@@ -12,6 +12,7 @@ from functools import partial
 from typing import Any, Final, cast
 
 import litellm
+from litellm.litellm_core_utils.exception_mapping_utils import exception_type
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.anthropic.common_utils import (
     flatten_unencrypted_web_search_results_in_anthropic_messages,
@@ -382,13 +383,18 @@ async def anthropic_messages(
     )
     ctx: Final = contextvars.copy_context()
     func_with_context: Final = partial(ctx.run, func)
-    init_response: Final = await loop.run_in_executor(None, func_with_context)
-
-    if asyncio.iscoroutine(init_response):
-        response = await init_response
-    else:
-        response = init_response
-    return response
+    try:
+        init_response: Final = await loop.run_in_executor(None, func_with_context)
+        if asyncio.iscoroutine(init_response):
+            return await init_response
+        return init_response
+    except Exception as e:  # noqa: BLE001  # the mapping boundary must see every provider-layer failure, like acompletion
+        raise exception_type(
+            model=model,
+            custom_llm_provider=custom_llm_provider,
+            original_exception=e,
+            extra_kwargs=kwargs,
+        )
 
 
 def validate_anthropic_api_metadata(metadata: dict | None = None) -> dict | None:
diff --git a/tests/e2e/logging/test_otel_trace_e2e.py b/tests/e2e/logging/test_otel_trace_e2e.py
index d7b28c170c2..52cb691e2b7 100644
--- a/tests/e2e/logging/test_otel_trace_e2e.py
+++ b/tests/e2e/logging/test_otel_trace_e2e.py
@@ -743,3 +743,61 @@ class TestOtelTraceCompleteness:
         )
         genai = next(span for span in hits[0].spans if span.operation_name == genai_span)
         _assert_error_span_contract(genai)
+
+    @pytest.mark.covers("logging.otel.failure.exports_metric", exercised_on=["messages"])
+    def test_failed_messages_error_span_attributes(
+        self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager
+    ) -> None:
+        """A failed `/v1/messages` request must carry the same error-span
+        contract as a failed `/chat/completions` request (LIT-6164). The
+        async messages entrypoint used to surface the provider handler's raw
+        BaseLLMException to the failure logger, so the model-call span came
+        out with error.type=BaseLLMException and no
+        litellm.provider.error.llm_provider attribute.
+
+        Same setup as the chat sibling: a deployment with an invalid upstream
+        API key passes proxy auth and fails at the provider with a real 401,
+        and failed requests are not billed, so no cost-write span."""
+        route = "/v1/messages"
+        _assert_otel_destination_configured(client)
+
+        model_name = f"otel-err-{unique_marker()}"
+        model_id = client.create_model(
+            model_name,
+            LiteLLMParamsBody(model="anthropic/claude-haiku-4-5", api_key=INVALID_UPSTREAM_API_KEY),
+        )
+        resources.defer(lambda: client.delete_model(model_id))
+        key = client.key_with_alias(f"otel-err-{unique_marker()}", models=[model_name])
+        resources.defer(lambda: client.delete_key(key))
+
+        deadline = time.monotonic() + client.proxy.poll_timeout
+        while True:
+            outcome = client.messages_raw(key, model_name, "trigger an upstream auth failure", max_tokens=16)
+            assert not outcome.ok, "the call must fail; the deployment's upstream key is invalid"
+            if "AnthropicException" in outcome.body or time.monotonic() >= deadline:
+                break
+            time.sleep(client.proxy.poll_interval)
+        assert "AnthropicException" in outcome.body, (
+            "never saw the mapped upstream provider failure before the deadline; either the key is "
+            "still propagating or the messages route surfaced the raw unmapped provider error - "
+            f"last outcome {outcome.status_code}: {outcome.body[:200]}"
+        )
+        assert outcome.status_code == 401, (
+            f"an upstream auth failure must map to 401, got {outcome.status_code}: {outcome.body[:200]}"
+        )
+        assert outcome.call_id is not None, "failed responses must still carry x-litellm-call-id"
+
+        genai_span = f"chat {model_name}"
+        hits = otel_reader.poll_traces_for_call(
+            call_id=outcome.call_id,
+            settled_names=_settled_names(route=route, genai_span=genai_span, require_cost_span=False),
+            settled_prefixes={DB_SPAN_PREFIX},
+        )
+        _assert_complete_trace(hits, route=route, genai_span=genai_span, require_cost_span=False)
+
+        root = next(span for span in hits[0].spans if not span.references)
+        assert str(_tag(root, "http.status_code")) == "401", (
+            f"the SERVER span must record the 401 the client received, got {_tag(root, 'http.status_code')!r}"
+        )
+        genai = next(span for span in hits[0].spans if span.operation_name == genai_span)
+        _assert_error_span_contract(genai)
diff --git a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py
index 6f7ea9da640..3eb8094f914 100644
--- a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py
+++ b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py
@@ -1092,3 +1092,30 @@ def test_bedrock_mantle_context_overflow_maps_to_context_window_exceeded():
 
     assert excinfo.value.status_code == 400
     assert "prompt is too long: 1055489 tokens > 1050000 maximum" in excinfo.value.message
+
+
+@pytest.mark.parametrize(
+    "status_code, expected_class",
+    [(401, litellm.AuthenticationError), (429, litellm.RateLimitError)],
+)
+def test_a_base_llm_exception_without_a_provider_branch_maps_by_status_code(
+    status_code, expected_class, quiet_exception_mapping
+):
+    """Regression test for LIT-6164. Native /v1/messages handlers raise raw
+    BaseLLMException, and providers without an exception_type branch (e.g.
+    minimax) must keep the upstream status instead of collapsing every failure
+    into a 500 APIConnectionError once that route maps its exceptions."""
+    from litellm.llms.base_llm.chat.transformation import BaseLLMException
+
+    original_exception = BaseLLMException(status_code=status_code, message="upstream rejected the call")
+
+    with pytest.raises(expected_class) as excinfo:
+        exception_type(
+            model="MiniMax-M2.5",
+            original_exception=original_exception,
+            custom_llm_provider="minimax",
+        )
+
+    assert excinfo.value.status_code == status_code
+    assert excinfo.value.llm_provider == "minimax"
+    assert "MinimaxException" in excinfo.value.message
diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py
index 9e58ded81bd..c88058bf215 100644
--- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py
+++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py
@@ -1286,3 +1286,56 @@ class TestMessagesStreamingSuccessLogging:
         assert payload["call_type"] == "acompletion"
         assert payload["total_tokens"] > 0
         assert payload["response_cost"] > 0
+
+
+class _FailureCapture(CustomLogger):
+    def __init__(self):
+        super().__init__()
+        self.error_information: List[Dict[str, Any]] = []
+
+    async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
+        payload = kwargs.get("standard_logging_object") or {}
+        self.error_information.append(payload.get("error_information") or {})
+
+
+@pytest.mark.asyncio
+async def test_anthropic_messages_maps_provider_exception_before_failure_logging(monkeypatch):
+    """Regression test for LIT-6164. The async /v1/messages entrypoint awaited the
+    provider handler without exception_type mapping, so the @client failure
+    handler (and every logger behind it, e.g. OTel error spans) saw the raw
+    BaseLLMException: error.type=BaseLLMException and no llm_provider."""
+    from litellm.llms.anthropic.experimental_pass_through.messages import handler
+
+    capture = _FailureCapture()
+    monkeypatch.setattr(litellm, "callbacks", [capture])
+
+    def upstream_rejects_the_key(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(
+            401,
+            json={"type": "error", "error": {"type": "authentication_error", "message": "invalid x-api-key"}},
+            request=request,
+        )
+
+    upstream = AsyncHTTPHandler()
+    upstream.client = httpx.AsyncClient(transport=httpx.MockTransport(upstream_rejects_the_key))
+
+    with pytest.raises(litellm.AuthenticationError) as excinfo:
+        await handler.anthropic_messages(
+            max_tokens=16,
+            messages=[{"role": "user", "content": "hi"}],
+            model="anthropic/claude-haiku-4-5",
+            custom_llm_provider="anthropic",
+            api_key="sk-invalid",
+            client=upstream,
+        )
+
+    assert excinfo.value.status_code == 401
+    assert excinfo.value.llm_provider == "anthropic"
+    assert "AnthropicException" in excinfo.value.message
+    assert '"authentication_error"' in excinfo.value.message
+
+    assert capture.error_information, "the failure handler must have logged the mapped exception"
+    error_information = capture.error_information[0]
+    assert error_information.get("error_class") == "AuthenticationError"
+    assert error_information.get("llm_provider") == "anthropic"
+    assert error_information.get("error_code") == "401"

From e2e16d7e2db166f6f7a6a7eec6b479f9a9093b4d Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 26 Aug 2026 06:51:33 +0000
Subject: [PATCH 094/105] fix(exceptions): map 403 to PermissionDeniedError in
 openai-like mapper

---
 litellm/litellm_core_utils/exception_mapping_utils.py    | 9 ++++++++-
 .../litellm_core_utils/test_exception_mapping_utils.py   | 8 ++++++--
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py
index b76c97ad2de..cf5e28073f5 100644
--- a/litellm/litellm_core_utils/exception_mapping_utils.py
+++ b/litellm/litellm_core_utils/exception_mapping_utils.py
@@ -755,12 +755,19 @@ def _map_openai_like_exception(
                 llm_provider=custom_llm_provider,
                 model=model,
             )
-        elif original_exception.status_code == 401 or original_exception.status_code == 403:
+        elif original_exception.status_code == 401:
             raise AuthenticationError(
                 message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
                 llm_provider=custom_llm_provider,
                 model=model,
             )
+        elif original_exception.status_code == 403:
+            raise PermissionDeniedError(
+                message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
+                llm_provider=custom_llm_provider,
+                model=model,
+                response=getattr(original_exception, "response", None),
+            )
         elif original_exception.status_code == 400:
             raise BadRequestError(
                 message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
diff --git a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py
index 3eb8094f914..4f000eb18eb 100644
--- a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py
+++ b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py
@@ -805,7 +805,7 @@ DEVIATIONS_FROM_THE_OPENAI_SHAPE = {
         503: UPSTREAM_STATUS_DISCARDED,
     },
     "databricks": {
-        403: (litellm.AuthenticationError, 401),
+        403: (litellm.PermissionDeniedError, 403),
         422: (litellm.BadRequestError, 400),
     },
     "gemini": {
@@ -1096,7 +1096,11 @@ def test_bedrock_mantle_context_overflow_maps_to_context_window_exceeded():
 
 @pytest.mark.parametrize(
     "status_code, expected_class",
-    [(401, litellm.AuthenticationError), (429, litellm.RateLimitError)],
+    [
+        (401, litellm.AuthenticationError),
+        (403, litellm.PermissionDeniedError),
+        (429, litellm.RateLimitError),
+    ],
 )
 def test_a_base_llm_exception_without_a_provider_branch_maps_by_status_code(
     status_code, expected_class, quiet_exception_mapping

From d2e4e7468503480dc7708e110a4d839ed7c68f87 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 25 Aug 2026 23:52:30 -0700
Subject: [PATCH 095/105] fix(otel): drop the generic BaseLLMException fallback
 from exception_type

The fallback mapped every unbranched provider error by status code on every route, which changed the exception class and HTTP status for those providers and failed four provider test suites in CI. The /v1/messages handler change alone covers the ticket, since the anthropic branch already maps its errors
---
 .../exception_mapping_utils.py                | 12 ---------
 .../test_exception_mapping_utils.py           | 27 -------------------
 2 files changed, 39 deletions(-)

diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py
index b76c97ad2de..4ee726b67de 100644
--- a/litellm/litellm_core_utils/exception_mapping_utils.py
+++ b/litellm/litellm_core_utils/exception_mapping_utils.py
@@ -2486,18 +2486,6 @@ def exception_type(
                     exception_provider=exception_provider,
                     extra_information=extra_information,
                 )
-            from litellm.llms.base_llm.chat.transformation import BaseLLMException
-
-            if custom_llm_provider and isinstance(original_exception, BaseLLMException):
-                _map_openai_like_exception(
-                    model=model,
-                    original_exception=mappable_exception,
-                    custom_llm_provider=custom_llm_provider,
-                    error_str=error_str,
-                    exception_type=exception_type,
-                    exception_provider=exception_provider,
-                    extra_information=extra_information,
-                )
         if "BadRequestError.__init__() missing 1 required positional argument: 'param'" in str(
             original_exception
         ):  # deal with edge-case invalid request error bug in openai-python sdk
diff --git a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py
index 3eb8094f914..6f7ea9da640 100644
--- a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py
+++ b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py
@@ -1092,30 +1092,3 @@ def test_bedrock_mantle_context_overflow_maps_to_context_window_exceeded():
 
     assert excinfo.value.status_code == 400
     assert "prompt is too long: 1055489 tokens > 1050000 maximum" in excinfo.value.message
-
-
-@pytest.mark.parametrize(
-    "status_code, expected_class",
-    [(401, litellm.AuthenticationError), (429, litellm.RateLimitError)],
-)
-def test_a_base_llm_exception_without_a_provider_branch_maps_by_status_code(
-    status_code, expected_class, quiet_exception_mapping
-):
-    """Regression test for LIT-6164. Native /v1/messages handlers raise raw
-    BaseLLMException, and providers without an exception_type branch (e.g.
-    minimax) must keep the upstream status instead of collapsing every failure
-    into a 500 APIConnectionError once that route maps its exceptions."""
-    from litellm.llms.base_llm.chat.transformation import BaseLLMException
-
-    original_exception = BaseLLMException(status_code=status_code, message="upstream rejected the call")
-
-    with pytest.raises(expected_class) as excinfo:
-        exception_type(
-            model="MiniMax-M2.5",
-            original_exception=original_exception,
-            custom_llm_provider="minimax",
-        )
-
-    assert excinfo.value.status_code == status_code
-    assert excinfo.value.llm_provider == "minimax"
-    assert "MinimaxException" in excinfo.value.message

From 3fe65029bfa6f0647d78fae20ab5c7342bb0cd3b Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Wed, 26 Aug 2026 00:09:10 -0700
Subject: [PATCH 096/105] fix(exceptions): map anthropic 403 to
 PermissionDeniedError

Now that /v1/messages routes provider failures through exception_type, an
Anthropic permission_error fell through the anthropic branch to the generic
APIConnectionError and reached the client as a 500 where the raw exception
used to answer 403. Map 403 to PermissionDeniedError so the status survives
on every route.
---
 .../exception_mapping_utils.py                |  7 ++++
 .../test_exception_mapping_utils.py           |  5 ++-
 ...erimental_pass_through_messages_handler.py | 35 +++++++++++++------
 3 files changed, 35 insertions(+), 12 deletions(-)

diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py
index 4ee726b67de..5cbc69669d7 100644
--- a/litellm/litellm_core_utils/exception_mapping_utils.py
+++ b/litellm/litellm_core_utils/exception_mapping_utils.py
@@ -550,6 +550,13 @@ def _map_anthropic_exception(
                 llm_provider="anthropic",
                 model=model,
             )
+        elif original_exception.status_code == 403:
+            raise PermissionDeniedError(
+                message=f"AnthropicException - {error_str}",
+                llm_provider="anthropic",
+                model=model,
+                response=original_exception.response,
+            )
         elif original_exception.status_code == 400 or original_exception.status_code == 413:
             raise BadRequestError(
                 message=f"AnthropicException - {error_str}",
diff --git a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py
index 6f7ea9da640..d5d5004dbe8 100644
--- a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py
+++ b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py
@@ -790,7 +790,10 @@ UPSTREAM_STATUS_DISCARDED = (litellm.APIConnectionError, 500)
 PROVIDERS_THAT_DISCARD_THE_UPSTREAM_STATUS = ("cloudflare", "ollama", "vllm")
 
 DEVIATIONS_FROM_THE_OPENAI_SHAPE = {
-    "anthropic": {403: UPSTREAM_STATUS_DISCARDED, 422: UPSTREAM_STATUS_DISCARDED},
+    "anthropic": {
+        403: (litellm.PermissionDeniedError, 403),
+        422: UPSTREAM_STATUS_DISCARDED,
+    },
     "azure": {500: (litellm.APIError, 500)},
     "bedrock": {
         403: UPSTREAM_STATUS_DISCARDED,
diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py
index c88058bf215..5c838789798 100644
--- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py
+++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py
@@ -1299,27 +1299,40 @@ class _FailureCapture(CustomLogger):
 
 
 @pytest.mark.asyncio
-async def test_anthropic_messages_maps_provider_exception_before_failure_logging(monkeypatch):
+@pytest.mark.parametrize(
+    "upstream_status, upstream_error_type, expected_exception",
+    [
+        (401, "authentication_error", litellm.AuthenticationError),
+        (403, "permission_error", litellm.PermissionDeniedError),
+    ],
+)
+async def test_anthropic_messages_maps_provider_exception_before_failure_logging(
+    monkeypatch, upstream_status, upstream_error_type, expected_exception
+):
     """Regression test for LIT-6164. The async /v1/messages entrypoint awaited the
     provider handler without exception_type mapping, so the @client failure
     handler (and every logger behind it, e.g. OTel error spans) saw the raw
-    BaseLLMException: error.type=BaseLLMException and no llm_provider."""
+    BaseLLMException: error.type=BaseLLMException and no llm_provider.
+
+    The 403 row pins the upstream status on the way through the mapper: Anthropic's
+    documented permission_error must reach the caller as a 403, never as the mapper's
+    APIConnectionError 500 fallthrough."""
     from litellm.llms.anthropic.experimental_pass_through.messages import handler
 
     capture = _FailureCapture()
     monkeypatch.setattr(litellm, "callbacks", [capture])
 
-    def upstream_rejects_the_key(request: httpx.Request) -> httpx.Response:
+    def upstream_rejects_the_request(request: httpx.Request) -> httpx.Response:
         return httpx.Response(
-            401,
-            json={"type": "error", "error": {"type": "authentication_error", "message": "invalid x-api-key"}},
+            upstream_status,
+            json={"type": "error", "error": {"type": upstream_error_type, "message": "rejected upstream"}},
             request=request,
         )
 
     upstream = AsyncHTTPHandler()
-    upstream.client = httpx.AsyncClient(transport=httpx.MockTransport(upstream_rejects_the_key))
+    upstream.client = httpx.AsyncClient(transport=httpx.MockTransport(upstream_rejects_the_request))
 
-    with pytest.raises(litellm.AuthenticationError) as excinfo:
+    with pytest.raises(expected_exception) as excinfo:
         await handler.anthropic_messages(
             max_tokens=16,
             messages=[{"role": "user", "content": "hi"}],
@@ -1329,13 +1342,13 @@ async def test_anthropic_messages_maps_provider_exception_before_failure_logging
             client=upstream,
         )
 
-    assert excinfo.value.status_code == 401
+    assert excinfo.value.status_code == upstream_status
     assert excinfo.value.llm_provider == "anthropic"
     assert "AnthropicException" in excinfo.value.message
-    assert '"authentication_error"' in excinfo.value.message
+    assert f'"{upstream_error_type}"' in excinfo.value.message
 
     assert capture.error_information, "the failure handler must have logged the mapped exception"
     error_information = capture.error_information[0]
-    assert error_information.get("error_class") == "AuthenticationError"
+    assert error_information.get("error_class") == expected_exception.__name__
     assert error_information.get("llm_provider") == "anthropic"
-    assert error_information.get("error_code") == "401"
+    assert error_information.get("error_code") == str(upstream_status)

From be9c17015661b920ad6cc9c65b977b271421a432 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Wed, 26 Aug 2026 00:14:32 -0700
Subject: [PATCH 097/105] test(e2e): let the Together replayed-reasoning case
 survive a single provider miss

---
 .../llm_translation/test_together_ai_e2e.py   | 52 ++++++++++++-------
 1 file changed, 32 insertions(+), 20 deletions(-)

diff --git a/tests/e2e/llm_translation/test_together_ai_e2e.py b/tests/e2e/llm_translation/test_together_ai_e2e.py
index 340edaed791..788b1858b73 100644
--- a/tests/e2e/llm_translation/test_together_ai_e2e.py
+++ b/tests/e2e/llm_translation/test_together_ai_e2e.py
@@ -5,14 +5,19 @@ in the proxy's own cost map that carries both capability flags. Two backends are
 pinned because the registry has no flag for what they prove: ``enable_thinking`` is a
 Qwen chat-template contract, and MiniMax-M3 is the serverless model whose template
 renders a replayed ``reasoning_content`` back into the prompt (Qwen and DeepSeek
-silently drop it). Requires TOGETHER_API_KEY on the proxy; no skip gate.
+silently drop it). MiniMax-M3 honors that replayed field on nearly every call, not
+every call (one miss in dozens of otherwise identical calls), so the replay case asks
+up to ``REPLAY_ATTEMPTS`` times and fails only when no answer carries the secret, which
+a proxy that strips the field guarantees. Requires TOGETHER_API_KEY on the proxy; no
+skip gate.
 """
 
 from __future__ import annotations
 
-from collections.abc import Mapping
+from collections.abc import Iterator, Mapping
 from dataclasses import dataclass
 from datetime import date
+from typing import Final
 
 import pytest
 from e2e_config import unique_marker
@@ -51,6 +56,7 @@ REASONING_REPLAY_BACKEND = "together_ai/MiniMaxAI/MiniMax-M3"
 SECRET_PROMPT = "Remember this for later and reply with just OK."
 SECRET_REASONING = "The user told me their favorite color is chartreuse. I must remember it."
 SECRET_QUESTION = "What is my favorite color? Answer with one word."
+REPLAY_ATTEMPTS: Final = 3
 
 ARITHMETIC_PROMPT = "What is 17 + 26? Answer with just the number."
 WEATHER_PROMPT = "What is the weather in Paris? Use the tool."
@@ -179,6 +185,18 @@ def _message(response: ChatResponse) -> OutMessage:
     return message
 
 
+def _carries_secret(answer: OutMessage) -> bool:
+    return answer.content is not None and "chartreuse" in answer.content.lower()
+
+
+def _answers_until_secret(client: PassthroughClient, key: str, body: ChatBody) -> Iterator[OutMessage]:
+    answers: Final = (_message(unwrap(client.proxy.chat(key, body))) for _ in range(REPLAY_ATTEMPTS))
+    for answer in answers:
+        yield answer
+        if _carries_secret(answer):
+            return
+
+
 def _deltas(result: StreamingResponse) -> list[_StreamDelta]:
     require_successful_call(result)
     assert result.is_streaming, f"response was not streamed: {result.headers}"
@@ -376,25 +394,19 @@ class TestTogetherChatCompletions:
         self, client: PassthroughClient, resources: ResourceManager
     ) -> None:
         model, key = _register(client, resources, REASONING_REPLAY_BACKEND)
-
-        answer = _message(
-            unwrap(
-                client.proxy.chat(
-                    key,
-                    ChatBody(
-                        model=model,
-                        messages=[
-                            ChatMessage(role="user", content=SECRET_PROMPT),
-                            ChatAssistantTurn(content="OK.", reasoning_content=SECRET_REASONING),
-                            ChatMessage(role="user", content=SECRET_QUESTION),
-                        ],
-                        max_tokens=512,
-                    ),
-                )
-            )
+        body: Final = ChatBody(
+            model=model,
+            messages=[
+                ChatMessage(role="user", content=SECRET_PROMPT),
+                ChatAssistantTurn(content="OK.", reasoning_content=SECRET_REASONING),
+                ChatMessage(role="user", content=SECRET_QUESTION),
+            ],
+            max_tokens=512,
         )
-        assert answer.content and "chartreuse" in answer.content.lower(), (
-            f"the replayed reasoning_content never reached Together: {answer}"
+
+        answers: Final = tuple(_answers_until_secret(client, key, body))
+        assert any(_carries_secret(answer) for answer in answers), (
+            f"the replayed reasoning_content never reached Together in {len(answers)} attempts: {answers}"
         )
 
     @pytest.mark.covers("llm.chat_completions.together_ai.basic.nonstream.cost_logged")

From 5cfc1608f95d6853dae7c0207ac0d5d375b32f21 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Wed, 26 Aug 2026 00:31:18 -0700
Subject: [PATCH 098/105] fix(anthropic): map only provider failures on the
 /v1/messages boundary

---
 .../messages/handler.py                       |  3 +-
 ...erimental_pass_through_messages_handler.py | 28 +++++++++++++++++++
 2 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py
index 7459d1b2da5..283c706e45e 100644
--- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py
+++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py
@@ -22,6 +22,7 @@ from litellm.llms.anthropic.common_utils import (
 from litellm.llms.base_llm.anthropic_messages.transformation import (
     BaseAnthropicMessagesConfig,
 )
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
 from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
 from litellm.types.llms.anthropic_messages.anthropic_request import AnthropicMetadata
@@ -388,7 +389,7 @@ async def anthropic_messages(
         if asyncio.iscoroutine(init_response):
             return await init_response
         return init_response
-    except Exception as e:  # noqa: BLE001  # the mapping boundary must see every provider-layer failure, like acompletion
+    except BaseLLMException as e:
         raise exception_type(
             model=model,
             custom_llm_provider=custom_llm_provider,
diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py
index 5c838789798..b690b3448ec 100644
--- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py
+++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py
@@ -7,6 +7,7 @@ from typing import Any, Dict, List
 import httpx
 import pytest
 from fastapi.testclient import TestClient
+from pydantic import ValidationError
 
 
 from unittest.mock import AsyncMock, MagicMock, patch
@@ -1352,3 +1353,30 @@ async def test_anthropic_messages_maps_provider_exception_before_failure_logging
     assert error_information.get("error_class") == expected_exception.__name__
     assert error_information.get("llm_provider") == "anthropic"
     assert error_information.get("error_code") == str(upstream_status)
+
+
+@pytest.mark.asyncio
+async def test_anthropic_messages_leaves_non_provider_failures_unmapped():
+    """The mapping boundary is for provider failures only. A request rejected before
+    the provider call (here invalid metadata) must surface as the original exception,
+    not as the mapper's APIConnectionError, whose message embeds a server traceback."""
+    from litellm.llms.anthropic.experimental_pass_through.messages import handler
+
+    def upstream_must_not_be_called(request: httpx.Request) -> httpx.Response:
+        raise AssertionError("the provider must not be called for a request rejected locally")
+
+    upstream = AsyncHTTPHandler()
+    upstream.client = httpx.AsyncClient(transport=httpx.MockTransport(upstream_must_not_be_called))
+
+    with pytest.raises(ValidationError) as excinfo:
+        await handler.anthropic_messages(
+            max_tokens=16,
+            messages=[{"role": "user", "content": "hi"}],
+            model="anthropic/claude-haiku-4-5",
+            custom_llm_provider="anthropic",
+            api_key="sk-invalid",
+            client=upstream,
+            metadata={"user_id": 123},
+        )
+
+    assert "Traceback" not in str(excinfo.value)

From 53037c34ed046cdf1c62af7dcfcf4c76161b658a Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Wed, 26 Aug 2026 00:47:28 -0700
Subject: [PATCH 099/105] fix(exceptions): map upstream status codes for
 providers with no exception_type branch

---
 .../exception_mapping_utils.py                | 131 +++++++++++++++++-
 .../test_exception_mapping_utils.py           | 124 ++++++++++++-----
 .../search/test_base_search_transformation.py |   2 +-
 .../llms/compactifai/test_compactifai.py      |   2 +-
 .../chat/test_langflow_chat_transformation.py |   2 +-
 .../test_vertex_gemma_transformation.py       |   6 +-
 6 files changed, 228 insertions(+), 39 deletions(-)

diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py
index 4ee726b67de..bfa86a3df6e 100644
--- a/litellm/litellm_core_utils/exception_mapping_utils.py
+++ b/litellm/litellm_core_utils/exception_mapping_utils.py
@@ -755,12 +755,19 @@ def _map_openai_like_exception(
                 llm_provider=custom_llm_provider,
                 model=model,
             )
-        elif original_exception.status_code == 401 or original_exception.status_code == 403:
+        elif original_exception.status_code == 401:
             raise AuthenticationError(
                 message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
                 llm_provider=custom_llm_provider,
                 model=model,
             )
+        elif original_exception.status_code == 403:
+            raise PermissionDeniedError(
+                message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
+                llm_provider=custom_llm_provider,
+                model=model,
+                response=_response_or_stub(original_exception, status_code=403),
+            )
         elif original_exception.status_code == 400:
             raise BadRequestError(
                 message=f"{custom_llm_provider.capitalize()}Exception - {original_exception.message}",
@@ -2187,6 +2194,120 @@ def _map_openrouter_exception(
         )
 
 
+def _response_or_stub(original_exception: _ProviderHTTPException, status_code: int) -> httpx.Response:
+    response: Final = original_exception.response if hasattr(original_exception, "response") else None
+    if response is not None:
+        return response
+    return httpx.Response(
+        status_code=status_code, request=httpx.Request(method="POST", url="https://docs.litellm.ai/docs")
+    )
+
+
+def _map_exception_by_status(
+    *,
+    model: str,
+    original_exception: _ProviderHTTPException,
+    custom_llm_provider: str,
+    error_str: str,
+    exception_provider: str,
+    extra_information: str,
+) -> None:
+    status_code: Final = original_exception.status_code if hasattr(original_exception, "status_code") else None
+    if not isinstance(status_code, int) or status_code < 400:
+        return
+    message: Final = f"{exception_provider} - {error_str}"
+    response: Final = original_exception.response if hasattr(original_exception, "response") else None
+    match status_code:
+        case 401:
+            raise AuthenticationError(
+                message=message,
+                llm_provider=custom_llm_provider,
+                model=model,
+                response=response,
+                litellm_debug_info=extra_information,
+            )
+        case 403:
+            raise PermissionDeniedError(
+                message=message,
+                llm_provider=custom_llm_provider,
+                model=model,
+                response=_response_or_stub(original_exception, status_code=status_code),
+                litellm_debug_info=extra_information,
+            )
+        case 404:
+            raise NotFoundError(
+                message=message,
+                model=model,
+                llm_provider=custom_llm_provider,
+                response=response,
+                litellm_debug_info=extra_information,
+            )
+        case 408:
+            raise Timeout(
+                message=message,
+                model=model,
+                llm_provider=custom_llm_provider,
+                litellm_debug_info=extra_information,
+            )
+        case 429:
+            raise RateLimitError(
+                message=message,
+                model=model,
+                llm_provider=custom_llm_provider,
+                response=response,
+                litellm_debug_info=extra_information,
+            )
+        case 500:
+            raise InternalServerError(
+                message=message,
+                llm_provider=custom_llm_provider,
+                model=model,
+                response=response,
+                litellm_debug_info=extra_information,
+            )
+        case 502:
+            raise BadGatewayError(
+                message=message,
+                llm_provider=custom_llm_provider,
+                model=model,
+                response=response,
+                litellm_debug_info=extra_information,
+            )
+        case 503:
+            raise ServiceUnavailableError(
+                message=message,
+                llm_provider=custom_llm_provider,
+                model=model,
+                response=response,
+                litellm_debug_info=extra_information,
+            )
+        case 504:
+            raise Timeout(
+                message=message,
+                model=model,
+                llm_provider=custom_llm_provider,
+                litellm_debug_info=extra_information,
+                exception_status_code=status_code,
+            )
+        case _ if status_code < 500:
+            raise BadRequestError(
+                message=message,
+                model=model,
+                llm_provider=custom_llm_provider,
+                response=response,
+                litellm_debug_info=extra_information,
+            )
+        case _:
+            raise APIError(
+                status_code=status_code,
+                message=message,
+                llm_provider=custom_llm_provider,
+                model=model,
+                request=original_exception.request if hasattr(original_exception, "request") else None,
+                litellm_debug_info=extra_information,
+            )
+
+
 def exception_type(
     model,
     original_exception,
@@ -2501,6 +2622,14 @@ def exception_type(
             For unmapped exceptions - raise the exception with traceback - https://github.com/BerriAI/litellm/issues/4201
             """
             exception_mapping_worked = True
+            _map_exception_by_status(
+                model=model,
+                original_exception=mappable_exception,
+                custom_llm_provider=custom_llm_provider,
+                error_str=error_str,
+                exception_provider=exception_provider,
+                extra_information=extra_information,
+            )
             if hasattr(original_exception, "request"):
                 raise APIConnectionError(
                     message=f"{exception_provider} - {error_str}",
diff --git a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py
index 6f7ea9da640..895044c8ad5 100644
--- a/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py
+++ b/tests/test_litellm/litellm_core_utils/test_exception_mapping_utils.py
@@ -13,6 +13,7 @@ from litellm.litellm_core_utils.exception_mapping_utils import (
     extract_and_raise_litellm_exception,
 )
 from litellm.llms.openai.common_utils import OpenAIError
+from litellm.types.utils import LlmProviders
 
 # Test cases for is_error_str_context_window_exceeded
 # Tuple format: (error_message, expected_result)
@@ -785,33 +786,24 @@ OPENAI_SHAPED = {
     503: (litellm.ServiceUnavailableError, 503),
 }
 
-UPSTREAM_STATUS_DISCARDED = (litellm.APIConnectionError, 500)
+PERMISSION_DENIED = (litellm.PermissionDeniedError, 403)
 
-PROVIDERS_THAT_DISCARD_THE_UPSTREAM_STATUS = ("cloudflare", "ollama", "vllm")
+STATUS_KEYED = {**OPENAI_SHAPED, 403: PERMISSION_DENIED}
 
 DEVIATIONS_FROM_THE_OPENAI_SHAPE = {
-    "anthropic": {403: UPSTREAM_STATUS_DISCARDED, 422: UPSTREAM_STATUS_DISCARDED},
+    "anthropic": {403: PERMISSION_DENIED},
     "azure": {500: (litellm.APIError, 500)},
     "bedrock": {
-        403: UPSTREAM_STATUS_DISCARDED,
+        403: PERMISSION_DENIED,
         500: (litellm.ServiceUnavailableError, 503),
     },
-    "cohere": {
-        401: UPSTREAM_STATUS_DISCARDED,
-        403: UPSTREAM_STATUS_DISCARDED,
-        404: UPSTREAM_STATUS_DISCARDED,
-        422: UPSTREAM_STATUS_DISCARDED,
-        429: UPSTREAM_STATUS_DISCARDED,
-        503: UPSTREAM_STATUS_DISCARDED,
-    },
+    "cloudflare": {403: PERMISSION_DENIED},
+    "cohere": {403: PERMISSION_DENIED},
     "databricks": {
-        403: (litellm.AuthenticationError, 401),
+        403: PERMISSION_DENIED,
         422: (litellm.BadRequestError, 400),
     },
-    "gemini": {
-        403: (litellm.PermissionDeniedError, 403),
-        422: UPSTREAM_STATUS_DISCARDED,
-    },
+    "gemini": {403: PERMISSION_DENIED},
     "huggingface": {
         404: (litellm.APIError, 404),
         422: (litellm.APIError, 422),
@@ -824,6 +816,7 @@ DEVIATIONS_FROM_THE_OPENAI_SHAPE = {
         500: (litellm.APIError, 500),
         503: (litellm.APIError, 503),
     },
+    "ollama": {403: PERMISSION_DENIED},
     "openrouter": {500: (litellm.APIError, 500)},
     "replicate": {
         403: (litellm.APIError, 500),
@@ -833,17 +826,11 @@ DEVIATIONS_FROM_THE_OPENAI_SHAPE = {
         503: (litellm.APIError, 500),
     },
     "sagemaker": {
-        403: UPSTREAM_STATUS_DISCARDED,
+        403: PERMISSION_DENIED,
         500: (litellm.ServiceUnavailableError, 503),
     },
-    "vertex_ai": {
-        403: (litellm.PermissionDeniedError, 403),
-        422: UPSTREAM_STATUS_DISCARDED,
-    },
-    **{
-        provider: dict.fromkeys(UPSTREAM_STATUS_CODES, UPSTREAM_STATUS_DISCARDED)
-        for provider in PROVIDERS_THAT_DISCARD_THE_UPSTREAM_STATUS
-    },
+    "vertex_ai": {403: PERMISSION_DENIED},
+    "vllm": {403: PERMISSION_DENIED},
 }
 
 PROVIDERS_WITH_A_HANDLER = (
@@ -875,6 +862,38 @@ PROVIDERS_WITH_A_HANDLER = (
     "xai",
 )
 
+PROVIDER_ALIASES_WITH_A_HANDLER = (
+    "aleph_alpha",
+    "anthropic_text",
+    "azure_text",
+    "bedrock_mantle",
+    "cohere_chat",
+    "custom_openai",
+    "lemonade",
+    "litellm_proxy",
+    "ollama_chat",
+    "predibase",
+    "sagemaker_chat",
+    "text-completion-openai",
+    "vertex_ai_beta",
+    "watsonx",
+)
+
+PROVIDERS_WITHOUT_A_HANDLER = tuple(
+    sorted(
+        frozenset(provider.value for provider in LlmProviders)
+        - frozenset(PROVIDERS_WITH_A_HANDLER)
+        - frozenset(PROVIDER_ALIASES_WITH_A_HANDLER)
+        - frozenset(litellm.openai_compatible_providers)
+    )
+)
+
+MINIMAX_401_BODY = (
+    '{"type":"error","error":{"type":"authorized_error","message":"login fail: Please carry the API secret key '
+    "in the 'Authorization' field of the request header (1004)\",\"http_code\":\"401\"},"
+    '"request_id":"06ddc9ba97ee6340e38f10e09787f547"}'
+)
+
 
 def _expected_for(provider: str, status_code: int) -> tuple[type[Exception], int]:
     return DEVIATIONS_FROM_THE_OPENAI_SHAPE.get(provider, {}).get(
@@ -938,6 +957,51 @@ def test_an_already_mapped_litellm_exception_passes_through_untouched(
     assert returned is already_mapped
 
 
+@pytest.mark.parametrize("status_code", UPSTREAM_STATUS_CODES)
+@pytest.mark.parametrize("provider", PROVIDERS_WITHOUT_A_HANDLER)
+def test_a_provider_without_a_handler_maps_by_the_upstream_status(
+    provider, status_code, quiet_exception_mapping
+):
+    expected_class, expected_status = STATUS_KEYED[status_code]
+
+    with pytest.raises(openai.APIError) as raised:
+        exception_type(
+            model="test-model",
+            original_exception=_UpstreamHTTPError(status_code=status_code),
+            custom_llm_provider=provider,
+        )
+
+    assert type(raised.value) is expected_class
+    assert raised.value.status_code == expected_status
+    assert raised.value.llm_provider == provider
+    assert raised.value.model == "test-model"
+
+
+def test_a_minimax_bad_key_is_an_authentication_error(quiet_exception_mapping):
+    from litellm.llms.base_llm.chat.transformation import BaseLLMException
+
+    with pytest.raises(litellm.AuthenticationError) as raised:
+        exception_type(
+            model="MiniMax-M2.5",
+            original_exception=BaseLLMException(status_code=401, message=MINIMAX_401_BODY),
+            custom_llm_provider="minimax",
+        )
+
+    assert raised.value.status_code == 401
+    assert raised.value.llm_provider == "minimax"
+    assert raised.value.message.startswith("litellm.AuthenticationError: MinimaxException - ")
+    assert "login fail" in raised.value.message
+
+
+def test_an_exception_without_a_status_is_still_a_connection_error(quiet_exception_mapping):
+    with pytest.raises(litellm.APIConnectionError):
+        exception_type(
+            model="MiniMax-M2.5",
+            original_exception=RuntimeError("socket hung up"),
+            custom_llm_provider="minimax",
+        )
+
+
 CONTEXT_WINDOW_MESSAGE = "This model's maximum context length is 4096 tokens."
 CONTENT_POLICY_MESSAGE = (
     '{"error": {"type": "invalid_request_error", "code": "content_policy_violation"}}'
@@ -993,9 +1057,7 @@ class _UpstreamErrorWithMessage(_UpstreamHTTPError):
 def test_a_full_context_window_reaches_the_caller_as_the_router_needs_it(
     provider, quiet_exception_mapping
 ):
-    if provider in PROVIDERS_THAT_DISCARD_THE_UPSTREAM_STATUS:
-        expected_class, expected_status = UPSTREAM_STATUS_DISCARDED
-    elif provider in PROVIDERS_THAT_RECOGNISE_A_FULL_CONTEXT_WINDOW:
+    if provider in PROVIDERS_THAT_RECOGNISE_A_FULL_CONTEXT_WINDOW:
         expected_class, expected_status = litellm.ContextWindowExceededError, 400
     else:
         expected_class, expected_status = litellm.BadRequestError, 400
@@ -1015,9 +1077,7 @@ def test_a_full_context_window_reaches_the_caller_as_the_router_needs_it(
 def test_a_content_policy_block_reaches_the_caller_as_the_router_needs_it(
     provider, quiet_exception_mapping
 ):
-    if provider in PROVIDERS_THAT_DISCARD_THE_UPSTREAM_STATUS:
-        expected_class, expected_status = UPSTREAM_STATUS_DISCARDED
-    elif provider in PROVIDERS_THAT_RECOGNISE_A_CONTENT_POLICY_BLOCK:
+    if provider in PROVIDERS_THAT_RECOGNISE_A_CONTENT_POLICY_BLOCK:
         expected_class, expected_status = litellm.ContentPolicyViolationError, 400
     else:
         expected_class, expected_status = litellm.BadRequestError, 400
diff --git a/tests/test_litellm/llms/base_llm/search/test_base_search_transformation.py b/tests/test_litellm/llms/base_llm/search/test_base_search_transformation.py
index e6aad7688d1..35a54332f66 100644
--- a/tests/test_litellm/llms/base_llm/search/test_base_search_transformation.py
+++ b/tests/test_litellm/llms/base_llm/search/test_base_search_transformation.py
@@ -322,7 +322,7 @@ async def test_query_param_key_not_leaked_with_dummy_caller_key(
         "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.get",
         fake_get,
     ):
-        with pytest.raises(litellm.APIConnectionError):
+        with pytest.raises(litellm.InternalServerError):
             await litellm.asearch(
                 query="secrets",
                 search_provider=provider,
diff --git a/tests/test_litellm/llms/compactifai/test_compactifai.py b/tests/test_litellm/llms/compactifai/test_compactifai.py
index fef0baf2884..fd31049731a 100644
--- a/tests/test_litellm/llms/compactifai/test_compactifai.py
+++ b/tests/test_litellm/llms/compactifai/test_compactifai.py
@@ -172,7 +172,7 @@ def test_compactifai_authentication_error(respx_mock):
         json=mock_error, status_code=401
     )
 
-    with pytest.raises(litellm.APIConnectionError) as exc_info:
+    with pytest.raises(litellm.AuthenticationError) as exc_info:
         litellm.completion(
             model="compactifai/cai-llama-3-1-8b-slim",
             messages=[{"role": "user", "content": "test"}],
diff --git a/tests/test_litellm/llms/langflow/chat/test_langflow_chat_transformation.py b/tests/test_litellm/llms/langflow/chat/test_langflow_chat_transformation.py
index 0c241add77b..383a7afbe93 100644
--- a/tests/test_litellm/llms/langflow/chat/test_langflow_chat_transformation.py
+++ b/tests/test_litellm/llms/langflow/chat/test_langflow_chat_transformation.py
@@ -233,7 +233,7 @@ def test_langflow_extra_body_cannot_inject_tweaks_into_run_payload():
         return resp
 
     with patch.object(HTTPHandler, "post", side_effect=fake_post):
-        with pytest.raises(litellm.APIConnectionError):
+        with pytest.raises(litellm.BadRequestError):
             litellm.completion(
                 model="langflow/my-flow",
                 messages=[{"role": "user", "content": "hello"}],
diff --git a/tests/test_litellm/llms/vertex_ai/vertex_gemma_models/test_vertex_gemma_transformation.py b/tests/test_litellm/llms/vertex_ai/vertex_gemma_models/test_vertex_gemma_transformation.py
index 39af9f08540..5f74f0f602f 100644
--- a/tests/test_litellm/llms/vertex_ai/vertex_gemma_models/test_vertex_gemma_transformation.py
+++ b/tests/test_litellm/llms/vertex_ai/vertex_gemma_models/test_vertex_gemma_transformation.py
@@ -238,7 +238,7 @@ class TestVertexGemmaCompletion:
 
         Expected: Proper error handling when 'predictions' field is missing
         """
-        from litellm.exceptions import APIConnectionError
+        from litellm.exceptions import BadRequestError
 
         # Invalid response without predictions field
         invalid_response = {
@@ -260,8 +260,8 @@ class TestVertexGemmaCompletion:
             mock_client.post = AsyncMock(return_value=mock_response)
             mock_get_client.return_value = mock_client
 
-            # Should raise exception (wrapped as APIConnectionError by LiteLLM)
-            with pytest.raises(APIConnectionError) as exc_info:
+            # Should raise exception (wrapped as BadRequestError by LiteLLM)
+            with pytest.raises(BadRequestError) as exc_info:
                 await litellm.acompletion(
                     model="vertex_ai/gemma/gemma-3-12b-it",
                     messages=[{"role": "user", "content": "Test"}],

From e0c101b4da83171f9cf27c526c44a81d622c34cf Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Wed, 26 Aug 2026 01:05:15 -0700
Subject: [PATCH 100/105] fix(passthrough): record ownership of streamed
 responses under managed ids

---
 litellm/proxy/common_utils/sse_keepalive.py   |  12 ++
 .../managed_id_rewriter.py                    | 124 +++++++++++++++++-
 .../pass_through_endpoints.py                 |  72 +++++++---
 .../streaming_handler.py                      |  14 +-
 .../test_managed_id_rewriter.py               | 111 +++++++++++++++-
 .../test_pass_through_endpoints.py            |  80 +++++++++++
 6 files changed, 379 insertions(+), 34 deletions(-)

diff --git a/litellm/proxy/common_utils/sse_keepalive.py b/litellm/proxy/common_utils/sse_keepalive.py
index 429fff99ae6..e3cebf9f6c6 100644
--- a/litellm/proxy/common_utils/sse_keepalive.py
+++ b/litellm/proxy/common_utils/sse_keepalive.py
@@ -91,6 +91,18 @@ def is_sse_content_type(content_type: str | None) -> bool:
     return content_type is not None and content_type.split(";", 1)[0].strip().lower() == _SSE_MEDIA_TYPE
 
 
+def split_complete_sse_frames(pending: bytes) -> tuple[bytes, bytes]:
+    """Split buffered SSE bytes into ``(complete_frames, unterminated_tail)``."""
+    lf_boundary_end: Final = pending.rfind(b"\n\n") + 2
+    crlf_boundary_end: Final = pending.rfind(b"\r\n\r\n") + 4
+    boundary_end: Final = max(
+        lf_boundary_end if lf_boundary_end >= 2 else 0, crlf_boundary_end if crlf_boundary_end >= 4 else 0
+    )
+    if boundary_end == 0:
+        return b"", pending
+    return pending[:boundary_end], pending[boundary_end:]
+
+
 def wrap_passthrough_sse_bytes_with_keepalive_pings(
     stream: AsyncGenerator[bytes, None],
     ping_interval_seconds: float | str | None,
diff --git a/litellm/proxy/pass_through_endpoints/managed_id_rewriter.py b/litellm/proxy/pass_through_endpoints/managed_id_rewriter.py
index 4de6ef04d76..bdc4c515803 100644
--- a/litellm/proxy/pass_through_endpoints/managed_id_rewriter.py
+++ b/litellm/proxy/pass_through_endpoints/managed_id_rewriter.py
@@ -32,7 +32,7 @@ from __future__ import annotations
 
 import json
 import re
-from collections.abc import Callable, Mapping, Sequence
+from collections.abc import AsyncGenerator, Callable, Mapping, Sequence
 from typing import (
     TYPE_CHECKING,
     Final,
@@ -43,7 +43,7 @@ from typing import (
 from urllib.parse import quote, unquote
 
 from fastapi import HTTPException
-from pydantic import JsonValue
+from pydantic import JsonValue, TypeAdapter, ValidationError
 
 from litellm._logging import verbose_proxy_logger
 from litellm.llms.base_llm.managed_resources.isolation import (
@@ -52,6 +52,7 @@ from litellm.llms.base_llm.managed_resources.isolation import (
 )
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.batches_endpoints.common_utils import validate_batch_list_limit
+from litellm.proxy.common_utils.sse_keepalive import split_complete_sse_frames
 from litellm.repositories.table_repositories import (
     ManagedFileRepository,
     ManagedObjectRepository,
@@ -820,6 +821,125 @@ async def rewrite_response_ids(
     return mutated if changed else body
 
 
+# ---------------------------------------------------------------------------
+# OUTPUT path — streamed Responses API bodies
+# ---------------------------------------------------------------------------
+
+_RESPONSE_ID_PREFIX: Final = "resp_"
+_STREAMED_RESPONSE_ID_SPEC: Final[_FieldSpec] = ("id", _RESPONSE_ID_PREFIX)
+_SSE_DATA_PREFIX: Final = "data:"
+_SSE_EVENT_ADAPTER: Final = TypeAdapter(Mapping[str, JsonValue])
+
+
+def _first_streamed_response(frames: bytes) -> tuple[str, Mapping[str, JsonValue]] | None:
+    for line in frames.decode("utf-8", errors="replace").splitlines():
+        if not line.startswith(_SSE_DATA_PREFIX):
+            continue
+        try:
+            event = _SSE_EVENT_ADAPTER.validate_json(line[len(_SSE_DATA_PREFIX) :])
+        except ValidationError:
+            continue
+        response = event.get("response")
+        if not isinstance(response, dict):
+            continue
+        raw_id = response.get("id")
+        if isinstance(raw_id, str) and raw_id.startswith(_RESPONSE_ID_PREFIX):
+            return raw_id, response
+    return None
+
+
+class _StreamedResponseIdRewriter:
+    __slots__ = ("_is_create_route", "_pending", "_prisma_client", "_provider", "_replacement", "_user_api_key_dict")
+
+    def __init__(
+        self,
+        provider: str,
+        user_api_key_dict: UserAPIKeyAuth,
+        prisma_client: PrismaClient,
+        is_create_route: bool,
+    ) -> None:
+        self._provider: Final = provider
+        self._user_api_key_dict: Final = user_api_key_dict
+        self._prisma_client: Final = prisma_client
+        self._is_create_route: Final = is_create_route
+        self._pending = b""
+        self._replacement: tuple[bytes, bytes] | None = None
+
+    async def feed(self, chunk: bytes) -> bytes:
+        complete_frames, self._pending = split_complete_sse_frames(self._pending + chunk)
+        if not complete_frames:
+            return b""
+        if self._replacement is None:
+            self._replacement = await self._mint(complete_frames)
+        return self._rewrite(complete_frames)
+
+    def flush(self) -> bytes:
+        tail: Final = self._pending
+        self._pending = b""
+        return self._rewrite(tail)
+
+    async def _mint(self, frames: bytes) -> tuple[bytes, bytes] | None:
+        first: Final = _first_streamed_response(frames)
+        if first is None:
+            return None
+        raw_id, snapshot = first
+        managed_id: Final = await _mint_or_reuse_object(
+            raw_id,
+            self._provider,
+            "response",
+            snapshot,
+            self._user_api_key_dict,
+            self._prisma_client,
+            self._is_create_route,
+        )
+        return raw_id.encode(), managed_id.encode()
+
+    def _rewrite(self, frames: bytes) -> bytes:
+        if self._replacement is None:
+            return frames
+        raw_id, managed_id = self._replacement
+        return frames.replace(raw_id, managed_id)
+
+
+async def rewrite_streamed_response_ids(
+    stream: AsyncGenerator[bytes, None],
+    provider: str,
+    method: str,
+    route: str,
+    user_api_key_dict: UserAPIKeyAuth,
+    prisma_client: PrismaClient,
+) -> AsyncGenerator[bytes, None]:
+    """
+    Record ownership of the response object streamed back by a Responses API
+    passthrough and swap its managed id into every SSE frame, so a streamed
+    response is owned and resolved exactly like a non-streamed one.
+
+    Streams for any other ``(provider, method, route)`` are relayed untouched.
+    """
+    from litellm.proxy.auth.auth_utils import normalize_request_route
+
+    canonical: Final = normalize_request_route(_canonical_path(route))
+    field_specs: Final = BUILTIN_OUTPUT_ID_FIELD_MAP.get((provider, method, canonical), ())
+    if _STREAMED_RESPONSE_ID_SPEC not in field_specs:
+        async for chunk in stream:
+            yield chunk
+        return
+
+    rewriter: Final = _StreamedResponseIdRewriter(
+        provider=provider,
+        user_api_key_dict=user_api_key_dict,
+        prisma_client=prisma_client,
+        is_create_route="{" not in canonical,
+    )
+    async for chunk in stream:
+        rewritten_frames = await rewriter.feed(chunk)
+        if rewritten_frames:
+            yield rewritten_frames
+    tail: Final = rewriter.flush()
+    if tail:
+        yield tail
+
+
 # ---------------------------------------------------------------------------
 # List-route interception — serve listing entirely from DB
 # ---------------------------------------------------------------------------
diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
index 60b85cb42d0..3d60f4f5f3a 100644
--- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
+++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
@@ -1209,14 +1209,19 @@ async def pass_through_request(
 
             return StreamingResponse(
                 wrap_passthrough_sse_bytes_with_keepalive_pings(
-                    stream=PassThroughStreamingHandler.chunk_processor(
-                        response=response,
-                        request_body=_parsed_body,
-                        litellm_logging_obj=logging_obj,
-                        endpoint_type=endpoint_type,
-                        start_time=start_time,
-                        passthrough_success_handler_obj=pass_through_endpoint_logging,
-                        url_route=str(url),
+                    stream=_own_streamed_managed_ids(
+                        stream=PassThroughStreamingHandler.chunk_processor(
+                            response=response,
+                            request_body=_parsed_body,
+                            litellm_logging_obj=logging_obj,
+                            endpoint_type=endpoint_type,
+                            start_time=start_time,
+                            passthrough_success_handler_obj=pass_through_endpoint_logging,
+                            url_route=str(url),
+                        ),
+                        managed_id_provider=_managed_id_provider,
+                        request=request,
+                        user_api_key_dict=user_api_key_dict,
                     ),
                     ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds,
                     upstream_headers=response.headers,
@@ -1285,14 +1290,19 @@ async def pass_through_request(
 
             return StreamingResponse(
                 wrap_passthrough_sse_bytes_with_keepalive_pings(
-                    stream=PassThroughStreamingHandler.chunk_processor(
-                        response=response,
-                        request_body=_parsed_body,
-                        litellm_logging_obj=logging_obj,
-                        endpoint_type=endpoint_type,
-                        start_time=start_time,
-                        passthrough_success_handler_obj=pass_through_endpoint_logging,
-                        url_route=str(url),
+                    stream=_own_streamed_managed_ids(
+                        stream=PassThroughStreamingHandler.chunk_processor(
+                            response=response,
+                            request_body=_parsed_body,
+                            litellm_logging_obj=logging_obj,
+                            endpoint_type=endpoint_type,
+                            start_time=start_time,
+                            passthrough_success_handler_obj=pass_through_endpoint_logging,
+                            url_route=str(url),
+                        ),
+                        managed_id_provider=_managed_id_provider,
+                        request=request,
+                        user_api_key_dict=user_api_key_dict,
                     ),
                     ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds,
                     upstream_headers=response.headers,
@@ -2441,6 +2451,36 @@ def _is_streaming_response(response: httpx.Response) -> bool:
     return False
 
 
+def _own_streamed_managed_ids(
+    stream: AsyncGenerator[bytes, None],
+    managed_id_provider: str | None,
+    request: Request,
+    user_api_key_dict: UserAPIKeyAuth,
+) -> AsyncGenerator[bytes, None]:
+    from litellm.proxy.proxy_server import general_settings, prisma_client, proxy_logging_obj
+
+    if (
+        managed_id_provider is None
+        or not general_settings.get("passthrough_managed_object_ids", False)
+        or prisma_client is None
+        or proxy_logging_obj.get_proxy_hook("managed_files") is None
+    ):
+        return stream
+    from litellm.proxy.auth.auth_utils import get_request_route
+    from litellm.proxy.pass_through_endpoints.managed_id_rewriter import (
+        rewrite_streamed_response_ids,
+    )
+
+    return rewrite_streamed_response_ids(
+        stream=stream,
+        provider=managed_id_provider,
+        method=request.method,
+        route=get_request_route(request),
+        user_api_key_dict=user_api_key_dict,
+        prisma_client=prisma_client,
+    )
+
+
 def _should_buffer_passthrough_response(response: httpx.Response) -> bool:
     """
     Decide from the response headers whether the body must be read into memory.
diff --git a/litellm/proxy/pass_through_endpoints/streaming_handler.py b/litellm/proxy/pass_through_endpoints/streaming_handler.py
index b71622fc33d..eea1b19dea3 100644
--- a/litellm/proxy/pass_through_endpoints/streaming_handler.py
+++ b/litellm/proxy/pass_through_endpoints/streaming_handler.py
@@ -10,6 +10,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
 from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
 from litellm.proxy._types import PassThroughEndpointLoggingResultValues
 from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
+from litellm.proxy.common_utils.sse_keepalive import split_complete_sse_frames
 from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType
 from litellm.types.utils import StandardPassThroughResponseObject
 
@@ -101,7 +102,7 @@ class PassThroughStreamingHandler:
                 async for chunk in response.aiter_bytes():
                     raw_bytes.append(chunk)
                     PassThroughStreamingHandler._stamp_first_chunk_if_needed(litellm_logging_obj)
-                    complete_frames, pending = PassThroughStreamingHandler._split_complete_sse_frames(
+                    complete_frames, pending = split_complete_sse_frames(
                         pending + chunk
                     )  # rebind-ok: SSE frame reassembly buffer across transport chunks
                     if complete_frames:
@@ -139,17 +140,6 @@ class PassThroughStreamingHandler:
                 except Exception as e:
                     verbose_proxy_logger.error("Error scheduling chunk_processor logging: %s", e)
 
-    @staticmethod
-    def _split_complete_sse_frames(pending: bytes) -> tuple[bytes, bytes]:
-        lf_boundary_end: Final = pending.rfind(b"\n\n") + 2
-        crlf_boundary_end: Final = pending.rfind(b"\r\n\r\n") + 4
-        boundary_end: Final = max(
-            lf_boundary_end if lf_boundary_end >= 2 else 0, crlf_boundary_end if crlf_boundary_end >= 4 else 0
-        )
-        if boundary_end == 0:
-            return b"", pending
-        return pending[:boundary_end], pending[boundary_end:]
-
     @staticmethod
     async def _route_streaming_logging_to_handler(
         litellm_logging_obj: LiteLLMLoggingObj,
diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_managed_id_rewriter.py b/tests/test_litellm/proxy/pass_through_endpoints/test_managed_id_rewriter.py
index f5bec4a2585..da4ccff7008 100644
--- a/tests/test_litellm/proxy/pass_through_endpoints/test_managed_id_rewriter.py
+++ b/tests/test_litellm/proxy/pass_through_endpoints/test_managed_id_rewriter.py
@@ -1,12 +1,15 @@
 import datetime
+import json
+from collections.abc import AsyncIterator, Iterable
 from unittest.mock import AsyncMock, MagicMock
 
 import pytest
 
 from litellm.proxy._types import ProxyException, UserAPIKeyAuth
-from litellm.proxy.pass_through_endpoints.managed_id_codec import new_managed_id
+from litellm.proxy.pass_through_endpoints.managed_id_codec import decode, new_managed_id
 from litellm.proxy.pass_through_endpoints.managed_id_rewriter import (
     list_passthrough_ids_from_db,
+    rewrite_streamed_response_ids,
 )
 
 
@@ -27,9 +30,39 @@ def _prisma_client(file_rows=None, batch_rows=None) -> MagicMock:
     pc.db.litellm_managedobjecttable.find_many = AsyncMock(
         side_effect=lambda *args, take=None, **kwargs: list(batch_rows or [])[:take]
     )
+    pc.db.litellm_managedobjecttable.upsert = AsyncMock(return_value=None)
     return pc
 
 
+RAW_RESPONSE_ID = "resp_0123456789abcdef"
+
+
+def _response_stream_bytes(raw_id: str = RAW_RESPONSE_ID) -> bytes:
+    events = (
+        ("response.created", {"type": "response.created", "response": {"id": raw_id, "status": "in_progress"}}),
+        ("response.output_text.delta", {"type": "response.output_text.delta", "delta": "mango"}),
+        ("response.completed", {"type": "response.completed", "response": {"id": raw_id, "status": "completed"}}),
+    )
+    return b"".join(f"event: {name}\ndata: {json.dumps(payload)}\n\n".encode() for name, payload in events)
+
+
+async def _chunks(payload: bytes, size: int) -> AsyncIterator[bytes]:
+    for start in range(0, len(payload), size):
+        yield payload[start : start + size]
+
+
+async def _collect(stream: AsyncIterator[bytes]) -> bytes:
+    return b"".join([chunk async for chunk in stream])
+
+
+def _response_ids(sse: bytes) -> Iterable[str]:
+    for line in sse.decode().splitlines():
+        if line.startswith("data:"):
+            event = json.loads(line[len("data:") :])
+            if "response" in event:
+                yield event["response"]["id"]
+
+
 def _file_row(unified_id: str) -> MagicMock:
     row = MagicMock()
     row.unified_file_id = unified_id
@@ -67,9 +100,7 @@ def _batch_row(unified_id: str) -> MagicMock:
         ),
     ],
 )
-async def test_list_batches_out_of_range_limit_raises_400(
-    limit, expected_message, expected_openai_code
-):
+async def test_list_batches_out_of_range_limit_raises_400(limit, expected_message, expected_openai_code):
     pc = _prisma_client(batch_rows=[_batch_row(new_managed_id("openai", "batch_abc"))])
 
     with pytest.raises(ProxyException) as exc:
@@ -147,3 +178,75 @@ async def test_list_files_drops_batch_guardrail_key_persisted_by_an_older_proxy(
     assert result is not None
     assert "litellm_batch_guardrail" not in result["data"][0]
     assert result["data"][0]["filename"] == "test.jsonl"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("chunk_size", [1, 7, 4096])
+async def test_streamed_response_is_owned_and_rewritten_across_chunk_boundaries(chunk_size: int):
+    """A streamed POST /v1/responses records the caller as owner once and returns
+    the minted id in every event, no matter how the transport splits the SSE bytes."""
+    pc = _prisma_client()
+
+    output = await _collect(
+        rewrite_streamed_response_ids(
+            stream=_chunks(_response_stream_bytes(), chunk_size),
+            provider="openai",
+            method="POST",
+            route="/openai_passthrough/v1/responses",
+            user_api_key_dict=_user(),
+            prisma_client=pc,
+        )
+    )
+
+    pc.db.litellm_managedobjecttable.upsert.assert_awaited_once()
+    created = pc.db.litellm_managedobjecttable.upsert.await_args.kwargs["data"]["create"]
+    assert created["created_by"] == "user-1"
+    assert created["team_id"] == "team-1"
+    assert created["file_purpose"] == "response"
+    assert created["model_object_id"] == f"passthrough:openai:{RAW_RESPONSE_ID}"
+    managed_id = created["unified_object_id"]
+    assert decode(managed_id).raw_provider_id == RAW_RESPONSE_ID
+    assert list(_response_ids(output)) == [managed_id, managed_id]
+    assert RAW_RESPONSE_ID.encode() not in output
+    assert output == _response_stream_bytes(managed_id)
+
+
+@pytest.mark.asyncio
+async def test_streamed_bytes_untouched_on_routes_without_a_response_id():
+    pc = _prisma_client()
+    payload = _response_stream_bytes()
+
+    output = await _collect(
+        rewrite_streamed_response_ids(
+            stream=_chunks(payload, 5),
+            provider="openai",
+            method="POST",
+            route="/openai_passthrough/v1/chat/completions",
+            user_api_key_dict=_user(),
+            prisma_client=pc,
+        )
+    )
+
+    assert output == payload
+    pc.db.litellm_managedobjecttable.upsert.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_streamed_response_stays_raw_and_intact_when_the_row_cannot_be_persisted():
+    pc = _prisma_client()
+    pc.db.litellm_managedobjecttable.upsert = AsyncMock(side_effect=RuntimeError("db down"))
+    payload = _response_stream_bytes()
+
+    output = await _collect(
+        rewrite_streamed_response_ids(
+            stream=_chunks(payload, 3),
+            provider="openai",
+            method="POST",
+            route="/openai_passthrough/v1/responses",
+            user_api_key_dict=_user(),
+            prisma_client=pc,
+        )
+    )
+
+    assert output == payload
+    pc.db.litellm_managedobjecttable.upsert.assert_awaited_once()
diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py
index 99a84d43c9b..a3f56adb86f 100644
--- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py
+++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py
@@ -1493,6 +1493,86 @@ async def test_pass_through_request_sse_response_marks_logging_obj_as_stream():
                 assert logging_obj.model_call_details["stream"] is True
 
 
+@pytest.mark.asyncio
+async def test_pass_through_request_streamed_response_is_owned_by_the_caller():
+    """
+    Regression: with passthrough_managed_object_ids on, a streamed
+    POST /openai_passthrough/v1/responses left the raw resp_ id in the stream and
+    recorded no owner, so any other key could read, continue, and delete it.
+    """
+    import litellm
+    from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
+    from litellm.types.llms.custom_http import httpxSpecialProvider
+
+    raw_id = "resp_0123456789abcdef"
+    upstream_body = (
+        b'event: response.created\ndata: {"type": "response.created", "response": {"id": "%s"}}\n\n'
+        b'event: response.completed\ndata: {"type": "response.completed", "response": {"id": "%s"}}\n\n'
+    ) % (raw_id.encode(), raw_id.encode())
+    prisma_client = MagicMock()
+    prisma_client.db.litellm_managedobjecttable.find_first = AsyncMock(return_value=None)
+    prisma_client.db.litellm_managedobjecttable.upsert = AsyncMock(return_value=None)
+    prisma_client.db.litellm_managedfiletable.find_first = AsyncMock(return_value=None)
+
+    def transport_handler(upstream_request: httpx.Request) -> httpx.Response:
+        return httpx.Response(200, content=upstream_body, headers={"content-type": "text/event-stream"})
+
+    real_handler = get_async_httpx_client(
+        llm_provider=httpxSpecialProvider.PassThroughEndpoint,
+        params={"timeout": resolve_pass_through_request_timeout(None)},
+    )
+    cache_dict = litellm.in_memory_llm_clients_cache.cache_dict
+    cache_key = next(key for key, cached in cache_dict.items() if cached is real_handler)
+    cache_dict[cache_key] = SimpleNamespace(client=httpx.AsyncClient(transport=httpx.MockTransport(transport_handler)))
+
+    mock_proxy_logging = MagicMock()
+    mock_proxy_logging.pre_call_hook = AsyncMock(side_effect=lambda user_api_key_dict, data, call_type: data)
+    mock_proxy_logging.post_call_failure_hook = AsyncMock()
+    mock_proxy_logging.post_call_response_headers_hook = AsyncMock(return_value={})
+    mock_proxy_logging.get_proxy_hook = MagicMock(return_value=MagicMock())
+
+    mock_request = MagicMock(spec=Request)
+    mock_request.method = "POST"
+    mock_request.scope = {"path": "/openai_passthrough/v1/responses"}
+    mock_request.url = MagicMock()
+    mock_request.url.path = "/openai_passthrough/v1/responses"
+    mock_request.body = AsyncMock(return_value=b'{"model": "gpt-5.1", "input": "hi", "stream": true}')
+    mock_request.headers = Headers({"content-type": "application/json"})
+    mock_request.query_params = QueryParams({})
+
+    flag_on = {"passthrough_managed_object_ids": True}
+    proxy_server_globals = (
+        patch("litellm.proxy.proxy_server.proxy_logging_obj", mock_proxy_logging),  # test-quality-ok: read at call time
+        patch("litellm.proxy.proxy_server.general_settings", flag_on),  # test-quality-ok: read at call time
+        patch("litellm.proxy.proxy_server.prisma_client", prisma_client),  # test-quality-ok: read at call time
+    )
+
+    try:
+        with ExitStack() as stack:
+            for patched_global in proxy_server_globals:
+                stack.enter_context(patched_global)
+            response = await pass_through_request(
+                request=mock_request,
+                target="https://api.openai.com/v1/responses",
+                custom_headers={},
+                user_api_key_dict=UserAPIKeyAuth(user_id="user-a", team_id="team-a"),
+                custom_llm_provider="openai",
+            )
+            streamed = b"".join([chunk async for chunk in response.body_iterator])
+    finally:
+        cache_dict[cache_key] = real_handler
+
+    assert response.status_code == 200
+    prisma_client.db.litellm_managedobjecttable.upsert.assert_awaited_once()
+    created = prisma_client.db.litellm_managedobjecttable.upsert.await_args.kwargs["data"]["create"]
+    assert created["created_by"] == "user-a"
+    assert created["team_id"] == "team-a"
+    assert created["model_object_id"] == f"passthrough:openai:{raw_id}"
+    managed_id = created["unified_object_id"]
+    assert raw_id.encode() not in streamed
+    assert streamed == upstream_body.replace(raw_id.encode(), managed_id.encode())
+
+
 @pytest.mark.asyncio
 async def test_create_pass_through_endpoint():
     """

From 6386a68c9c37d6ccb2983e99854da0b1b0b7b84a Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Wed, 26 Aug 2026 01:09:14 -0700
Subject: [PATCH 101/105] fix(router): fail fast on PermissionDeniedError with
 a single deployment

---
 litellm/router.py                 |  2 +-
 tests/test_litellm/test_router.py | 42 +++++++++++++++++++++++++++++++
 2 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/litellm/router.py b/litellm/router.py
index de60d46e01d..1e9b23b2fa1 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -7344,7 +7344,7 @@ class Router:
             ):
                 raise error  # then raise the error
 
-        if isinstance(error, openai.AuthenticationError):
+        if isinstance(error, (openai.AuthenticationError, openai.PermissionDeniedError)):
             """
             - if other deployments available -> retry
             - else -> raise error
diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py
index 5ee6160fbdb..cceb034a20b 100644
--- a/tests/test_litellm/test_router.py
+++ b/tests/test_litellm/test_router.py
@@ -10607,3 +10607,45 @@ async def test_async_function_with_fallbacks_scrubs_spoofed_values_from_sibling_
     assert litellm_metadata["client_key"] == "client_value"
     assert metadata["attempted_fallbacks"] == 0
     assert metadata["original_model_group"] == "gpt-3.5-turbo"
+
+
+def _permission_denied_error() -> litellm.PermissionDeniedError:
+    return litellm.PermissionDeniedError(
+        message="OpenrouterException - this key has no access to the model",
+        llm_provider="openrouter",
+        model="openrouter/openai/gpt-4o",
+        response=httpx.Response(status_code=403, request=httpx.Request(method="POST", url="https://openrouter.ai")),
+    )
+
+
+def test_permission_denied_error_is_not_retried_against_a_single_deployment():
+    router = litellm.Router(
+        model_list=[
+            {"model_name": "gpt-4o", "litellm_params": {"model": "openrouter/openai/gpt-4o", "api_key": "sk-test"}},
+        ]
+    )
+
+    with pytest.raises(litellm.PermissionDeniedError):
+        router.should_retry_this_error(
+            error=_permission_denied_error(),
+            healthy_deployments=router.model_list,
+            all_deployments=router.model_list,
+        )
+
+
+def test_permission_denied_error_is_retried_when_other_deployments_exist():
+    router = litellm.Router(
+        model_list=[
+            {"model_name": "gpt-4o", "litellm_params": {"model": "openrouter/openai/gpt-4o", "api_key": "sk-test"}},
+            {"model_name": "gpt-4o", "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-test"}},
+        ]
+    )
+
+    assert (
+        router.should_retry_this_error(
+            error=_permission_denied_error(),
+            healthy_deployments=router.model_list,
+            all_deployments=router.model_list,
+        )
+        is True
+    )

From 6a9662a5a8863976481dea0460027806608fbd79 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Wed, 26 Aug 2026 01:36:18 -0700
Subject: [PATCH 102/105] fix(passthrough): recognize CR-only SSE frame
 delimiters when minting streamed managed ids

---
 litellm/proxy/common_utils/sse_keepalive.py   |  5 ++--
 .../managed_id_rewriter.py                    |  4 ----
 .../proxy/common_utils/test_sse_keepalive.py  | 14 +++++++++++
 .../test_managed_id_rewriter.py               | 23 +++++++++++++++++++
 4 files changed, 39 insertions(+), 7 deletions(-)

diff --git a/litellm/proxy/common_utils/sse_keepalive.py b/litellm/proxy/common_utils/sse_keepalive.py
index e3cebf9f6c6..26fccf8ee82 100644
--- a/litellm/proxy/common_utils/sse_keepalive.py
+++ b/litellm/proxy/common_utils/sse_keepalive.py
@@ -93,10 +93,9 @@ def is_sse_content_type(content_type: str | None) -> bool:
 
 def split_complete_sse_frames(pending: bytes) -> tuple[bytes, bytes]:
     """Split buffered SSE bytes into ``(complete_frames, unterminated_tail)``."""
-    lf_boundary_end: Final = pending.rfind(b"\n\n") + 2
-    crlf_boundary_end: Final = pending.rfind(b"\r\n\r\n") + 4
     boundary_end: Final = max(
-        lf_boundary_end if lf_boundary_end >= 2 else 0, crlf_boundary_end if crlf_boundary_end >= 4 else 0
+        (pending.rfind(delimiter) + len(delimiter) for delimiter in _SSE_FRAME_DELIMITERS if delimiter in pending),
+        default=0,
     )
     if boundary_end == 0:
         return b"", pending
diff --git a/litellm/proxy/pass_through_endpoints/managed_id_rewriter.py b/litellm/proxy/pass_through_endpoints/managed_id_rewriter.py
index bdc4c515803..23cfef6576c 100644
--- a/litellm/proxy/pass_through_endpoints/managed_id_rewriter.py
+++ b/litellm/proxy/pass_through_endpoints/managed_id_rewriter.py
@@ -821,10 +821,6 @@ async def rewrite_response_ids(
     return mutated if changed else body
 
 
-# ---------------------------------------------------------------------------
-# OUTPUT path — streamed Responses API bodies
-# ---------------------------------------------------------------------------
-
 _RESPONSE_ID_PREFIX: Final = "resp_"
 _STREAMED_RESPONSE_ID_SPEC: Final[_FieldSpec] = ("id", _RESPONSE_ID_PREFIX)
 _SSE_DATA_PREFIX: Final = "data:"
diff --git a/tests/test_litellm/proxy/common_utils/test_sse_keepalive.py b/tests/test_litellm/proxy/common_utils/test_sse_keepalive.py
index 89ae74920fe..69b92f5e4d7 100644
--- a/tests/test_litellm/proxy/common_utils/test_sse_keepalive.py
+++ b/tests/test_litellm/proxy/common_utils/test_sse_keepalive.py
@@ -10,6 +10,7 @@ from litellm.proxy.common_utils.sse_keepalive import (
     ANTHROPIC_PING_SSE_CHUNK,
     SSE_COMMENT_PING_BYTES,
     resolve_ttft_keepalive_interval,
+    split_complete_sse_frames,
     wrap_passthrough_sse_bytes_with_keepalive_pings,
     wrap_sse_stream_with_keepalive_pings,
 )
@@ -18,6 +19,19 @@ MESSAGE_START_CHUNK: Final = 'data: {"type": "message_start"}\n\n'
 TEXT_DELTA_CHUNK: Final = 'data: {"type": "content_block_delta"}\n\n'
 
 
+@pytest.mark.parametrize("delimiter", [b"\n\n", b"\r\n\r\n", b"\r\r"])
+def test_split_complete_sse_frames_recognizes_every_sse_frame_delimiter(delimiter: bytes):
+    newline: Final = delimiter[: len(delimiter) // 2]
+    frame: Final = b"event: response.created" + newline + b"data: {}" + delimiter
+    tail: Final = b"data: partial"
+
+    assert split_complete_sse_frames(frame + tail) == (frame, tail)
+
+
+def test_split_complete_sse_frames_holds_bytes_with_no_complete_frame():
+    assert split_complete_sse_frames(b"data: unterminated") == (b"", b"data: unterminated")
+
+
 @pytest.mark.asyncio
 async def test_pings_fill_mid_stream_silence_and_preserve_chunk_order():
     async def gappy_stream() -> AsyncGenerator[str, None]:
diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_managed_id_rewriter.py b/tests/test_litellm/proxy/pass_through_endpoints/test_managed_id_rewriter.py
index da4ccff7008..dc8c49b93d8 100644
--- a/tests/test_litellm/proxy/pass_through_endpoints/test_managed_id_rewriter.py
+++ b/tests/test_litellm/proxy/pass_through_endpoints/test_managed_id_rewriter.py
@@ -211,6 +211,29 @@ async def test_streamed_response_is_owned_and_rewritten_across_chunk_boundaries(
     assert output == _response_stream_bytes(managed_id)
 
 
+@pytest.mark.asyncio
+async def test_streamed_response_with_cr_only_frame_delimiters_is_still_owned_and_rewritten():
+    """SSE also terminates lines with a lone CR; those frames must mint and rewrite too."""
+    pc = _prisma_client()
+    payload = _response_stream_bytes().replace(b"\n", b"\r")
+
+    output = await _collect(
+        rewrite_streamed_response_ids(
+            stream=_chunks(payload, 7),
+            provider="openai",
+            method="POST",
+            route="/openai_passthrough/v1/responses",
+            user_api_key_dict=_user(),
+            prisma_client=pc,
+        )
+    )
+
+    pc.db.litellm_managedobjecttable.upsert.assert_awaited_once()
+    managed_id = pc.db.litellm_managedobjecttable.upsert.await_args.kwargs["data"]["create"]["unified_object_id"]
+    assert RAW_RESPONSE_ID.encode() not in output
+    assert output == _response_stream_bytes(managed_id).replace(b"\n", b"\r")
+
+
 @pytest.mark.asyncio
 async def test_streamed_bytes_untouched_on_routes_without_a_response_id():
     pc = _prisma_client()

From 498ba9dd62397b2c51456524cdb99c728653becb Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Wed, 26 Aug 2026 01:38:13 -0700
Subject: [PATCH 103/105] fix(proxy): encrypt streamed responses ids on
 /openai/v1/responses and /responses aliases

The streaming security hook only encrypted response ids when request_route
matched "/v1/responses" exactly, so streamed creates on the /openai/v1/responses
and /responses aliases leaked the plain managed id. A second virtual key could
GET, continue, and DELETE another key's response. Normalize the route (strip the
provider prefix, accept the /responses alias) before gating, mirroring the
non-streaming hook which has no route gate.
---
 litellm/proxy/hooks/responses_id_security.py  | 18 +++-
 .../test_responses_id_security.py             | 97 ++++++++++++++++++-
 2 files changed, 112 insertions(+), 3 deletions(-)

diff --git a/litellm/proxy/hooks/responses_id_security.py b/litellm/proxy/hooks/responses_id_security.py
index 3dafcc08551..21d12c8f720 100644
--- a/litellm/proxy/hooks/responses_id_security.py
+++ b/litellm/proxy/hooks/responses_id_security.py
@@ -28,6 +28,21 @@ if TYPE_CHECKING:
     from litellm.proxy._types import UserAPIKeyAuth
 
 
+_RESPONSES_API_PROVIDER_PREFIX: Final = "/openai"
+_RESPONSES_API_CREATE_ROUTES: Final = frozenset({"/v1/responses", "/responses"})
+
+
+def _is_responses_api_create_route(request_route: str | None) -> bool:
+    if request_route is None:
+        return False
+    canonical: Final = (
+        request_route[len(_RESPONSES_API_PROVIDER_PREFIX) :]
+        if request_route.startswith(_RESPONSES_API_PROVIDER_PREFIX + "/")
+        else request_route
+    )
+    return canonical in _RESPONSES_API_CREATE_ROUTES
+
+
 class ResponsesIDSecurity(CustomLogger):
     def __init__(self):
         pass
@@ -267,8 +282,7 @@ class ResponsesIDSecurity(CustomLogger):
         async for chunk in response:
             if (
                 isinstance(chunk, BaseLiteLLMOpenAIResponseObject)
-                and user_api_key_dict.request_route
-                == "/v1/responses"  # only encrypt the response id for the responses api
+                and _is_responses_api_create_route(user_api_key_dict.request_route)
                 and not general_settings.get("disable_responses_id_security", False)
             ):
                 chunk = self._encrypt_response_id(chunk, user_api_key_dict, request_encryption_cache)
diff --git a/tests/test_litellm/test_responses_id_security.py b/tests/test_litellm/test_responses_id_security.py
index 17487030cc1..f1cb9eccff0 100644
--- a/tests/test_litellm/test_responses_id_security.py
+++ b/tests/test_litellm/test_responses_id_security.py
@@ -9,7 +9,11 @@ from unittest.mock import AsyncMock, MagicMock, patch
 import pytest
 from fastapi import HTTPException
 
-from litellm.proxy.hooks.responses_id_security import ResponsesIDSecurity
+from litellm.proxy.hooks.responses_id_security import (
+    ResponsesIDSecurity,
+    _is_responses_api_create_route,
+)
+from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject
 from litellm.types.llms.openai import ResponsesAPIResponse
 from litellm.types.utils import SpecialEnums
 
@@ -575,6 +579,97 @@ class TestAsyncPreCallHook:
                     assert "team" in exc_info.value.detail.lower()
 
 
+class TestIsResponsesApiCreateRoute:
+    """Test the route gate that decides whether a streamed response id is encrypted."""
+
+    @pytest.mark.parametrize(
+        "route",
+        [
+            "/v1/responses",
+            "/responses",
+            "/openai/v1/responses",
+        ],
+    )
+    def test_create_routes_match(self, route):
+        assert _is_responses_api_create_route(route) is True
+
+    @pytest.mark.parametrize(
+        "route",
+        [
+            None,
+            "/chat/completions",
+            "/openai/v1/chat/completions",
+            "/v1/responses/{response_id}",
+            "/openai/v1/responses/{response_id}",
+            "/v1/responsesX",
+            "/responsesX",
+        ],
+    )
+    def test_non_create_routes_do_not_match(self, route):
+        assert _is_responses_api_create_route(route) is False
+
+
+class TestAsyncPostCallStreamingIteratorHook:
+    """Regression test for LIT-6167: streamed responses on /openai/v1/responses and
+    /responses must have their ids security-encrypted, not just on the exact
+    /v1/responses path. Uses real encryption so the id must round-trip back to the
+    raw provider id plus the caller's user/team, which is the access-control wrapper
+    the aliases were leaking without."""
+
+    @staticmethod
+    async def _agen(chunks):
+        for chunk in chunks:
+            yield chunk
+
+    async def _drain_streamed_id(self, responses_id_security, route, monkeypatch):
+        monkeypatch.setenv("LITELLM_SALT_KEY", "sk-test-salt-key-abcdefghij")
+        chunk = BaseLiteLLMOpenAIResponseObject(id="resp_rawprovider123")
+
+        mock_auth = MagicMock()
+        mock_auth.user_id = "user-a"
+        mock_auth.team_id = "team-a"
+        mock_auth.request_route = route
+
+        collected = [
+            out
+            async for out in responses_id_security.async_post_call_streaming_iterator_hook(
+                user_api_key_dict=mock_auth,
+                response=self._agen([chunk]),
+                request_data={},
+            )
+        ]
+        return collected[0].id
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize(
+        "route",
+        ["/v1/responses", "/responses", "/openai/v1/responses"],
+    )
+    async def test_streamed_id_encrypted_on_all_responses_routes(
+        self, responses_id_security, route, monkeypatch
+    ):
+        streamed_id = await self._drain_streamed_id(responses_id_security, route, monkeypatch)
+
+        assert streamed_id != "resp_rawprovider123"
+        assert responses_id_security._is_encrypted_response_id(streamed_id)
+        assert responses_id_security._decrypt_response_id(streamed_id) == (
+            "resp_rawprovider123",
+            "user-a",
+            "team-a",
+        )
+
+    @pytest.mark.asyncio
+    async def test_streamed_id_untouched_on_non_responses_route(
+        self, responses_id_security, monkeypatch
+    ):
+        streamed_id = await self._drain_streamed_id(
+            responses_id_security, "/chat/completions", monkeypatch
+        )
+
+        assert streamed_id == "resp_rawprovider123"
+        assert not responses_id_security._is_encrypted_response_id(streamed_id)
+
+
 class TestAsyncPostCallSuccessHook:
     """Test async_post_call_success_hook function"""
 

From 54cbc4470585dafac75486c588d5fd27db7c6eaa Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Wed, 26 Aug 2026 01:46:29 -0700
Subject: [PATCH 104/105] test(cost-calc): pin the rate fallbacks inside a
 tiered-pricing tier

_get_tiered_base_costs documents that tiered pricing is all-or-nothing: a
tier is picked from the request's input tokens, and any rate that tier does
not declare falls back to the tier's own input rate so one request is never
priced from two tiers.

Nothing checked that. Every existing tiered test supplies a fully populated
tier, so the fallbacks were never reached: deleting them from the source
left the whole suite green. The fallbacks are not hypothetical either. Of
the 66 tiered rows shipped in model_prices_and_context_window.json, 54
declare no cache-creation rate and 44 declare no cache-read rate, so the
fallback is what prices their cached tokens today.

Adds three tests on the generic path:
  - a tier with no cache rates bills cached and cache-creation tokens at
    that tier's input rate, ignoring the model's top-level cache rates
  - a tier with no above-1hr rate bills 1h cache writes at the tier's
    cache-creation rate rather than zero
  - a tier with no input rate is not a priced tier at all, so the model's
    flat rates still apply instead of billing input at zero

Test-only change, no source touched.
---
 .../llm_cost_calc/test_llm_cost_calc_utils.py | 130 ++++++++++++++++++
 1 file changed, 130 insertions(+)

diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
index 6f513ce1bd4..c875bf5b535 100644
--- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
+++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
@@ -716,6 +716,136 @@ def test_generic_cost_per_token_tier_without_an_output_rate_bills_the_model_rate
         litellm.model_cost.pop(model, None)
 
 
+def test_generic_cost_per_token_tier_without_cache_rates_bills_cache_at_the_tier_input_rate():
+    model = "litellm-test-tiered-no-cache-rates"
+    custom_llm_provider = "openrouter"
+    litellm.register_model(
+        {
+            model: {
+                "litellm_provider": custom_llm_provider,
+                "mode": "chat",
+                "cache_read_input_token_cost": 9e-09,
+                "cache_creation_input_token_cost": 9e-06,
+                "tiered_pricing": [
+                    {
+                        "range": [0, 32000],
+                        "input_cost_per_token": 4.6e-07,
+                        "output_cost_per_token": 2.3e-06,
+                    },
+                    {
+                        "range": [32000, 128000],
+                        "input_cost_per_token": 7e-07,
+                        "output_cost_per_token": 3.5e-06,
+                    },
+                ],
+            }
+        }
+    )
+
+    try:
+        uncached = Usage(prompt_tokens=40000, completion_tokens=100, total_tokens=40100)
+        cached = Usage(
+            prompt_tokens=40000,
+            completion_tokens=100,
+            total_tokens=40100,
+            prompt_tokens_details=PromptTokensDetailsWrapper(
+                cached_tokens=5000, cache_creation_tokens=15000
+            ),
+        )
+        uncached_prompt_cost, _ = generic_cost_per_token(
+            model=model,
+            usage=uncached,
+            custom_llm_provider=custom_llm_provider,
+        )
+        cached_prompt_cost, cached_completion_cost = generic_cost_per_token(
+            model=model,
+            usage=cached,
+            custom_llm_provider=custom_llm_provider,
+        )
+
+        tier_input_rate = 7e-07
+        assert round(cached_prompt_cost, 12) == round(40000 * tier_input_rate, 12)
+        assert round(cached_prompt_cost, 12) == round(uncached_prompt_cost, 12)
+        assert round(cached_completion_cost, 12) == round(100 * 3.5e-06, 12)
+    finally:
+        litellm.model_cost.pop(model, None)
+
+
+def test_generic_cost_per_token_tier_without_a_1hr_cache_rate_bills_the_tier_cache_creation_rate():
+    model = "litellm-test-tiered-no-1hr-cache-rate"
+    custom_llm_provider = "openrouter"
+    litellm.register_model(
+        {
+            model: {
+                "litellm_provider": custom_llm_provider,
+                "mode": "chat",
+                "cache_creation_input_token_cost_above_1hr": 9e-05,
+                "tiered_pricing": [
+                    {
+                        "range": [0, 128000],
+                        "input_cost_per_token": 7e-07,
+                        "output_cost_per_token": 3.5e-06,
+                        "cache_creation_input_token_cost": 8.75e-07,
+                    }
+                ],
+            }
+        }
+    )
+
+    try:
+        usage = Usage(
+            prompt_tokens=1000,
+            completion_tokens=10,
+            total_tokens=1010,
+            prompt_tokens_details=PromptTokensDetailsWrapper(
+                cache_creation_tokens=800,
+                cache_creation_token_details=CacheCreationTokenDetails(
+                    ephemeral_5m_input_tokens=300, ephemeral_1h_input_tokens=500
+                ),
+            ),
+        )
+        prompt_cost, completion_cost = generic_cost_per_token(
+            model=model,
+            usage=usage,
+            custom_llm_provider=custom_llm_provider,
+        )
+
+        tier_cache_creation_rate = 8.75e-07
+        expected_prompt = (200 * 7e-07) + (800 * tier_cache_creation_rate)
+        assert round(prompt_cost, 12) == round(expected_prompt, 12)
+        assert round(completion_cost, 12) == round(10 * 3.5e-06, 12)
+    finally:
+        litellm.model_cost.pop(model, None)
+
+
+def test_generic_cost_per_token_tier_without_an_input_rate_is_not_a_priced_tier():
+    model = "litellm-test-tiered-no-input-rate"
+    custom_llm_provider = "openrouter"
+    litellm.register_model(
+        {
+            model: {
+                "litellm_provider": custom_llm_provider,
+                "mode": "chat",
+                "input_cost_per_token": 1e-06,
+                "output_cost_per_token": 2e-06,
+                "tiered_pricing": [{"range": [0, 128000], "output_cost_per_token": 3.5e-06}],
+            }
+        }
+    )
+
+    try:
+        usage = Usage(prompt_tokens=1000, completion_tokens=100, total_tokens=1100)
+        prompt_cost, completion_cost = generic_cost_per_token(
+            model=model,
+            usage=usage,
+            custom_llm_provider=custom_llm_provider,
+        )
+        assert round(prompt_cost, 12) == round(1000 * 1e-06, 12)
+        assert round(completion_cost, 12) == round(100 * 2e-06, 12)
+    finally:
+        litellm.model_cost.pop(model, None)
+
+
 def test_router_deployment_with_input_only_tiers_bills_completions_at_the_backend_rate():
     """Regression: the router registers a deployment's custom pricing as a standalone
     model_cost entry holding only the supplied fields, so an input-only tier table left

From 95f8373e3cf202f93f132b375b0b4ef537f94b8b Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Wed, 26 Aug 2026 01:52:01 -0700
Subject: [PATCH 105/105] test(responses): drive streamed-id regression via
 production ResponseCompletedEvent shape

The streamed-id regression test built a bare BaseLiteLLMOpenAIResponseObject with a
top-level id, hitting the wrong _encrypt_response_id branch. A real streamed create
emits ResponseCompletedEvent, whose client-visible id lives on event.response.id, so
the test now drives that production event shape and reads collected[0].response.id.
Mutating the alias route gate or disabling the .response.id encryption branch both
fail the test.
---
 .../test_responses_id_security.py             | 37 +++++++++++++++----
 1 file changed, 29 insertions(+), 8 deletions(-)

diff --git a/tests/test_litellm/test_responses_id_security.py b/tests/test_litellm/test_responses_id_security.py
index f1cb9eccff0..763ee4dac00 100644
--- a/tests/test_litellm/test_responses_id_security.py
+++ b/tests/test_litellm/test_responses_id_security.py
@@ -13,8 +13,11 @@ from litellm.proxy.hooks.responses_id_security import (
     ResponsesIDSecurity,
     _is_responses_api_create_route,
 )
-from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject
-from litellm.types.llms.openai import ResponsesAPIResponse
+from litellm.types.llms.openai import (
+    ResponseCompletedEvent,
+    ResponsesAPIResponse,
+    ResponsesAPIStreamEvents,
+)
 from litellm.types.utils import SpecialEnums
 
 
@@ -612,18 +615,36 @@ class TestIsResponsesApiCreateRoute:
 class TestAsyncPostCallStreamingIteratorHook:
     """Regression test for LIT-6167: streamed responses on /openai/v1/responses and
     /responses must have their ids security-encrypted, not just on the exact
-    /v1/responses path. Uses real encryption so the id must round-trip back to the
-    raw provider id plus the caller's user/team, which is the access-control wrapper
-    the aliases were leaking without."""
+    /v1/responses path. A streamed create emits ResponseCompletedEvent, whose
+    client-visible id lives on event.response.id, so the test drives that production
+    event shape (not a top-level id) and uses real encryption, asserting the id
+    round-trips back to the raw provider id plus the caller's user/team, which is the
+    access-control wrapper the aliases were leaking without."""
 
     @staticmethod
     async def _agen(chunks):
         for chunk in chunks:
             yield chunk
 
+    @staticmethod
+    def _completed_event(response_id):
+        return ResponseCompletedEvent(
+            type=ResponsesAPIStreamEvents.RESPONSE_COMPLETED,
+            response=ResponsesAPIResponse(
+                id=response_id,
+                created_at=0,
+                model="gpt-5.1",
+                object="response",
+                output=[],
+                parallel_tool_calls=False,
+                tool_choice="auto",
+                tools=[],
+            ),
+        )
+
     async def _drain_streamed_id(self, responses_id_security, route, monkeypatch):
         monkeypatch.setenv("LITELLM_SALT_KEY", "sk-test-salt-key-abcdefghij")
-        chunk = BaseLiteLLMOpenAIResponseObject(id="resp_rawprovider123")
+        event = self._completed_event("resp_rawprovider123")
 
         mock_auth = MagicMock()
         mock_auth.user_id = "user-a"
@@ -634,11 +655,11 @@ class TestAsyncPostCallStreamingIteratorHook:
             out
             async for out in responses_id_security.async_post_call_streaming_iterator_hook(
                 user_api_key_dict=mock_auth,
-                response=self._agen([chunk]),
+                response=self._agen([event]),
                 request_data={},
             )
         ]
-        return collected[0].id
+        return collected[0].response.id
 
     @pytest.mark.asyncio
     @pytest.mark.parametrize(