From 429ad06972b7263435a551a1a5f12f3f985afb53 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 29 Aug 2026 14:23:57 -0700
Subject: [PATCH 01/42] fix(guardrails): write structured_messages rewrites
 back into /v1/responses input

Message-rewriting guardrails such as Headroom return their rewrite in
structured_messages and leave texts untouched. The responses guardrail
translation only mapped texts back, so compression never reached the
upstream request on /v1/responses while the retrieve tool still got
injected. Convert the returned messages back to Responses input (plus
instructions) the way the chat and Anthropic handlers already do, and
keep developer messages as input_text in the chat-to-responses bridge.

Resolves LIT-6494
---
 .../transformation.py                         |  2 +-
 .../guardrail_translation/handler.py          | 47 +++++++--
 ...responses_transformation_transformation.py | 13 +++
 ...test_openai_responses_guardrail_handler.py | 96 +++++++++++++++++++
 .../guardrail_hooks/test_headroom.py          | 34 +++++++
 5 files changed, 181 insertions(+), 11 deletions(-)

diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py
index 85fb0bc8dc6..63c66a4c2c5 100644
--- a/litellm/completion_extras/litellm_responses_transformation/transformation.py
+++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py
@@ -931,7 +931,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         return OpenAiResponsesToChatCompletionStreamIterator(streaming_response, sync_stream, json_mode)
 
     def _convert_content_str_to_input_text(self, content: str, role: str) -> dict[str, object]:
-        if role == "user" or role == "system" or role == "tool":
+        if role in ("user", "system", "developer", "tool"):
             return {"type": "input_text", "text": content}
         else:
             return {"type": "output_text", "text": content}
diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py
index 7c5d8ac99ad..4320edb8414 100644
--- a/litellm/llms/openai/responses/guardrail_translation/handler.py
+++ b/litellm/llms/openai/responses/guardrail_translation/handler.py
@@ -38,6 +38,7 @@ from typing_extensions import ReadOnly, TypedDict
 
 from litellm._logging import verbose_proxy_logger
 from litellm.completion_extras.litellm_responses_transformation.transformation import (
+    LiteLLMResponsesTransformationHandler,
     OpenAiResponsesToChatCompletionStreamIterator,
 )
 from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
@@ -149,8 +150,15 @@ class OpenAIResponsesHandler(BaseTranslation):
                 input_type="request",
                 logging_obj=litellm_logging_obj,
             )
-            guardrailed_texts = guardrailed_inputs.get("texts", [])
-            data["input"] = guardrailed_texts[0] if guardrailed_texts else input_data
+            guardrailed_structured_messages = guardrailed_inputs.get("structured_messages")
+            if (
+                guardrailed_structured_messages is not None
+                and guardrailed_structured_messages is not structured_messages
+            ):
+                self._write_back_structured_messages(data, guardrailed_structured_messages)
+            else:
+                guardrailed_texts = guardrailed_inputs.get("texts", [])
+                data["input"] = guardrailed_texts[0] if guardrailed_texts else input_data
             self._apply_guardrailed_tools_to_data(data, original_tools, guardrailed_inputs.get("tools"))
             verbose_proxy_logger.debug("OpenAI Responses API: Processed string input")
             return data
@@ -198,24 +206,43 @@ class OpenAIResponsesHandler(BaseTranslation):
                 logging_obj=litellm_logging_obj,
             )
 
-            guardrailed_texts = guardrailed_inputs.get("texts", [])
             self._apply_guardrailed_tools_to_data(
                 data,
                 original_tools_list,
                 guardrailed_inputs.get("tools"),
             )
 
-            # Step 3: Map guardrail responses back to original input structure
-            await self._apply_guardrail_responses_to_input(
-                messages=input_data,
-                responses=guardrailed_texts,
-                task_mappings=task_mappings,
-            )
+            guardrailed_structured_messages = guardrailed_inputs.get("structured_messages")
+            if (
+                guardrailed_structured_messages is not None
+                and guardrailed_structured_messages is not structured_messages
+            ):
+                self._write_back_structured_messages(data, guardrailed_structured_messages)
+            else:
+                # Step 3: Map guardrail responses back to original input structure
+                await self._apply_guardrail_responses_to_input(
+                    messages=input_data,
+                    responses=guardrailed_inputs.get("texts", []),
+                    task_mappings=task_mappings,
+                )
 
-        verbose_proxy_logger.debug("OpenAI Responses API: Processed input messages: %s", input_data)
+        verbose_proxy_logger.debug("OpenAI Responses API: Processed input messages: %s", data.get("input"))
 
         return data
 
+    @staticmethod
+    def _write_back_structured_messages(data: dict, structured_messages: Sequence[AllMessageValues]) -> None:
+        input_items, instructions = (
+            LiteLLMResponsesTransformationHandler().convert_chat_completion_messages_to_responses_api(
+                list(structured_messages)
+            )
+        )
+        data["input"] = input_items
+        if instructions is None:
+            data.pop("instructions", None)
+            return
+        data["instructions"] = instructions
+
     def extract_request_tool_names(self, data: dict) -> list[str]:
         """Extract tool names from Responses API request (tools[].name for function
         and custom, tools[].server_label for mcp)."""
diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py
index 21b60d7a216..6590718878d 100644
--- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py
+++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py
@@ -1295,6 +1295,19 @@ def test_text_plus_tool_calls_sequence():
 # =============================================================================
 
 
+def test_developer_message_content_uses_input_text():
+    handler = LiteLLMResponsesTransformationHandler()
+
+    input_items, instructions = handler.convert_chat_completion_messages_to_responses_api(
+        [{"role": "developer", "content": "Always answer in French."}]
+    )
+
+    assert instructions is None
+    assert input_items == [
+        {"type": "message", "role": "developer", "content": [{"type": "input_text", "text": "Always answer in French."}]}
+    ]
+
+
 def test_tool_message_output_uses_input_text_not_output_text():
     """
     Test that tool message content uses input_text type, not output_text.
diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py
index 447175b09a6..63f8babf760 100644
--- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py
+++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py
@@ -1229,3 +1229,99 @@ class TestOpenAIResponsesHandlerToolInjection:
         names = [t.get("name") for t in result["tools"]]
         assert "get_weather" in names
         assert "injected_tool" in names
+
+
+COMPRESSED_MARKER = "[compressed document; retrieve the full text with hash=b573993006976af767214fac]"
+
+
+class StructuredRewriteGuardrail(CustomGuardrail):
+    """Guardrail that rewrites whole messages via structured_messages and leaves
+    texts untouched, the way message-compressing guardrails do."""
+
+    async def apply_guardrail(
+        self,
+        inputs: GenericGuardrailAPIInputs,
+        request_data: dict,
+        input_type: Literal["request", "response"],
+        logging_obj: Optional[Any] = None,
+    ) -> GenericGuardrailAPIInputs:
+        messages = list(inputs.get("structured_messages") or [])
+        first_user = next(i for i, m in enumerate(messages) if m.get("role") == "user")
+        rewritten = [
+            {**m, "content": COMPRESSED_MARKER} if i == first_user else m for i, m in enumerate(messages)
+        ]
+        return {**inputs, "structured_messages": rewritten}
+
+
+def _texts(item: dict) -> list[str]:
+    content = item.get("content")
+    if isinstance(content, str):
+        return [content]
+    return [part["text"] for part in content]
+
+
+class TestStructuredMessagesWriteBack:
+    """A guardrail's structured_messages rewrite must land in the Responses request,
+    not only the per-text mapping the chat handler shares with it."""
+
+    @pytest.mark.asyncio
+    async def test_list_input_gets_rewritten_messages_and_keeps_instructions(self):
+        handler = OpenAIResponsesHandler()
+        data = {
+            "model": "gpt-5.6",
+            "instructions": "Answer from the memo only.",
+            "input": [
+                {"role": "user", "content": "memo " * 400},
+                {"role": "assistant", "content": "Understood."},
+                {"role": "user", "content": "What is the codename?"},
+            ],
+        }
+
+        result = await handler.process_input_messages(data, StructuredRewriteGuardrail())
+
+        assert result["instructions"] == "Answer from the memo only."
+        user_items = [item for item in result["input"] if item.get("role") == "user"]
+        assert [_texts(item) for item in user_items] == [[COMPRESSED_MARKER], ["What is the codename?"]]
+        assert not any(item.get("role") == "system" for item in result["input"])
+        assert _texts(next(item for item in result["input"] if item.get("role") == "assistant")) == ["Understood."]
+
+    @pytest.mark.asyncio
+    async def test_string_input_becomes_rewritten_message_list(self):
+        handler = OpenAIResponsesHandler()
+        data = {"model": "gpt-5.6", "input": "memo " * 400}
+
+        result = await handler.process_input_messages(data, StructuredRewriteGuardrail())
+
+        assert [_texts(item) for item in result["input"]] == [[COMPRESSED_MARKER]]
+        assert "instructions" not in result
+
+    @pytest.mark.asyncio
+    async def test_developer_item_survives_write_back_as_input_text(self):
+        handler = OpenAIResponsesHandler()
+        data = {
+            "model": "gpt-5.6",
+            "input": [
+                {"role": "developer", "content": "Always answer in French."},
+                {"role": "user", "content": "memo " * 400},
+                {"role": "user", "content": "What is the codename?"},
+            ],
+        }
+
+        result = await handler.process_input_messages(data, StructuredRewriteGuardrail())
+
+        developer = next(item for item in result["input"] if item.get("role") == "developer")
+        assert developer["content"] == [{"type": "input_text", "text": "Always answer in French."}]
+
+    @pytest.mark.asyncio
+    async def test_same_inputs_object_back_keeps_the_text_mapping(self):
+        handler = OpenAIResponsesHandler()
+        original_input = [
+            {"role": "user", "content": "Hello"},
+            {"role": "user", "content": [{"type": "input_text", "text": "Again"}]},
+        ]
+        data = {"model": "gpt-5.6", "input": original_input}
+
+        result = await handler.process_input_messages(data, MockGuardrail())
+
+        assert result["input"] is original_input
+        assert [_texts(item) for item in result["input"]] == [["Hello [GUARDRAILED]"], ["Again [GUARDRAILED]"]]
diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py
index 1fbc975e40a..37b9852d3e1 100644
--- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py
+++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py
@@ -954,6 +954,40 @@ async def test_passthrough_handler_does_not_log_headroom_as_run(
     assert "headroom" not in _applied_guardrails(data)
 
 
+@pytest.mark.asyncio
+async def test_responses_request_sends_compressed_input_and_retrieve_tool_upstream(
+    guardrail: HeadroomGuardrail,
+):
+    """Regression for LIT-6494: on /v1/responses the compressed messages must be
+    written back into `input`, not only the retrieve tool into `tools`, or the
+    model keeps reading the full document and never calls headroom_retrieve."""
+    from litellm.llms.openai.responses.guardrail_translation.handler import OpenAIResponsesHandler
+
+    data = {
+        "model": "gpt-5.6",
+        "instructions": ORIGINAL_MESSAGES[0]["content"],
+        "input": [{"role": m["role"], "content": m["content"]} for m in ORIGINAL_MESSAGES[1:]],
+        "tools": [{"type": "function", "name": "get_weather", "parameters": {"type": "object", "properties": {}}}],
+    }
+
+    with patch.object(
+        guardrail.async_handler,
+        "post",
+        new_callable=AsyncMock,
+        return_value=_make_compress_response(COMPRESSED_MESSAGES_WITH_HASH),
+    ):
+        result = await OpenAIResponsesHandler().process_input_messages(data=data, guardrail_to_apply=guardrail)
+
+    assert result["instructions"] == ORIGINAL_MESSAGES[0]["content"]
+    assert [item["content"][0]["text"] for item in result["input"]] == [
+        COMPRESSED_MESSAGES_WITH_HASH[0]["content"],
+        ORIGINAL_MESSAGES[2]["content"],
+        ORIGINAL_MESSAGES[3]["content"],
+    ]
+    assert "A" * 5000 not in json.dumps(result["input"])
+    assert [tool["name"] for tool in result["tools"]] == ["get_weather", HEADROOM_RETRIEVE_TOOL_NAME]
+
+
 @pytest.mark.asyncio
 async def test_apply_guardrail_http_error_raises():
     guardrail = _make_guardrail()

From ef96af51211318ea9bbc9d49947c75b71a3f00b5 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 29 Aug 2026 14:34:42 -0700
Subject: [PATCH 02/42] fix(headroom): resolve CCR retrieval on streaming
 /v1/responses

Streaming /v1/responses requests that carried Headroom's retrieve tool were
sent upstream as streams, so the model's headroom_retrieve function_call was
streamed straight back to a client that never declared the tool and the
retrieval never resolved. Chat completions already avoid this by converting
the request to non-stream in the pre-call deployment hook, letting the
agentic loop resolve the retrieve, and fake-streaming the final answer.

The hook now converts responses call types too, the responses handler wraps
the resolved result as a fake stream whenever any interception converted the
stream (shared converted_stream_requested helper instead of per-integration
key checks), and the follow-up request filter drops every non-code-interpreter
interception key through is_interception_internal_key.

Resolves LIT-6481
---
 litellm/llms/custom_httpx/llm_http_handler.py |  11 +-
 .../guardrail_hooks/headroom/headroom.py      |   5 +-
 litellm/types/integrations/custom_logger.py   |   8 ++
 .../guardrail_hooks/test_headroom.py          | 117 +++++++++++++++++-
 4 files changed, 132 insertions(+), 9 deletions(-)

diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py
index 573ba85416f..52a91a1d06d 100644
--- a/litellm/llms/custom_httpx/llm_http_handler.py
+++ b/litellm/llms/custom_httpx/llm_http_handler.py
@@ -97,9 +97,12 @@ from litellm.types.containers.main import (
 )
 from litellm.types.files import StreamingMediaUploadConfig, TwoStepFileUploadConfig
 from litellm.types.integrations.custom_logger import (
+    NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES,
     AgenticLoopPlan,
     AgenticLoopRequestPatch,
     AgenticLoopSafetyError,
+    converted_stream_requested,
+    is_interception_internal_key,
 )
 from litellm.types.llms.anthropic_messages.anthropic_response import (
     AnthropicMessagesResponse,
@@ -2932,10 +2935,7 @@ class BaseLLMHTTPHandler:
         )
 
         result: Final = final_response if final_response is not None else initial_response
-        interception_converted_stream: Final = litellm_params.get(
-            "_code_interpreter_interception_converted_stream"
-        ) or litellm_params.get("_websearch_interception_converted_stream")
-        if interception_converted_stream and not litellm_params.get("_agentic_loop_depth"):
+        if converted_stream_requested(litellm_params) and not litellm_params.get("_agentic_loop_depth"):
             return self._wrap_responses_response_as_fake_stream(
                 result=result,
                 model=model,
@@ -5399,8 +5399,7 @@ class BaseLLMHTTPHandler:
         kwargs_for_followup: Final = {
             k: v
             for k, v in kwargs.items()
-            if not k.startswith("_websearch_interception")
-            and not k.startswith("_compression_interception")
+            if not is_interception_internal_key(k, prefixes=NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES)
             and k != "_code_interpreter_interception_converted_stream"
             and k not in internal_keys
             and k not in optional_params
diff --git a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py
index e2d2fffb2df..08753bdaed9 100644
--- a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py
+++ b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py
@@ -49,6 +49,9 @@ if TYPE_CHECKING:
     from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel
 
 BYPASS_HEADER: Final = "x-headroom-bypass"
+_STREAM_CONVERTIBLE_CALL_TYPES: Final = frozenset(
+    (CallTypes.completion, CallTypes.acompletion, CallTypes.responses, CallTypes.aresponses)
+)
 HEADROOM_RETRIEVE_TOOL_NAME: Final = "headroom_retrieve"
 _HASH_PATTERN: Final = re.compile(r"hash=([a-f0-9]{24})")
 _HASH_CACHE_TTL_SECONDS: Final = 15 * 60
@@ -724,7 +727,7 @@ class HeadroomGuardrail(CustomGuardrail):
     ) -> dict[str, Any] | None:  # mutable-ok: overrides CustomLogger hook whose contract is a plain dict
         base_result: Final = await super().async_pre_call_deployment_hook(kwargs, call_type)
         effective: Final = base_result if base_result is not None else kwargs
-        if call_type not in (CallTypes.completion, CallTypes.acompletion):
+        if call_type not in _STREAM_CONVERTIBLE_CALL_TYPES:
             return base_result
         if not effective.get("stream"):
             return base_result
diff --git a/litellm/types/integrations/custom_logger.py b/litellm/types/integrations/custom_logger.py
index 9a714e1724e..5de58a20242 100644
--- a/litellm/types/integrations/custom_logger.py
+++ b/litellm/types/integrations/custom_logger.py
@@ -1,3 +1,4 @@
+from collections.abc import Mapping
 from typing import Any, Final
 
 from pydantic import BaseModel, Field
@@ -29,6 +30,13 @@ def is_interception_internal_key(
     return any(key.startswith(prefix) for prefix in prefixes)
 
 
+CONVERTED_STREAM_KEYS: Final = frozenset(f"{prefix}_converted_stream" for prefix in INTERCEPTION_INTERNAL_PREFIXES)
+
+
+def converted_stream_requested(params: Mapping[str, object]) -> bool:
+    return any(bool(params.get(key)) for key in CONVERTED_STREAM_KEYS)
+
+
 class AgenticLoopSafetyError(ValueError):
     """
     Raised when an agentic-loop safety rail refuses a rerun.
diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py
index 37b9852d3e1..e1c4cc0922a 100644
--- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py
+++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py
@@ -1984,6 +1984,58 @@ def _openai_text_payload(content: str) -> dict:
     return _openai_completion_payload({"role": "assistant", "content": content}, "stop")
 
 
+def _responses_retrieve_tool_definition() -> dict:
+    return {"type": "function", **_retrieve_tool_definition()["function"]}
+
+
+def _openai_responses_payload(output_item: dict) -> dict:
+    return {
+        "id": "resp_ccr",
+        "object": "response",
+        "created_at": 1700000000,
+        "status": "completed",
+        "model": "gpt-4o",
+        "output": [output_item],
+        "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
+        "parallel_tool_calls": True,
+        "tool_choice": "auto",
+        "tools": [],
+        "error": None,
+        "incomplete_details": None,
+        "instructions": None,
+        "metadata": {},
+        "temperature": 1.0,
+        "top_p": 1.0,
+        "text": {"format": {"type": "text"}},
+        "truncation": "disabled",
+    }
+
+
+def _openai_responses_retrieve_call_payload() -> dict:
+    return _openai_responses_payload(
+        {
+            "type": "function_call",
+            "id": "fc_ccr",
+            "call_id": "call_ccr",
+            "name": HEADROOM_RETRIEVE_TOOL_NAME,
+            "arguments": json.dumps({"hash": CCR_HASH}),
+            "status": "completed",
+        }
+    )
+
+
+def _openai_responses_text_payload(text: str) -> dict:
+    return _openai_responses_payload(
+        {
+            "type": "message",
+            "id": "msg_ccr",
+            "role": "assistant",
+            "status": "completed",
+            "content": [{"type": "output_text", "text": text, "annotations": []}],
+        }
+    )
+
+
 @pytest.mark.parametrize(
     "call_type, stream, tools, expect_conversion",
     [
@@ -1992,12 +2044,14 @@ def _openai_text_payload(content: str) -> dict:
         (CallTypes.acompletion, False, [_retrieve_tool_definition()], False),
         (CallTypes.acompletion, True, [{"type": "function", "function": {"name": "get_weather"}}], False),
         (CallTypes.acompletion, True, None, False),
-        (CallTypes.aresponses, True, [_retrieve_tool_definition()], False),
+        (CallTypes.aresponses, True, [_retrieve_tool_definition()], True),
+        (CallTypes.responses, True, [_responses_retrieve_tool_definition()], True),
+        (CallTypes.aresponses, False, [_retrieve_tool_definition()], False),
         (CallTypes.anthropic_messages, True, [_retrieve_tool_definition()], False),
     ],
 )
 @pytest.mark.asyncio
-async def test_pre_call_deployment_hook_converts_stream_only_for_ccr_chat_completions(
+async def test_pre_call_deployment_hook_converts_stream_only_for_ccr_chat_completions_and_responses(
     guardrail: HeadroomGuardrail,
     call_type: CallTypes,
     stream: bool,
@@ -2128,6 +2182,65 @@ async def test_streaming_chat_completion_resolves_ccr_retrieval_end_to_end(
     assert not any(key.startswith("_headroom_interception") for key in followup_body)
 
 
+@pytest.mark.asyncio
+async def test_streaming_responses_resolves_ccr_retrieval_end_to_end(
+    guardrail: HeadroomGuardrail,
+    respx_mock: respx.MockRouter,
+    monkeypatch: pytest.MonkeyPatch,
+):
+    """Regression test for LIT-6481: streaming /v1/responses must resolve the
+    retrieve tool call server-side exactly like streaming /chat/completions does,
+    instead of streaming a headroom_retrieve function_call to the client."""
+    original_content = "the full uncompressed document"
+    final_answer = "the document says hello"
+    guardrail._issued_hashes_by_call_id["ccr-call-id"] = (
+        frozenset({CCR_HASH}),
+        time.monotonic() + 999,
+    )
+
+    monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
+    monkeypatch.setattr(litellm, "callbacks", [guardrail])
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    upstream = respx_mock.post("https://api.openai.com/v1/responses").mock(
+        side_effect=[
+            httpx.Response(200, json=_openai_responses_retrieve_call_payload()),
+            httpx.Response(200, json=_openai_responses_text_payload(final_answer)),
+        ]
+    )
+
+    with patch.object(
+        guardrail.async_handler,
+        "get",
+        new_callable=AsyncMock,
+        return_value=_make_retrieve_response(original_content),
+    ) as mock_get:
+        response = await litellm.aresponses(
+            model="openai/gpt-4o",
+            input=[{"role": "user", "content": f"summarize hash={CCR_HASH}"}],
+            tools=[_responses_retrieve_tool_definition()],
+            stream=True,
+            litellm_call_id="ccr-call-id",
+        )
+        events = [event async for event in response]
+
+    streamed_text = "".join(
+        getattr(event, "delta", "") for event in events if getattr(event, "type", None) == "response.output_text.delta"
+    )
+    assert streamed_text == final_answer
+    assert not any("function_call" in str(getattr(event, "type", "")) for event in events)
+    assert not any(
+        getattr(getattr(event, "item", None), "type", None) == "function_call" for event in events
+    )
+    mock_get.assert_called_once()
+    assert CCR_HASH in (mock_get.call_args.kwargs.get("url") or mock_get.call_args.args[0])
+
+    assert len(upstream.calls) == 2
+    followup_body = json.loads(upstream.calls[1].request.content)
+    assert not followup_body.get("stream")
+    assert original_content in json.dumps(followup_body["input"])
+    assert not any(key.startswith("_headroom_interception") for key in followup_body)
+
+
 # ---------------------------------------------------------------------------
 # LIT-5018: the turn the model is being asked to act on is never compressed.
 #

From 201ada99828bd7e62de465cd4ceed97002af99a6 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 29 Aug 2026 14:41:08 -0700
Subject: [PATCH 03/42] fix(headroom): check the responses converted-stream
 flag on the agentic kwargs dict

---
 litellm/llms/custom_httpx/llm_http_handler.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py
index 52a91a1d06d..ad25a44ab7f 100644
--- a/litellm/llms/custom_httpx/llm_http_handler.py
+++ b/litellm/llms/custom_httpx/llm_http_handler.py
@@ -2921,6 +2921,7 @@ class BaseLLMHTTPHandler:
             logging_obj=logging_obj,
         )
 
+        agentic_kwargs: Final = dict(litellm_params)
         final_response: Final = await self._call_agentic_completion_hooks(
             response=initial_response,
             model=model,
@@ -2930,12 +2931,12 @@ class BaseLLMHTTPHandler:
             logging_obj=logging_obj,
             stream=False,
             custom_llm_provider=custom_llm_provider,
-            kwargs=dict(litellm_params),
+            kwargs=agentic_kwargs,
             api_surface="responses",
         )
 
         result: Final = final_response if final_response is not None else initial_response
-        if converted_stream_requested(litellm_params) and not litellm_params.get("_agentic_loop_depth"):
+        if converted_stream_requested(agentic_kwargs) and not agentic_kwargs.get("_agentic_loop_depth"):
             return self._wrap_responses_response_as_fake_stream(
                 result=result,
                 model=model,

From 1695b7f7b1931bcc59c50975c8ccdbb460ad7c04 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 29 Aug 2026 15:02:20 -0700
Subject: [PATCH 04/42] fix(headroom): fake-stream converted sync /v1/responses
 calls too

The sync response_api_handler agentic branch returned the completed ResponsesAPIResponse for a request the Headroom guardrail had converted from streaming, so litellm.responses(stream=True) handed callers a non-iterable object. Wrap it in the same fake stream the async path uses
---
 litellm/llms/custom_httpx/llm_http_handler.py | 14 ++++-
 .../guardrail_hooks/test_headroom.py          | 52 +++++++++++++++++++
 2 files changed, 64 insertions(+), 2 deletions(-)

diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py
index ad25a44ab7f..2ec5377e217 100644
--- a/litellm/llms/custom_httpx/llm_http_handler.py
+++ b/litellm/llms/custom_httpx/llm_http_handler.py
@@ -2744,6 +2744,7 @@ class BaseLLMHTTPHandler:
         )
 
         if self._has_agentic_completion_hook(logging_obj):
+            agentic_kwargs: Final = dict(litellm_params)
             final_response: Final = run_async_function(
                 self._call_agentic_completion_hooks,
                 response=initial_response,
@@ -2754,10 +2755,19 @@ class BaseLLMHTTPHandler:
                 logging_obj=logging_obj,
                 stream=False,
                 custom_llm_provider=custom_llm_provider,
-                kwargs=dict(litellm_params),
+                kwargs=agentic_kwargs,
                 api_surface="responses",
             )
-            return final_response if final_response is not None else initial_response
+            result: Final = final_response if final_response is not None else initial_response
+            if converted_stream_requested(agentic_kwargs) and not agentic_kwargs.get("_agentic_loop_depth"):
+                return self._wrap_responses_response_as_fake_stream(
+                    result=result,
+                    model=model,
+                    responses_api_provider_config=responses_api_provider_config,
+                    logging_obj=logging_obj,
+                    custom_llm_provider=custom_llm_provider,
+                )
+            return result
 
         return initial_response
 
diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py
index e1c4cc0922a..c297f593617 100644
--- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py
+++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py
@@ -2241,6 +2241,58 @@ async def test_streaming_responses_resolves_ccr_retrieval_end_to_end(
     assert not any(key.startswith("_headroom_interception") for key in followup_body)
 
 
+def test_sync_streaming_responses_resolves_ccr_retrieval_end_to_end(
+    guardrail: HeadroomGuardrail,
+    respx_mock: respx.MockRouter,
+    monkeypatch: pytest.MonkeyPatch,
+):
+    """The synchronous responses() path converts the stream the same way, so it
+    must hand back a stream iterator with the resolved answer rather than the
+    completed response object."""
+    original_content = "the full uncompressed document"
+    final_answer = "the document says hello"
+    guardrail._issued_hashes_by_call_id["ccr-call-id"] = (
+        frozenset({CCR_HASH}),
+        time.monotonic() + 999,
+    )
+
+    monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    monkeypatch.setattr(litellm, "callbacks", [guardrail])
+    upstream = respx_mock.post("https://api.openai.com/v1/responses").mock(
+        side_effect=[
+            httpx.Response(200, json=_openai_responses_retrieve_call_payload()),
+            httpx.Response(200, json=_openai_responses_text_payload(final_answer)),
+        ]
+    )
+
+    with patch.object(
+        guardrail.async_handler,
+        "get",
+        new_callable=AsyncMock,
+        return_value=_make_retrieve_response(original_content),
+    ) as mock_get:
+        response = litellm.responses(
+            model="openai/gpt-4o",
+            input=[{"role": "user", "content": f"summarize hash={CCR_HASH}"}],
+            tools=[_responses_retrieve_tool_definition()],
+            stream=True,
+            litellm_call_id="ccr-call-id",
+        )
+        events = list(response)
+
+    streamed_text = "".join(
+        getattr(event, "delta", "") for event in events if getattr(event, "type", None) == "response.output_text.delta"
+    )
+    assert streamed_text == final_answer
+    assert not any(
+        getattr(getattr(event, "item", None), "type", None) == "function_call" for event in events
+    )
+    mock_get.assert_called_once()
+    assert len(upstream.calls) == 2
+    assert not json.loads(upstream.calls[1].request.content).get("stream")
+
+
 # ---------------------------------------------------------------------------
 # LIT-5018: the turn the model is being asked to act on is never compressed.
 #

From 6bd3699d4338193b23f8e937061cbab7c4af2e49 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 29 Aug 2026 16:39:14 -0700
Subject: [PATCH 05/42] fix(responses): keep guardrailed input items and bridge
 stream usage intact

- _write_back_structured_messages now patches only the rewritten rows back
  into the original input items, so reasoning items (encrypted_content),
  function_call ids, and web_search_call items survive a guardrail rewrite
  verbatim; rewrites that cannot be row-mapped fall back to the previous
  full conversion
- the responses bridge stream snapshot restores usage hidden in
  _hidden_params when stream_options is unset, so converted fake streams
  report real input_tokens instead of 0
---
 litellm/llms/custom_httpx/llm_http_handler.py |   4 +-
 .../guardrail_translation/handler.py          | 135 +++++++++++++++++-
 .../streaming_iterator.py                     |   6 +
 ...test_openai_responses_guardrail_handler.py | 129 ++++++++++++++++-
 .../guardrail_hooks/test_headroom.py          |   2 +-
 .../test_streaming_iterator_transformation.py |  15 ++
 6 files changed, 279 insertions(+), 12 deletions(-)

diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py
index 2ec5377e217..fcb988705eb 100644
--- a/litellm/llms/custom_httpx/llm_http_handler.py
+++ b/litellm/llms/custom_httpx/llm_http_handler.py
@@ -2744,7 +2744,7 @@ class BaseLLMHTTPHandler:
         )
 
         if self._has_agentic_completion_hook(logging_obj):
-            agentic_kwargs: Final = dict(litellm_params)
+            agentic_kwargs: Final = dict(litellm_params)  # mutable-ok: agentic hooks mutate kwargs in place
             final_response: Final = run_async_function(
                 self._call_agentic_completion_hooks,
                 response=initial_response,
@@ -2931,7 +2931,7 @@ class BaseLLMHTTPHandler:
             logging_obj=logging_obj,
         )
 
-        agentic_kwargs: Final = dict(litellm_params)
+        agentic_kwargs: Final = dict(litellm_params)  # mutable-ok: agentic hooks mutate kwargs in place
         final_response: Final = await self._call_agentic_completion_hooks(
             response=initial_response,
             model=model,
diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py
index 4320edb8414..8c014feeded 100644
--- a/litellm/llms/openai/responses/guardrail_translation/handler.py
+++ b/litellm/llms/openai/responses/guardrail_translation/handler.py
@@ -28,7 +28,8 @@ Output: response.output is List[GenericResponseOutputItem] where each has:
     - text: str
 """
 
-from collections.abc import Sequence
+from collections.abc import Mapping, Sequence
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Union, cast
 
 from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall
@@ -50,6 +51,7 @@ from litellm.types.llms.openai import (
     ChatCompletionToolCallChunk,
     ChatCompletionToolParam,
     OpenAIMcpServerTool,
+    ResponsesAPIOptionalRequestParams,
     ResponsesAPIStreamEvents,
 )
 from litellm.types.responses.main import (
@@ -81,6 +83,119 @@ class ResponsesStreamChunk(TypedDict, total=False):
     text: ReadOnly[str]
 
 
+_PATCHABLE_ITEM_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
+    {"function_call_output": "output", "message": "content"}
+)
+
+_EMPTY_RESPONSES_REQUEST: Final[ResponsesAPIOptionalRequestParams] = {}
+
+
+def _item_rewrite_field(item: Mapping[str, object]) -> str | None:
+    item_type: Final = item.get("type")
+    if item_type is None:
+        return "content" if "content" in item else None
+    if not isinstance(item_type, str):
+        return None
+    return _PATCHABLE_ITEM_FIELDS.get(item_type)
+
+
+def _rewritten_input_item(item: Mapping[str, object], rewritten: object) -> Mapping[str, object] | None:
+    field: Final = _item_rewrite_field(item)
+    if field is None or not isinstance(rewritten, Mapping):
+        return None
+    rewritten_content: Final = rewritten.get("content")
+    if isinstance(item.get(field), str) and isinstance(rewritten_content, str):
+        return {**item, field: rewritten_content}  # mutable-ok: request input items must stay JSON-plain dicts
+    rewritten_row: Final = cast("AllMessageValues", rewritten)  # cast-ok: guardrails hand back chat-shaped rows
+    converted_items, _ = LiteLLMResponsesTransformationHandler().convert_chat_completion_messages_to_responses_api(
+        [rewritten_row]  # mutable-ok: converter signature takes a list
+    )
+    if len(converted_items) != 1 or not isinstance(converted_items[0], Mapping):
+        return None
+    first_converted: Final = cast("Mapping[str, object]", converted_items[0])  # cast-ok: isinstance-checked above
+    converted_value: Final = first_converted.get(field)
+    if converted_value is None:
+        return None
+    return {**item, field: converted_value}  # mutable-ok: request input items must stay JSON-plain dicts
+
+
+def _input_item_provenance(
+    raw_input: Sequence[object],
+    expected_messages: Sequence[object],
+) -> tuple[Mapping[int, int], frozenset[int]] | None:
+    if not all(isinstance(item, Mapping) for item in raw_input):
+        return None
+    prefixes: Final = tuple(
+        LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
+            input=cast("ResponseInputParam", raw_input[:count]),  # cast-ok: items checked as Mappings above
+            responses_api_request=_EMPTY_RESPONSES_REQUEST,
+        )
+        for count in range(len(raw_input) + 1)
+    )
+    if tuple(prefixes[-1]) != tuple(expected_messages):
+        return None
+    item_for_message: Final = MappingProxyType(
+        {
+            message_index: item_index
+            for item_index in range(len(raw_input))
+            for message_index in range(len(prefixes[item_index]), len(prefixes[item_index + 1]))
+        }
+    )
+    tainted: Final = frozenset(
+        message_index
+        for item_index in range(len(raw_input))
+        for message_index in range(len(prefixes[item_index]))
+        if prefixes[item_index + 1][message_index] != prefixes[item_index][message_index]
+    )
+    return item_for_message, tainted
+
+
+def _patch_rewritten_rows_into_input(
+    data: dict,
+    original_messages: Sequence[object],
+    structured_messages: Sequence[object],
+) -> bool:
+    raw_input: Final = data.get("input")
+    if not isinstance(raw_input, list) or len(original_messages) != len(structured_messages):
+        return False
+    offset: Final = 1 if data.get("instructions") else 0
+    provenance: Final = _input_item_provenance(raw_input, tuple(original_messages)[offset:])
+    if provenance is None:
+        return False
+    item_for_message, tainted = provenance
+    changed: Final = tuple(
+        (index, rewritten)
+        for index, (original, rewritten) in enumerate(zip(original_messages, structured_messages))
+        if original != rewritten
+    )
+    instruction_rewrites: Final = tuple(rewritten for index, rewritten in changed if index < offset)
+    rewritten_instructions: Final = (
+        instruction_rewrites[0].get("content")
+        if instruction_rewrites and isinstance(instruction_rewrites[0], Mapping)
+        else None
+    )
+    if instruction_rewrites and not isinstance(rewritten_instructions, str):
+        return False
+    body_changes: Final = tuple((index - offset, rewritten) for index, rewritten in changed if index >= offset)
+    if any(message_index in tainted or message_index not in item_for_message for message_index, _ in body_changes):
+        return False
+    replacements: Final = MappingProxyType(
+        {
+            item_for_message[message_index]: _rewritten_input_item(
+                cast("Mapping[str, object]", raw_input[item_for_message[message_index]]),  # cast-ok: checked Mappings
+                rewritten,
+            )
+            for message_index, rewritten in body_changes
+        }
+    )
+    if len(replacements) != len(body_changes) or any(item is None for item in replacements.values()):
+        return False
+    data["input"] = [replacements.get(index, item) for index, item in enumerate(raw_input)]  # mutable-ok: JSON body
+    if isinstance(rewritten_instructions, str):
+        data["instructions"] = rewritten_instructions
+    return True
+
+
 class OpenAIResponsesHandler(BaseTranslation):
     """
     Handler for processing OpenAI Responses API with guardrails.
@@ -155,9 +270,9 @@ class OpenAIResponsesHandler(BaseTranslation):
                 guardrailed_structured_messages is not None
                 and guardrailed_structured_messages is not structured_messages
             ):
-                self._write_back_structured_messages(data, guardrailed_structured_messages)
+                self._write_back_structured_messages(data, structured_messages or (), guardrailed_structured_messages)
             else:
-                guardrailed_texts = guardrailed_inputs.get("texts", [])
+                guardrailed_texts = guardrailed_inputs.get("texts") or ()
                 data["input"] = guardrailed_texts[0] if guardrailed_texts else input_data
             self._apply_guardrailed_tools_to_data(data, original_tools, guardrailed_inputs.get("tools"))
             verbose_proxy_logger.debug("OpenAI Responses API: Processed string input")
@@ -217,12 +332,12 @@ class OpenAIResponsesHandler(BaseTranslation):
                 guardrailed_structured_messages is not None
                 and guardrailed_structured_messages is not structured_messages
             ):
-                self._write_back_structured_messages(data, guardrailed_structured_messages)
+                self._write_back_structured_messages(data, structured_messages or (), guardrailed_structured_messages)
             else:
                 # Step 3: Map guardrail responses back to original input structure
                 await self._apply_guardrail_responses_to_input(
                     messages=input_data,
-                    responses=guardrailed_inputs.get("texts", []),
+                    responses=guardrailed_inputs.get("texts", []),  # mutable-ok: callee signature takes a list
                     task_mappings=task_mappings,
                 )
 
@@ -231,10 +346,16 @@ class OpenAIResponsesHandler(BaseTranslation):
         return data
 
     @staticmethod
-    def _write_back_structured_messages(data: dict, structured_messages: Sequence[AllMessageValues]) -> None:
+    def _write_back_structured_messages(
+        data: dict,
+        original_messages: Sequence[object],
+        structured_messages: Sequence[AllMessageValues],
+    ) -> None:
+        if _patch_rewritten_rows_into_input(data, original_messages, structured_messages):
+            return
         input_items, instructions = (
             LiteLLMResponsesTransformationHandler().convert_chat_completion_messages_to_responses_api(
-                list(structured_messages)
+                list(structured_messages)  # mutable-ok: converter signature takes a list
             )
         )
         data["input"] = input_items
diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py
index 8b1eeb30306..f32599e8098 100644
--- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py
+++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py
@@ -557,6 +557,12 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
         hidden_params: Final = getattr(chunk, "_hidden_params", None)
         if hidden_params is not None:
             chunk_dict["_hidden_params"] = dict(hidden_params) if isinstance(hidden_params, dict) else hidden_params
+        if (
+            chunk_dict.get("usage") is None
+            and isinstance(hidden_params, dict)
+            and hidden_params.get("usage") is not None
+        ):
+            chunk_dict["usage"] = hidden_params["usage"]
         return chunk_dict
 
     def create_reasoning_summary_text_done_event(
diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py
index 63f8babf760..87449a4b1e8 100644
--- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py
+++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py
@@ -1253,6 +1253,43 @@ class StructuredRewriteGuardrail(CustomGuardrail):
         return {**inputs, "structured_messages": rewritten}
 
 
+class ToolOutputRewriteGuardrail(CustomGuardrail):
+    """Guardrail that compresses the first tool-result row, the way Headroom does."""
+
+    async def apply_guardrail(
+        self,
+        inputs: GenericGuardrailAPIInputs,
+        request_data: dict,
+        input_type: Literal["request", "response"],
+        logging_obj: Optional[Any] = None,
+    ) -> GenericGuardrailAPIInputs:
+        messages = list(inputs.get("structured_messages") or [])
+        first_tool = next(i for i, m in enumerate(messages) if isinstance(m, dict) and m.get("role") == "tool")
+        rewritten = [
+            {**m, "content": COMPRESSED_MARKER} if i == first_tool else m for i, m in enumerate(messages)
+        ]
+        return {**inputs, "structured_messages": rewritten}
+
+
+class DroppingRewriteGuardrail(CustomGuardrail):
+    """Guardrail that rewrites the first user row and drops the last row, so the
+    rewrite can only land through the full-conversion fallback."""
+
+    async def apply_guardrail(
+        self,
+        inputs: GenericGuardrailAPIInputs,
+        request_data: dict,
+        input_type: Literal["request", "response"],
+        logging_obj: Optional[Any] = None,
+    ) -> GenericGuardrailAPIInputs:
+        messages = list(inputs.get("structured_messages") or [])
+        first_user = next(i for i, m in enumerate(messages) if isinstance(m, dict) and m.get("role") == "user")
+        rewritten = [
+            {**m, "content": COMPRESSED_MARKER} if i == first_user else m for i, m in enumerate(messages)
+        ]
+        return {**inputs, "structured_messages": rewritten[:-1]}
+
+
 def _texts(item: dict) -> list[str]:
     content = item.get("content")
     if isinstance(content, str):
@@ -1296,7 +1333,93 @@ class TestStructuredMessagesWriteBack:
         assert "instructions" not in result
 
     @pytest.mark.asyncio
-    async def test_developer_item_survives_write_back_as_input_text(self):
+    async def test_developer_item_preserved_verbatim_by_row_patch(self):
+        handler = OpenAIResponsesHandler()
+        developer_item = {"role": "developer", "content": "Always answer in French."}
+        data = {
+            "model": "gpt-5.6",
+            "input": [
+                developer_item,
+                {"role": "user", "content": "memo " * 400},
+                {"role": "user", "content": "What is the codename?"},
+            ],
+        }
+
+        result = await handler.process_input_messages(data, StructuredRewriteGuardrail())
+
+        assert result["input"][0] is developer_item
+        assert developer_item["content"] == "Always answer in French."
+        assert _texts(result["input"][1]) == [COMPRESSED_MARKER]
+        assert _texts(result["input"][2]) == ["What is the codename?"]
+
+    @pytest.mark.asyncio
+    async def test_reasoning_and_function_call_items_survive_tool_output_compression(self):
+        handler = OpenAIResponsesHandler()
+        reasoning_item = {
+            "id": "rs_123",
+            "type": "reasoning",
+            "summary": [],
+            "encrypted_content": "gAAAAA-signed-reasoning",
+        }
+        function_call_item = {
+            "id": "fc_123",
+            "type": "function_call",
+            "call_id": "call_abc",
+            "name": "read_document",
+            "arguments": '{"path": "memo.txt"}',
+            "status": "completed",
+        }
+        data = {
+            "model": "gpt-5.6",
+            "instructions": "Answer from the memo only.",
+            "input": [
+                reasoning_item,
+                function_call_item,
+                {"type": "function_call_output", "call_id": "call_abc", "output": "memo " * 400},
+                {"role": "user", "content": "What is the codename?"},
+            ],
+        }
+
+        result = await handler.process_input_messages(data, ToolOutputRewriteGuardrail())
+
+        assert result["instructions"] == "Answer from the memo only."
+        assert result["input"][0] is reasoning_item
+        assert reasoning_item["encrypted_content"] == "gAAAAA-signed-reasoning"
+        assert result["input"][1] is function_call_item
+        assert function_call_item["id"] == "fc_123"
+        assert result["input"][2] == {
+            "type": "function_call_output",
+            "call_id": "call_abc",
+            "output": COMPRESSED_MARKER,
+        }
+        assert result["input"][3] == {"role": "user", "content": "What is the codename?"}
+
+    @pytest.mark.asyncio
+    async def test_web_search_call_item_preserved_verbatim(self):
+        handler = OpenAIResponsesHandler()
+        web_search_item = {
+            "id": "ws_123",
+            "type": "web_search_call",
+            "status": "completed",
+            "action": {"type": "search", "query": "codename memo"},
+        }
+        data = {
+            "model": "gpt-5.6",
+            "input": [
+                web_search_item,
+                {"role": "user", "content": "memo " * 400},
+                {"role": "user", "content": "What is the codename?"},
+            ],
+        }
+
+        result = await handler.process_input_messages(data, StructuredRewriteGuardrail())
+
+        assert result["input"][0] is web_search_item
+        assert _texts(result["input"][1]) == [COMPRESSED_MARKER]
+        assert _texts(result["input"][2]) == ["What is the codename?"]
+
+    @pytest.mark.asyncio
+    async def test_row_count_change_falls_back_to_full_conversion(self):
         handler = OpenAIResponsesHandler()
         data = {
             "model": "gpt-5.6",
@@ -1307,10 +1430,12 @@ class TestStructuredMessagesWriteBack:
             ],
         }
 
-        result = await handler.process_input_messages(data, StructuredRewriteGuardrail())
+        result = await handler.process_input_messages(data, DroppingRewriteGuardrail())
 
+        assert len(result["input"]) == 2
         developer = next(item for item in result["input"] if item.get("role") == "developer")
         assert developer["content"] == [{"type": "input_text", "text": "Always answer in French."}]
+        assert _texts(next(item for item in result["input"] if item.get("role") == "user")) == [COMPRESSED_MARKER]
 
     @pytest.mark.asyncio
     async def test_same_inputs_object_back_keeps_the_text_mapping(self):
diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py
index c297f593617..22c4f46e3fb 100644
--- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py
+++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py
@@ -979,7 +979,7 @@ async def test_responses_request_sends_compressed_input_and_retrieve_tool_upstre
         result = await OpenAIResponsesHandler().process_input_messages(data=data, guardrail_to_apply=guardrail)
 
     assert result["instructions"] == ORIGINAL_MESSAGES[0]["content"]
-    assert [item["content"][0]["text"] for item in result["input"]] == [
+    assert [item["content"] for item in result["input"]] == [
         COMPRESSED_MESSAGES_WITH_HASH[0]["content"],
         ORIGINAL_MESSAGES[2]["content"],
         ORIGINAL_MESSAGES[3]["content"],
diff --git a/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py
index 823f656ddc5..3ffd44b2fbe 100644
--- a/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py
+++ b/tests/test_litellm/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py
@@ -24,6 +24,7 @@ from litellm.types.utils import (
     ModelResponse,
     ModelResponseStream,
     StreamingChoices,
+    Usage,
 )
 
 CHAT_COMPLETION_ID = "chatcmpl-77d33d09-effa-4cd2-9c0d-c742d4358256"
@@ -523,3 +524,17 @@ async def test_streaming_response_id_falls_back_when_upstream_yields_nothing():
     assert response_ids
     assert len(set(response_ids)) == 1
     assert response_ids[0].startswith("resp_")
+
+
+def test_completed_event_restores_usage_hidden_by_stream_options_none():
+    final_chunk = _chunk("", finish_reason="stop")
+    final_chunk._hidden_params = {"usage": Usage(prompt_tokens=117, completion_tokens=5, total_tokens=122)}
+    iterator = _build_iterator([_chunk("the document says hello"), final_chunk])
+
+    events = list(iterator)
+
+    completed = next(
+        event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED
+    )
+    assert completed.response.usage.input_tokens == 117
+    assert completed.response.usage.output_tokens == 5

From 98a9a7e525590d9af15d26ea3db0257d14b42d43 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 29 Aug 2026 21:12:41 -0700
Subject: [PATCH 06/42] fix(streaming): carry hidden usage on the async
 fake-stream final chunk

The sync __next__ exhaustion branch stores calculate_total_usage() in the
final chunk's _hidden_params when stream_options is None, but the async
__anext__ sibling branch never did. Converted (fake) streams, like the ones
the Headroom guardrail produces by flipping streaming /v1/responses calls to
non-streaming, are consumed async, so their real usage never reached the
completion-to-responses bridge and it token-counted from scratch, reporting
input_tokens=0. Mirror the sync branch's hidden-usage block into the async
exhaustion branch and add a regression test that async-iterates a
CustomStreamWrapper over a MockResponseIterator and asserts the final chunk
carries the mock response's usage.
---
 .../litellm_core_utils/streaming_handler.py   |  3 ++
 .../test_streaming_handler.py                 | 37 +++++++++++++++++++
 2 files changed, 40 insertions(+)

diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py
index 1e0b778d244..d9641a6306b 100644
--- a/litellm/litellm_core_utils/streaming_handler.py
+++ b/litellm/litellm_core_utils/streaming_handler.py
@@ -2336,6 +2336,9 @@ class CustomStreamWrapper:
         else:
             self.sent_last_chunk = True
             processed_chunk: Final = self.finish_reason_handler()
+            if self.stream_options is None:
+                usage: Final = calculate_total_usage(chunks=self.chunks)
+                processed_chunk._hidden_params["usage"] = usage  # pyright: ignore[reportPrivateUsage]  # sync parity
             # see sync __next__'s sibling branch: deliberately do NOT restore
             # here - this chunk is still this call's own data, and restoring
             # before returning it would corrupt the caller's own log
diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py
index 7f54fbfb4c2..7caf401ce38 100644
--- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py
+++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py
@@ -4692,3 +4692,40 @@ async def test_async_stream_assembled_response_keeps_vertex_traffic_type(logging
     assembled = litellm.stream_chunk_builder(chunks=received, messages=[{"role": "user", "content": "hi"}])
     assert assembled is not None
     assert assembled._hidden_params["provider_specific_fields"]["traffic_type"] == "ON_DEMAND_FLEX"
+
+
+@pytest.mark.asyncio
+async def test_async_fake_stream_final_chunk_carries_hidden_usage(logging_obj: Logging):
+    from litellm.llms.base_llm.base_model_iterator import MockResponseIterator
+    from litellm.types.utils import ModelResponse
+
+    model_response = ModelResponse(
+        id="chatcmpl-fake-stream",
+        model="my-random-model",
+        choices=[
+            {
+                "index": 0,
+                "message": {"role": "assistant", "content": "hello world"},
+                "finish_reason": "stop",
+            }
+        ],
+    )
+    model_response.usage = Usage(prompt_tokens=1234, completion_tokens=7, total_tokens=1241)
+
+    wrapper = CustomStreamWrapper(
+        completion_stream=MockResponseIterator(model_response=model_response),
+        model="my-random-model",
+        custom_llm_provider="anthropic",
+        logging_obj=logging_obj,
+    )
+
+    final_chunk = None
+    async for chunk in wrapper:
+        final_chunk = chunk
+
+    assert final_chunk is not None
+    hidden_usage = final_chunk._hidden_params.get("usage")
+    assert hidden_usage is not None
+    assert hidden_usage.prompt_tokens == 1234
+    assert hidden_usage.completion_tokens == 7
+    assert hidden_usage.total_tokens == 1241

From 574010a2cec368ba2fcffbc6fd8717161b33a3b1 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 29 Aug 2026 22:04:26 -0700
Subject: [PATCH 07/42] fix(responses): make guardrail input provenance O(n)
 and guard non-list structured_messages

_input_item_provenance converted every input prefix, so an n-item request paid
for n+1 full conversions. It now converts each item once, glues consecutive
function_call items (plus their trailing-assistant context) into units so the
transform's tool_call merging is reproduced inside the unit conversion, and
verifies the unit concatenation against one full conversion, bailing to the
full-conversion fallback on any mismatch. Messages from multi-item units are
tainted, which keeps parallel tool calls patchable exactly like the old prefix
pass while unpredicted merges fall back safely.

A guardrail handing back a non-list structured_messages payload (the
HiddenLayer v2 evaluation dict) previously fell through the length-mismatch
fallback and 500ed converting the dict's keys as messages. The write-back is
now skipped for non-list payloads, restoring the previous no-write-back
behavior on the Responses surface.

Also refreshes the compresr texts-mirror docstring, which still claimed the
Responses translation cannot round-trip structured_messages.
---
 .../guardrail_translation/handler.py          |  75 ++++-
 .../guardrail_hooks/compresr/compresr.py      |   7 +-
 ...test_openai_responses_guardrail_handler.py | 296 ++++++++++++++++++
 3 files changed, 365 insertions(+), 13 deletions(-)

diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py
index 8c014feeded..bd58fb4cdbc 100644
--- a/litellm/llms/openai/responses/guardrail_translation/handler.py
+++ b/litellm/llms/openai/responses/guardrail_translation/handler.py
@@ -29,6 +29,7 @@ Output: response.output is List[GenericResponseOutputItem] where each has:
 """
 
 from collections.abc import Mapping, Sequence
+from itertools import accumulate
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Union, cast
 
@@ -119,33 +120,85 @@ def _rewritten_input_item(item: Mapping[str, object], rewritten: object) -> Mapp
     return {**item, field: converted_value}  # mutable-ok: request input items must stay JSON-plain dicts
 
 
+def _is_function_call_item(item: object) -> bool:
+    return isinstance(item, Mapping) and item.get("type") in ("function_call", "custom_tool_call")
+
+
+def _last_message_role(messages: Sequence[object]) -> str | None:
+    if not messages:
+        return None
+    last: Final = messages[-1]
+    role: Final = last.get("role") if isinstance(last, Mapping) else getattr(last, "role", None)
+    return role if isinstance(role, str) else None
+
+
+def _provenance_unit_bounds(
+    raw_input: Sequence[object],
+    solo_conversions: Sequence[Sequence[object]],
+) -> tuple[tuple[int, int], ...]:
+    trailing_roles: Final = tuple(
+        accumulate(
+            (_last_message_role(messages) for messages in solo_conversions),
+            lambda previous, current: current if current is not None else previous,
+        )
+    )
+    start_indexes: Final = tuple(
+        index
+        for index in range(len(raw_input))
+        if index == 0 or not (_is_function_call_item(raw_input[index]) and trailing_roles[index - 1] == "assistant")
+    )
+    return tuple(zip(start_indexes, (*start_indexes[1:], len(raw_input))))
+
+
 def _input_item_provenance(
     raw_input: Sequence[object],
     expected_messages: Sequence[object],
 ) -> tuple[Mapping[int, int], frozenset[int]] | None:
     if not all(isinstance(item, Mapping) for item in raw_input):
         return None
-    prefixes: Final = tuple(
+    solo_conversions: Final = tuple(
         LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
-            input=cast("ResponseInputParam", raw_input[:count]),  # cast-ok: items checked as Mappings above
+            input=cast("ResponseInputParam", [item]),  # cast-ok: items checked as Mappings above
             responses_api_request=_EMPTY_RESPONSES_REQUEST,
         )
-        for count in range(len(raw_input) + 1)
+        for item in raw_input
     )
-    if tuple(prefixes[-1]) != tuple(expected_messages):
+    full_conversion: Final = tuple(
+        LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
+            input=cast("ResponseInputParam", list(raw_input)),  # cast-ok: items checked as Mappings above
+            responses_api_request=_EMPTY_RESPONSES_REQUEST,
+        )
+    )
+    if full_conversion != tuple(expected_messages):
         return None
+    units: Final = _provenance_unit_bounds(raw_input, solo_conversions)
+    unit_messages: Final = tuple(
+        tuple(solo_conversions[start])
+        if end - start == 1
+        else tuple(
+            LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
+                input=cast("ResponseInputParam", list(raw_input[start:end])),  # cast-ok: checked as Mappings above
+                responses_api_request=_EMPTY_RESPONSES_REQUEST,
+            )
+        )
+        for start, end in units
+    )
+    if tuple(message for messages in unit_messages for message in messages) != full_conversion:
+        return None
+    boundaries: Final = tuple(accumulate((len(messages) for messages in unit_messages), initial=0))
     item_for_message: Final = MappingProxyType(
         {
-            message_index: item_index
-            for item_index in range(len(raw_input))
-            for message_index in range(len(prefixes[item_index]), len(prefixes[item_index + 1]))
+            message_index: start
+            for unit_index, (start, end) in enumerate(units)
+            if end - start == 1
+            for message_index in range(boundaries[unit_index], boundaries[unit_index + 1])
         }
     )
     tainted: Final = frozenset(
         message_index
-        for item_index in range(len(raw_input))
-        for message_index in range(len(prefixes[item_index]))
-        if prefixes[item_index + 1][message_index] != prefixes[item_index][message_index]
+        for unit_index, (start, end) in enumerate(units)
+        if end - start > 1
+        for message_index in range(boundaries[unit_index], boundaries[unit_index + 1])
     )
     return item_for_message, tainted
 
@@ -351,6 +404,8 @@ class OpenAIResponsesHandler(BaseTranslation):
         original_messages: Sequence[object],
         structured_messages: Sequence[AllMessageValues],
     ) -> None:
+        if not isinstance(structured_messages, list):
+            return
         if _patch_rewritten_rows_into_input(data, original_messages, structured_messages):
             return
         input_items, instructions = (
diff --git a/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py b/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py
index 5c14d03f50e..84b328a9c60 100644
--- a/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py
+++ b/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py
@@ -916,9 +916,10 @@ class CompresrGuardrail(CustomGuardrail):
     def _mirror_texts_channel(input_texts: object, applied: _CompressionResult) -> list[object] | None:
         """Compressed content mirrored into the Responses `texts` channel.
 
-        The chat/Anthropic handlers round-trip ``structured_messages``; the
-        Responses translation cannot rebuild its input from chat messages and
-        instead writes back through ``texts``. This matches by value, so a
+        The chat/Anthropic/Responses handlers round-trip
+        ``structured_messages``; translations without that round-trip write
+        back through ``texts``, so the compressed content is mirrored there
+        too. This matches by value, so a
         replacement is applied only when it is unambiguous: one compression per
         text, and every occurrence in ``texts`` accounted for by a compressed
         target. Anything else is left uncompressed rather than risk a wrong or
diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py
index 87449a4b1e8..86c023dd7a4 100644
--- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py
+++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py
@@ -1450,3 +1450,299 @@ class TestStructuredMessagesWriteBack:
 
         assert result["input"] is original_input
         assert [_texts(item) for item in result["input"]] == [["Hello [GUARDRAILED]"], ["Again [GUARDRAILED]"]]
+
+
+class AllToolOutputsRewriteGuardrail(CustomGuardrail):
+    """Guardrail that compresses every tool-result row."""
+
+    async def apply_guardrail(
+        self,
+        inputs: GenericGuardrailAPIInputs,
+        request_data: dict,
+        input_type: Literal["request", "response"],
+        logging_obj: Optional[Any] = None,
+    ) -> GenericGuardrailAPIInputs:
+        messages = list(inputs.get("structured_messages") or [])
+        rewritten = [
+            {**m, "content": COMPRESSED_MARKER} if isinstance(m, dict) and m.get("role") == "tool" else m
+            for m in messages
+        ]
+        return {**inputs, "structured_messages": rewritten}
+
+
+class AssistantRewriteGuardrail(CustomGuardrail):
+    """Guardrail that rewrites the first assistant row's content."""
+
+    async def apply_guardrail(
+        self,
+        inputs: GenericGuardrailAPIInputs,
+        request_data: dict,
+        input_type: Literal["request", "response"],
+        logging_obj: Optional[Any] = None,
+    ) -> GenericGuardrailAPIInputs:
+        messages = list(inputs.get("structured_messages") or [])
+        first = next(i for i, m in enumerate(messages) if isinstance(m, dict) and m.get("role") == "assistant")
+        rewritten = [{**m, "content": COMPRESSED_MARKER} if i == first else m for i, m in enumerate(messages)]
+        return {**inputs, "structured_messages": rewritten}
+
+
+class DictStructuredMessagesGuardrail(CustomGuardrail):
+    """Guardrail that hands back a raw evaluation dict instead of a message list,
+    the way HiddenLayer v2 does."""
+
+    async def apply_guardrail(
+        self,
+        inputs: GenericGuardrailAPIInputs,
+        request_data: dict,
+        input_type: Literal["request", "response"],
+        logging_obj: Optional[Any] = None,
+    ) -> GenericGuardrailAPIInputs:
+        return {**inputs, "structured_messages": {"evaluation": "allowed", "messages": []}}
+
+
+def _parallel_tool_call_input() -> list:
+    return [
+        {"id": "fc_1", "type": "function_call", "call_id": "call_1", "name": "read_a", "arguments": "{}"},
+        {"id": "fc_2", "type": "function_call", "call_id": "call_2", "name": "read_b", "arguments": "{}"},
+        {"type": "function_call_output", "call_id": "call_1", "output": "memo " * 400},
+        {"type": "function_call_output", "call_id": "call_2", "output": "note " * 400},
+        {"role": "user", "content": "What is the codename?"},
+    ]
+
+
+class TestProvenancePatching:
+    """The O(n) provenance pass must keep patching rewritten rows in place for the
+    shapes real agent loops produce, and fall back safely everywhere else."""
+
+    @pytest.mark.asyncio
+    async def test_parallel_tool_call_outputs_both_patched(self):
+        handler = OpenAIResponsesHandler()
+        raw_input = _parallel_tool_call_input()
+        fc_1, fc_2 = raw_input[0], raw_input[1]
+        data = {"model": "gpt-5.6", "input": raw_input}
+
+        result = await handler.process_input_messages(data, AllToolOutputsRewriteGuardrail())
+
+        assert result["input"][0] is fc_1
+        assert result["input"][1] is fc_2
+        assert result["input"][2] == {"type": "function_call_output", "call_id": "call_1", "output": COMPRESSED_MARKER}
+        assert result["input"][3] == {"type": "function_call_output", "call_id": "call_2", "output": COMPRESSED_MARKER}
+        assert result["input"][4] == {"role": "user", "content": "What is the codename?"}
+
+    @pytest.mark.asyncio
+    async def test_assistant_turn_with_tool_call_keeps_items_verbatim(self):
+        handler = OpenAIResponsesHandler()
+        assistant_item = {"role": "assistant", "content": "Let me read the memo."}
+        function_call_item = {
+            "id": "fc_9",
+            "type": "function_call",
+            "call_id": "call_9",
+            "name": "read_document",
+            "arguments": '{"path": "memo.txt"}',
+        }
+        data = {
+            "model": "gpt-5.6",
+            "input": [
+                assistant_item,
+                function_call_item,
+                {"type": "function_call_output", "call_id": "call_9", "output": "memo " * 400},
+                {"role": "user", "content": "What is the codename?"},
+            ],
+        }
+
+        result = await handler.process_input_messages(data, ToolOutputRewriteGuardrail())
+
+        assert result["input"][0] is assistant_item
+        assert result["input"][1] is function_call_item
+        assert result["input"][2] == {"type": "function_call_output", "call_id": "call_9", "output": COMPRESSED_MARKER}
+
+    @pytest.mark.asyncio
+    async def test_rewrite_of_merged_tool_call_message_falls_back(self):
+        handler = OpenAIResponsesHandler()
+        raw_input = _parallel_tool_call_input()
+        data = {"model": "gpt-5.6", "input": raw_input}
+
+        result = await handler.process_input_messages(data, AssistantRewriteGuardrail())
+
+        assert not any(item is original for item in result["input"] for original in raw_input)
+        assistant_items = [item for item in result["input"] if item.get("role") == "assistant"]
+        assert [_texts(item) for item in assistant_items] == [[COMPRESSED_MARKER]]
+
+    @pytest.mark.asyncio
+    async def test_rewrite_of_lone_function_call_message_falls_back(self):
+        handler = OpenAIResponsesHandler()
+        data = {
+            "model": "gpt-5.6",
+            "input": [
+                {"id": "fc_1", "type": "function_call", "call_id": "call_1", "name": "read_a", "arguments": "{}"},
+                {"type": "function_call_output", "call_id": "call_1", "output": "memo memo"},
+                {"role": "user", "content": "What is the codename?"},
+            ],
+        }
+
+        raw_input = data["input"]
+        result = await handler.process_input_messages(data, AssistantRewriteGuardrail())
+
+        assert not any(item is original for item in result["input"] for original in raw_input)
+        assistant_items = [item for item in result["input"] if item.get("role") == "assistant"]
+        assert [_texts(item) for item in assistant_items] == [[COMPRESSED_MARKER]]
+
+    def test_provenance_bails_on_non_mapping_item(self):
+        from litellm.llms.openai.responses.guardrail_translation.handler import _input_item_provenance
+
+        assert _input_item_provenance(["not a mapping"], []) is None
+
+    def test_provenance_bails_when_expected_messages_disagree(self):
+        from litellm.llms.openai.responses.guardrail_translation.handler import _input_item_provenance
+
+        assert _input_item_provenance([{"role": "user", "content": "hi"}], [{"role": "user", "content": "bye"}]) is None
+
+    def test_provenance_bails_on_unpredicted_merge(self):
+        from litellm.llms.openai.responses.guardrail_translation.handler import _input_item_provenance
+        from litellm.responses.litellm_completion_transformation.transformation import (
+            LiteLLMCompletionResponsesConfig,
+        )
+
+        raw_input = [
+            {"id": "fc_1", "type": "function_call", "call_id": "call_1", "name": "read_a", "arguments": "{}"},
+            {"role": "assistant", "content": "Reading the memo now."},
+        ]
+        expected = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
+            input=raw_input, responses_api_request={}
+        )
+        assert len(expected) == 1
+        assert _input_item_provenance(raw_input, expected) is None
+
+    def test_provenance_maps_and_taints_parallel_tool_calls(self):
+        from litellm.llms.openai.responses.guardrail_translation.handler import _input_item_provenance
+        from litellm.responses.litellm_completion_transformation.transformation import (
+            LiteLLMCompletionResponsesConfig,
+        )
+
+        raw_input = _parallel_tool_call_input()
+        expected = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
+            input=raw_input, responses_api_request={}
+        )
+        provenance = _input_item_provenance(raw_input, expected)
+        assert provenance is not None
+        item_for_message, tainted = provenance
+        assert tainted == {0}
+        assert dict(item_for_message) == {1: 2, 2: 3, 3: 4}
+
+
+class TestDictStructuredMessagesGuard:
+    """A guardrail handing back a non-list structured_messages payload must not
+    blow up the request; the write-back is skipped instead."""
+
+    @pytest.mark.asyncio
+    async def test_list_input_survives_dict_structured_messages(self):
+        handler = OpenAIResponsesHandler()
+        original_input = [{"role": "user", "content": "Hello"}]
+        data = {"model": "gpt-5.6", "input": original_input}
+
+        result = await handler.process_input_messages(data, DictStructuredMessagesGuardrail())
+
+        assert result["input"] is original_input
+        assert result["input"] == [{"role": "user", "content": "Hello"}]
+
+    @pytest.mark.asyncio
+    async def test_string_input_survives_dict_structured_messages(self):
+        handler = OpenAIResponsesHandler()
+        data = {"model": "gpt-5.6", "input": "Hello there"}
+
+        result = await handler.process_input_messages(data, DictStructuredMessagesGuardrail())
+
+        assert result["input"] == "Hello there"
+
+
+class SystemRewriteGuardrail(CustomGuardrail):
+    """Guardrail that rewrites the system row, the way prompt-hardening guardrails do."""
+
+    def __init__(self, rewritten_content: Any = COMPRESSED_MARKER):
+        super().__init__()
+        self.rewritten_content = rewritten_content
+
+    async def apply_guardrail(
+        self,
+        inputs: GenericGuardrailAPIInputs,
+        request_data: dict,
+        input_type: Literal["request", "response"],
+        logging_obj: Optional[Any] = None,
+    ) -> GenericGuardrailAPIInputs:
+        messages = list(inputs.get("structured_messages") or [])
+        first = next(i for i, m in enumerate(messages) if isinstance(m, dict) and m.get("role") == "system")
+        rewritten = [
+            {**m, "content": self.rewritten_content} if i == first else m for i, m in enumerate(messages)
+        ]
+        return {**inputs, "structured_messages": rewritten}
+
+
+class TestPatchEdgeBranches:
+    @pytest.mark.asyncio
+    async def test_multimodal_user_item_rewritten_through_conversion(self):
+        handler = OpenAIResponsesHandler()
+        data = {
+            "model": "gpt-5.6",
+            "input": [
+                {"role": "user", "content": [{"type": "input_text", "text": "memo " * 400}]},
+                {"role": "user", "content": "What is the codename?"},
+            ],
+        }
+
+        result = await handler.process_input_messages(data, StructuredRewriteGuardrail())
+
+        assert _texts(result["input"][0]) == [COMPRESSED_MARKER]
+        assert result["input"][1] == {"role": "user", "content": "What is the codename?"}
+
+    @pytest.mark.asyncio
+    async def test_instructions_rewrite_lands_in_instructions_field(self):
+        handler = OpenAIResponsesHandler()
+        user_item = {"role": "user", "content": "What is the codename?"}
+        data = {
+            "model": "gpt-5.6",
+            "instructions": "Answer from the memo only.",
+            "input": [user_item],
+        }
+
+        result = await handler.process_input_messages(data, SystemRewriteGuardrail())
+
+        assert result["instructions"] == COMPRESSED_MARKER
+        assert result["input"][0] is user_item
+
+    @pytest.mark.asyncio
+    async def test_non_string_instructions_rewrite_falls_back(self):
+        handler = OpenAIResponsesHandler()
+        user_item = {"role": "user", "content": "What is the codename?"}
+        data = {
+            "model": "gpt-5.6",
+            "instructions": "Answer from the memo only.",
+            "input": [user_item],
+        }
+
+        result = await handler.process_input_messages(
+            data, SystemRewriteGuardrail(rewritten_content=[{"type": "text", "text": COMPRESSED_MARKER}])
+        )
+
+        assert result["input"][0] is not user_item
+
+    @pytest.mark.asyncio
+    async def test_unpredicted_merge_falls_back_through_patch(self):
+        handler = OpenAIResponsesHandler()
+        raw_input = [
+            {"id": "fc_1", "type": "function_call", "call_id": "call_1", "name": "read_a", "arguments": "{}"},
+            {"role": "assistant", "content": "Reading the memo now."},
+            {"type": "function_call_output", "call_id": "call_1", "output": "memo memo"},
+            {"role": "user", "content": "memo " * 400},
+        ]
+        data = {"model": "gpt-5.6", "input": raw_input}
+
+        result = await handler.process_input_messages(data, StructuredRewriteGuardrail())
+
+        assert not any(item is original for item in result["input"] for original in raw_input)
+        user_items = [item for item in result["input"] if item.get("role") == "user"]
+        assert _texts(user_items[0]) == [COMPRESSED_MARKER]
+
+    def test_item_rewrite_field_ignores_non_string_type(self):
+        from litellm.llms.openai.responses.guardrail_translation.handler import _item_rewrite_field
+
+        assert _item_rewrite_field({"type": 123, "content": "hello"}) is None

From 3004b12e900dc521e68f3f4e02e2e6bb53d50dd8 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sun, 30 Aug 2026 12:40:40 -0700
Subject: [PATCH 08/42] refactor(responses): unify guardrail input processing
 and drop mutable request params

The staging merge tightened the ruff-strict and type-discipline budgets, so the
two `data: dict` parameters the write-back helpers introduced (LIT001) and the
17-branch `process_input_messages` (C901) no longer fit.

Fold the duplicated string/list guardrail flow into one path: a pure
`_extract_guardrail_inputs` builds the guardrail payload, the write-back
helpers become pure functions returning `_RequestFields` (patched input items
plus the resulting instructions value), and the request dict is only mutated
in `process_input_messages` itself. `_apply_guardrail_responses_to_input`
takes Sequence views since it only reads. A non-list `structured_messages`
payload now falls through to the plain texts write-back, matching the
pre-write-back behavior for guardrails that never touch structured messages.

Handler file deltas vs the merge base: LIT001 57 -> 53, LIT002 42 -> 40,
LIT010 28 -> 18, C901 3 -> 3.
---
 .../guardrail_translation/handler.py          | 258 ++++++++----------
 1 file changed, 121 insertions(+), 137 deletions(-)

diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py
index bd58fb4cdbc..76a755799d0 100644
--- a/litellm/llms/openai/responses/guardrail_translation/handler.py
+++ b/litellm/llms/openai/responses/guardrail_translation/handler.py
@@ -31,7 +31,7 @@ Output: response.output is List[GenericResponseOutputItem] where each has:
 from collections.abc import Mapping, Sequence
 from itertools import accumulate
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, Union, cast
+from typing import TYPE_CHECKING, Any, Final, NamedTuple, Union, cast
 
 from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall
 from openai.types.responses.tool_param import FunctionToolParam
@@ -203,18 +203,28 @@ def _input_item_provenance(
     return item_for_message, tainted
 
 
-def _patch_rewritten_rows_into_input(
-    data: dict,
+class _RequestFields(NamedTuple):
+    input: tuple[object, ...]
+    instructions: str | None
+
+
+class _ExtractedInputs(NamedTuple):
+    inputs: GenericGuardrailAPIInputs
+    task_mappings: tuple[tuple[int, int | None], ...]
+
+
+def _patched_request_fields(
+    raw_input: object,
+    instructions: object,
     original_messages: Sequence[object],
     structured_messages: Sequence[object],
-) -> bool:
-    raw_input: Final = data.get("input")
+) -> _RequestFields | None:
     if not isinstance(raw_input, list) or len(original_messages) != len(structured_messages):
-        return False
-    offset: Final = 1 if data.get("instructions") else 0
+        return None
+    offset: Final = 1 if instructions else 0
     provenance: Final = _input_item_provenance(raw_input, tuple(original_messages)[offset:])
     if provenance is None:
-        return False
+        return None
     item_for_message, tainted = provenance
     changed: Final = tuple(
         (index, rewritten)
@@ -225,13 +235,14 @@ def _patch_rewritten_rows_into_input(
     rewritten_instructions: Final = (
         instruction_rewrites[0].get("content")
         if instruction_rewrites and isinstance(instruction_rewrites[0], Mapping)
-        else None
+        else instructions
     )
-    if instruction_rewrites and not isinstance(rewritten_instructions, str):
-        return False
+    instructions_value: Final = rewritten_instructions if isinstance(rewritten_instructions, str) else None
+    if rewritten_instructions is not None and instructions_value is None:
+        return None
     body_changes: Final = tuple((index - offset, rewritten) for index, rewritten in changed if index >= offset)
     if any(message_index in tainted or message_index not in item_for_message for message_index, _ in body_changes):
-        return False
+        return None
     replacements: Final = MappingProxyType(
         {
             item_for_message[message_index]: _rewritten_input_item(
@@ -242,11 +253,28 @@ def _patch_rewritten_rows_into_input(
         }
     )
     if len(replacements) != len(body_changes) or any(item is None for item in replacements.values()):
-        return False
-    data["input"] = [replacements.get(index, item) for index, item in enumerate(raw_input)]  # mutable-ok: JSON body
-    if isinstance(rewritten_instructions, str):
-        data["instructions"] = rewritten_instructions
-    return True
+        return None
+    return _RequestFields(
+        input=tuple(replacements.get(index, item) for index, item in enumerate(raw_input)),
+        instructions=instructions_value,
+    )
+
+
+def _written_back_request_fields(
+    raw_input: object,
+    instructions: object,
+    original_messages: Sequence[object],
+    structured_messages: Sequence[AllMessageValues],
+) -> _RequestFields | None:
+    if not isinstance(structured_messages, list):
+        return None
+    patched: Final = _patched_request_fields(raw_input, instructions, original_messages, structured_messages)
+    if patched is not None:
+        return patched
+    input_items, converted_instructions = (
+        LiteLLMResponsesTransformationHandler().convert_chat_completion_messages_to_responses_api(structured_messages)
+    )
+    return _RequestFields(input=tuple(input_items), instructions=converted_instructions)
 
 
 class OpenAIResponsesHandler(BaseTranslation):
@@ -288,136 +316,92 @@ class OpenAIResponsesHandler(BaseTranslation):
         Handles both string input and list of message objects.
         """
         input_data: Final[str | ResponseInputParam | None] = data.get("input")
-        tools_to_check: Final[list[ChatCompletionToolParam]] = []
-        if input_data is None:
+        if not isinstance(input_data, (str, list)):
             return data
-
         structured_messages: Final = self.get_structured_messages(data)
-
-        # Handle simple string input
-        if isinstance(input_data, str):
-            inputs = GenericGuardrailAPIInputs(texts=[input_data])
-            original_tools: list[dict[str, object]] = []
-
-            # Extract and transform tools if present
-            if "tools" in data and data["tools"]:
-                original_tools = list(data["tools"])
-                self._extract_and_transform_tools(data["tools"], tools_to_check)
-                if tools_to_check:
-                    inputs["tools"] = tools_to_check
-            if structured_messages:
-                inputs["structured_messages"] = structured_messages
-            # Include model information if available
-            model = data.get("model")
-            if model:
-                inputs["model"] = model
-
-            guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
-                inputs=inputs,
-                request_data=data,
-                input_type="request",
-                logging_obj=litellm_logging_obj,
-            )
-            guardrailed_structured_messages = guardrailed_inputs.get("structured_messages")
-            if (
-                guardrailed_structured_messages is not None
-                and guardrailed_structured_messages is not structured_messages
-            ):
-                self._write_back_structured_messages(data, structured_messages or (), guardrailed_structured_messages)
+        extracted: Final = self._extract_guardrail_inputs(data, input_data)
+        if not extracted.inputs.get("texts"):
+            return data
+        if structured_messages:
+            extracted.inputs["structured_messages"] = structured_messages
+        original_tools: Final[list[dict[str, object]]] = list(data.get("tools") or [])
+        guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail(
+            inputs=extracted.inputs,
+            request_data=data,
+            input_type="request",
+            logging_obj=litellm_logging_obj,
+        )
+        self._apply_guardrailed_tools_to_data(data, original_tools, guardrailed_inputs.get("tools"))
+        written_back: Final = self._written_back_request_fields(data, structured_messages, guardrailed_inputs)
+        if written_back is not None:
+            data["input"] = list(written_back.input)  # mutable-ok: JSON body
+            if written_back.instructions is None:
+                data.pop("instructions", None)
             else:
-                guardrailed_texts = guardrailed_inputs.get("texts") or ()
-                data["input"] = guardrailed_texts[0] if guardrailed_texts else input_data
-            self._apply_guardrailed_tools_to_data(data, original_tools, guardrailed_inputs.get("tools"))
-            verbose_proxy_logger.debug("OpenAI Responses API: Processed string input")
-            return data
-
-        # Handle list input (ResponseInputParam)
-        if not isinstance(input_data, list):
-            return data
+                data["instructions"] = written_back.instructions
+        elif isinstance(input_data, str):
+            guardrailed_texts: Final = guardrailed_inputs.get("texts") or ()
+            data["input"] = guardrailed_texts[0] if guardrailed_texts else input_data
+        else:
+            await self._apply_guardrail_responses_to_input(
+                messages=input_data,
+                responses=guardrailed_inputs.get("texts") or (),
+                task_mappings=extracted.task_mappings,
+            )
+        verbose_proxy_logger.debug("OpenAI Responses API: Processed input messages: %s", data.get("input"))
+        return data
 
+    def _extract_guardrail_inputs(
+        self,
+        data: Mapping[str, object],
+        input_data: "str | ResponseInputParam",
+    ) -> _ExtractedInputs:
         texts_to_check: Final[list[str]] = []
         images_to_check: Final[list[str]] = []
         task_mappings: Final[list[tuple[int, int | None]]] = []
-        original_tools_list: Final[list[dict[str, object]]] = list(data.get("tools") or [])
-
-        # Step 1: Extract all text content, images, and tools
-        for msg_idx, message in enumerate(input_data):
-            self._extract_input_text_and_images(
-                message=message,
-                msg_idx=msg_idx,
-                texts_to_check=texts_to_check,
-                images_to_check=images_to_check,
-                task_mappings=task_mappings,
-            )
-
-        # Extract and transform tools if present
-        if "tools" in data and data["tools"]:
-            self._extract_and_transform_tools(data["tools"], tools_to_check)
-
-        # Step 2: Apply guardrail to all texts in batch
-        if texts_to_check:
-            inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
-            if images_to_check:
-                inputs["images"] = images_to_check
-            if tools_to_check:
-                inputs["tools"] = tools_to_check
-            if structured_messages:
-                inputs["structured_messages"] = structured_messages
-            # Include model information if available
-            model = data.get("model")
-            if model:
-                inputs["model"] = model
-            guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
-                inputs=inputs,
-                request_data=data,
-                input_type="request",
-                logging_obj=litellm_logging_obj,
-            )
-
-            self._apply_guardrailed_tools_to_data(
-                data,
-                original_tools_list,
-                guardrailed_inputs.get("tools"),
-            )
-
-            guardrailed_structured_messages = guardrailed_inputs.get("structured_messages")
-            if (
-                guardrailed_structured_messages is not None
-                and guardrailed_structured_messages is not structured_messages
-            ):
-                self._write_back_structured_messages(data, structured_messages or (), guardrailed_structured_messages)
-            else:
-                # Step 3: Map guardrail responses back to original input structure
-                await self._apply_guardrail_responses_to_input(
-                    messages=input_data,
-                    responses=guardrailed_inputs.get("texts", []),  # mutable-ok: callee signature takes a list
+        tools_to_check: Final[list[ChatCompletionToolParam]] = []
+        if isinstance(input_data, str):
+            texts_to_check.append(input_data)
+        else:
+            for msg_idx, message in enumerate(input_data):
+                self._extract_input_text_and_images(
+                    message=message,
+                    msg_idx=msg_idx,
+                    texts_to_check=texts_to_check,
+                    images_to_check=images_to_check,
                     task_mappings=task_mappings,
                 )
-
-        verbose_proxy_logger.debug("OpenAI Responses API: Processed input messages: %s", data.get("input"))
-
-        return data
+        tools: Final = data.get("tools")
+        if tools:
+            self._extract_and_transform_tools(
+                cast("list[FunctionToolParam | OpenAIMcpServerTool]", tools),  # cast-ok: request body tools
+                tools_to_check,
+            )
+        inputs: Final = GenericGuardrailAPIInputs(texts=texts_to_check)
+        if images_to_check:
+            inputs["images"] = images_to_check
+        if tools_to_check:
+            inputs["tools"] = tools_to_check
+        model: Final = data.get("model")
+        if isinstance(model, str):
+            inputs["model"] = model
+        return _ExtractedInputs(inputs=inputs, task_mappings=tuple(task_mappings))
 
     @staticmethod
-    def _write_back_structured_messages(
-        data: dict,
-        original_messages: Sequence[object],
-        structured_messages: Sequence[AllMessageValues],
-    ) -> None:
-        if not isinstance(structured_messages, list):
-            return
-        if _patch_rewritten_rows_into_input(data, original_messages, structured_messages):
-            return
-        input_items, instructions = (
-            LiteLLMResponsesTransformationHandler().convert_chat_completion_messages_to_responses_api(
-                list(structured_messages)  # mutable-ok: converter signature takes a list
-            )
+    def _written_back_request_fields(
+        data: Mapping[str, object],
+        structured_messages: Sequence[AllMessageValues] | None,
+        guardrailed_inputs: GenericGuardrailAPIInputs,
+    ) -> _RequestFields | None:
+        guardrailed: Final = guardrailed_inputs.get("structured_messages")
+        if guardrailed is None or guardrailed is structured_messages:
+            return None
+        return _written_back_request_fields(
+            data.get("input"),
+            data.get("instructions"),
+            structured_messages or (),
+            guardrailed,
         )
-        data["input"] = input_items
-        if instructions is None:
-            data.pop("instructions", None)
-            return
-        data["instructions"] = instructions
 
     def extract_request_tool_names(self, data: dict) -> list[str]:
         """Extract tool names from Responses API request (tools[].name for function
@@ -543,8 +527,8 @@ class OpenAIResponsesHandler(BaseTranslation):
     async def _apply_guardrail_responses_to_input(
         self,
         messages: Any,  # Can be List[Dict[str, Any]] or ResponseInputParam
-        responses: list[str],
-        task_mappings: list[tuple[int, int | None]],
+        responses: Sequence[str],
+        task_mappings: Sequence[tuple[int, int | None]],
     ) -> None:
         """
         Apply guardrail responses back to input messages.

From 58c5223ab4b34d6bf013b7bafb673691e9d23d2a Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sun, 30 Aug 2026 12:59:58 -0700
Subject: [PATCH 09/42] refactor(responses): rename write-back helper out of
 the method's name

The recursion detector in code-quality reads the staticmethod
_written_back_request_fields calling the module-level function of the
same name as a recursive call. Renaming the module-level helper to
_patch_or_convert_request_fields removes the shadowing and describes
what it does: patch changed rows in place, else fall back to full
conversion.
---
 .../llms/openai/responses/guardrail_translation/handler.py    | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py
index e7a00a15a23..c676eaa7711 100644
--- a/litellm/llms/openai/responses/guardrail_translation/handler.py
+++ b/litellm/llms/openai/responses/guardrail_translation/handler.py
@@ -264,7 +264,7 @@ def _patched_request_fields(
     )
 
 
-def _written_back_request_fields(
+def _patch_or_convert_request_fields(
     raw_input: object,
     instructions: object,
     original_messages: Sequence[object],
@@ -408,7 +408,7 @@ class OpenAIResponsesHandler(BaseTranslation):
         guardrailed: Final = guardrailed_inputs.get("structured_messages")
         if guardrailed is None or guardrailed is structured_messages:
             return None
-        return _written_back_request_fields(
+        return _patch_or_convert_request_fields(
             data.get("input"),
             data.get("instructions"),
             structured_messages or (),

From 1072de94de8e9ce6de0e1e6197caa9ab207abbf6 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 31 Aug 2026 10:00:55 -0700
Subject: [PATCH 10/42] fix(azure_ai): only reclassify as azure when api_base
 is a classic Azure OpenAI endpoint

---
 litellm/llms/azure_ai/chat/transformation.py  | 28 +++++++------
 .../chat/test_azure_ai_transformation.py      | 40 +++++++++++++++++++
 .../test_gpt_5_5_model_metadata.py            |  3 +-
 3 files changed, 56 insertions(+), 15 deletions(-)

diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py
index 7fe9d3dec52..8797f4dc400 100644
--- a/litellm/llms/azure_ai/chat/transformation.py
+++ b/litellm/llms/azure_ai/chat/transformation.py
@@ -207,20 +207,22 @@ class AzureAIStudioConfig(OpenAIConfig):
                 message["content"] = texts
         return stripped_messages
 
-    def _is_azure_openai_model(self, model: str, api_base: str | None) -> bool:
-        try:
-            if "/" in model:
-                model = model.split("/", 1)[1]
-            if (
-                model in litellm.open_ai_chat_completion_models
-                or model in litellm.open_ai_text_completion_models
-                or model in litellm.open_ai_embedding_models
-            ):
-                return True
-
-        except Exception:
+    def _is_foundry_model_inference_base(self, api_base: str) -> bool:
+        parsed: Final = urlparse(api_base)
+        host: Final = parsed.hostname
+        if host is None or not host.endswith(".services.ai.azure.com"):
             return False
-        return False
+        return "/openai/deployments" not in parsed.path
+
+    def _is_azure_openai_model(self, model: str, api_base: str | None) -> bool:
+        if api_base is None or self._is_foundry_model_inference_base(api_base):
+            return False
+        stripped_model: Final = model.split("/", 1)[1] if "/" in model else model
+        return (
+            stripped_model in litellm.open_ai_chat_completion_models
+            or stripped_model in litellm.open_ai_text_completion_models
+            or stripped_model in litellm.open_ai_embedding_models
+        )
 
     def _get_openai_compatible_provider_info(
         self,
diff --git a/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py
index 11a727c9635..33fbb4e8fc7 100644
--- a/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py
+++ b/tests/test_litellm/llms/azure_ai/chat/test_azure_ai_transformation.py
@@ -31,6 +31,46 @@ async def test_get_openai_compatible_provider_info():
     assert custom_llm_provider == "azure"
 
 
+@pytest.mark.parametrize(
+    "model, api_base, expected_provider",
+    [
+        ("azure_ai/gpt-4o", "https://my-resource.services.ai.azure.com", "azure_ai"),
+        ("azure_ai/gpt-4o", "https://my-resource.services.ai.azure.com/models", "azure_ai"),
+        ("azure_ai/gpt-5.4-nano", "https://my-resource.services.ai.azure.com", "azure_ai"),
+        ("azure_ai/gpt-4o", "https://my-resource.openai.azure.com", "azure"),
+        (
+            "azure_ai/gpt-4o",
+            "https://my-resource.services.ai.azure.com/openai/deployments/gpt-4o/chat/completions"
+            "?api-version=2024-08-01-preview",
+            "azure",
+        ),
+        ("azure_ai/mistral-large-latest", "https://my-resource.services.ai.azure.com", "azure_ai"),
+        ("azure_ai/mistral-large-latest", "https://my-resource.openai.azure.com", "azure_ai"),
+    ],
+)
+def test_foundry_base_keeps_azure_ai_provider(model: str, api_base: str, expected_provider: str):
+    """Regression for #38276: a Foundry .services.ai.azure.com base must not be reclassified as azure."""
+    config = AzureAIStudioConfig()
+    (
+        _,
+        _,
+        custom_llm_provider,
+    ) = config._get_openai_compatible_provider_info(
+        model=model,
+        api_base=api_base,
+        api_key="my-key",
+        custom_llm_provider="azure_ai",
+    )
+    assert custom_llm_provider == expected_provider
+
+
+def test_is_azure_openai_model_without_api_base_keeps_azure_ai():
+    """Metadata lookups (get_model_info, supports_* checks) carry no api_base and must not flip the provider."""
+    config = AzureAIStudioConfig()
+    assert config._is_azure_openai_model(model="azure_ai/gpt-4o", api_base=None) is False
+    assert config._is_azure_openai_model(model="azure_ai/gpt-4o", api_base="https://my-res.openai.azure.com") is True
+
+
 def test_azure_ai_validate_environment():
     config = AzureAIStudioConfig()
     headers = config.validate_environment(
diff --git a/tests/test_litellm/test_gpt_5_5_model_metadata.py b/tests/test_litellm/test_gpt_5_5_model_metadata.py
index 1c12a48ed9d..a60fa9466e6 100644
--- a/tests/test_litellm/test_gpt_5_5_model_metadata.py
+++ b/tests/test_litellm/test_gpt_5_5_model_metadata.py
@@ -47,8 +47,7 @@ def test_azure_ai_gpt_5_5_model_info(model):
 
     routed_model, provider, _, _ = get_llm_provider(model=model)
     assert routed_model == model.split("/", 1)[1]
-    # azure_ai/* models resolve under the azure provider in get_llm_provider
-    assert provider == "azure"
+    assert provider == "azure_ai"
 
 
 def test_azure_ai_gpt_5_5_backup_matches_main():

From 604f1fde5014d3a966e1a2ee814e28f83bf97549 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 31 Aug 2026 11:46:33 -0700
Subject: [PATCH 11/42] fix(azure_ai): route Foundry embeddings to the /models
 inference route

---
 litellm/llms/azure_ai/chat/transformation.py  |  7 +-
 litellm/llms/azure_ai/common_utils.py         |  9 +++
 litellm/llms/azure_ai/embed/handler.py        | 19 ++++-
 .../embed/test_azure_ai_embed_handler.py      | 69 +++++++++++++++++++
 4 files changed, 96 insertions(+), 8 deletions(-)
 create mode 100644 tests/test_litellm/llms/azure_ai/embed/test_azure_ai_embed_handler.py

diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py
index 8797f4dc400..f2d405e9a17 100644
--- a/litellm/llms/azure_ai/chat/transformation.py
+++ b/litellm/llms/azure_ai/chat/transformation.py
@@ -15,6 +15,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     filter_value_from_dict,
 )
 from litellm.llms.azure.common_utils import BaseAzureLLM
+from litellm.llms.azure_ai.common_utils import is_foundry_model_inference_base
 from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
 from litellm.llms.openai.common_utils import drop_params_from_unprocessable_entity_error
 from litellm.llms.openai.openai import OpenAIConfig
@@ -208,11 +209,7 @@ class AzureAIStudioConfig(OpenAIConfig):
         return stripped_messages
 
     def _is_foundry_model_inference_base(self, api_base: str) -> bool:
-        parsed: Final = urlparse(api_base)
-        host: Final = parsed.hostname
-        if host is None or not host.endswith(".services.ai.azure.com"):
-            return False
-        return "/openai/deployments" not in parsed.path
+        return is_foundry_model_inference_base(api_base)
 
     def _is_azure_openai_model(self, model: str, api_base: str | None) -> bool:
         if api_base is None or self._is_foundry_model_inference_base(api_base):
diff --git a/litellm/llms/azure_ai/common_utils.py b/litellm/llms/azure_ai/common_utils.py
index 26a90157455..aa34bab5b2e 100644
--- a/litellm/llms/azure_ai/common_utils.py
+++ b/litellm/llms/azure_ai/common_utils.py
@@ -1,5 +1,6 @@
 from collections.abc import Mapping
 from typing import Final, Literal
+from urllib.parse import urlparse
 
 import litellm
 from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
@@ -10,6 +11,14 @@ from litellm.types.router import GenericLiteLLMParams
 AzureAIApiKeyHeader = Literal["Authorization", "api-key", "Api-Key", "Ocp-Apim-Subscription-Key"]
 
 
+def is_foundry_model_inference_base(api_base: str) -> bool:
+    parsed: Final = urlparse(api_base)
+    host: Final = parsed.hostname
+    if host is None or not host.endswith(".services.ai.azure.com"):
+        return False
+    return "/openai/deployments" not in parsed.path
+
+
 def get_azure_ai_entra_token(litellm_params: Mapping[str, object] | None = None) -> str | None:
     """
     Resolve an Entra ID / OAuth access token for an Azure AI Foundry deployment.
diff --git a/litellm/llms/azure_ai/embed/handler.py b/litellm/llms/azure_ai/embed/handler.py
index 65c3997c099..c65edbf56e6 100644
--- a/litellm/llms/azure_ai/embed/handler.py
+++ b/litellm/llms/azure_ai/embed/handler.py
@@ -1,8 +1,10 @@
 from typing import Final
+from urllib.parse import urlsplit, urlunsplit
 
 from openai import OpenAI
 
 import litellm
+from litellm.llms.azure_ai.common_utils import is_foundry_model_inference_base
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
     HTTPHandler,
@@ -16,6 +18,16 @@ from litellm.utils import convert_to_model_response_object
 from .cohere_transformation import AzureAICohereConfig
 
 
+def _foundry_models_route_base(api_base: str | None) -> str | None:
+    if api_base is None or not is_foundry_model_inference_base(api_base):
+        return api_base
+    parts: Final = urlsplit(api_base)
+    path: Final = parts.path.rstrip("/")
+    if path.endswith("/models"):
+        return api_base
+    return urlunsplit((parts.scheme, parts.netloc, f"{path}/models", parts.query, parts.fragment))
+
+
 class AzureAIEmbedding(OpenAIChatCompletion):
     def _process_response(
         self,
@@ -214,6 +226,7 @@ class AzureAIEmbedding(OpenAIChatCompletion):
 
         assemble result in-order, and return
         """
+        resolved_api_base: Final = _foundry_models_route_base(api_base)
         if aembedding is True:
             return self.async_embedding(
                 model,
@@ -223,7 +236,7 @@ class AzureAIEmbedding(OpenAIChatCompletion):
                 model_response,
                 optional_params,
                 api_key,
-                api_base,
+                resolved_api_base,
                 client,
             )
 
@@ -245,7 +258,7 @@ class AzureAIEmbedding(OpenAIChatCompletion):
                 model_response=model_response,
                 optional_params=optional_params,
                 api_key=api_key,
-                api_base=api_base,
+                api_base=resolved_api_base,
                 client=client,
             )
 
@@ -262,7 +275,7 @@ class AzureAIEmbedding(OpenAIChatCompletion):
                 model_response,
                 optional_params,
                 api_key,
-                api_base,
+                resolved_api_base,
                 client=(client if client is not None and isinstance(client, OpenAI) else None),
                 aembedding=aembedding,
                 shared_session=shared_session,
diff --git a/tests/test_litellm/llms/azure_ai/embed/test_azure_ai_embed_handler.py b/tests/test_litellm/llms/azure_ai/embed/test_azure_ai_embed_handler.py
new file mode 100644
index 00000000000..0401629171d
--- /dev/null
+++ b/tests/test_litellm/llms/azure_ai/embed/test_azure_ai_embed_handler.py
@@ -0,0 +1,69 @@
+import httpx
+import pytest
+import respx
+
+from litellm import embedding
+from litellm.llms.azure_ai.embed.handler import _foundry_models_route_base
+
+EMBEDDING_PAYLOAD = {
+    "object": "list",
+    "data": [{"object": "embedding", "embedding": [0.1, 0.2], "index": 0}],
+    "model": "text-embedding-3-small",
+    "usage": {"prompt_tokens": 2, "total_tokens": 2},
+}
+
+
+@pytest.mark.parametrize(
+    ("api_base", "expected"),
+    [
+        (
+            "https://my-foundry.services.ai.azure.com",
+            "https://my-foundry.services.ai.azure.com/models",
+        ),
+        (
+            "https://my-foundry.services.ai.azure.com/",
+            "https://my-foundry.services.ai.azure.com/models",
+        ),
+        (
+            "https://my-foundry.services.ai.azure.com?api-version=2024-05-01-preview",
+            "https://my-foundry.services.ai.azure.com/models?api-version=2024-05-01-preview",
+        ),
+        (
+            "https://my-foundry.services.ai.azure.com/models",
+            "https://my-foundry.services.ai.azure.com/models",
+        ),
+        (
+            "https://my-foundry.services.ai.azure.com/openai/deployments/text-embedding-3-small",
+            "https://my-foundry.services.ai.azure.com/openai/deployments/text-embedding-3-small",
+        ),
+        (
+            "https://my-resource.openai.azure.com",
+            "https://my-resource.openai.azure.com",
+        ),
+        (
+            "https://Mistral-serverless.eastus2.models.ai.azure.com",
+            "https://Mistral-serverless.eastus2.models.ai.azure.com",
+        ),
+        (None, None),
+    ],
+)
+def test_foundry_models_route_base(api_base, expected):
+    assert _foundry_models_route_base(api_base) == expected
+
+
+@respx.mock
+def test_azure_ai_embedding_calls_foundry_models_route():
+    route = respx.post("https://my-foundry.services.ai.azure.com/models/embeddings").mock(
+        return_value=httpx.Response(200, json=EMBEDDING_PAYLOAD)
+    )
+
+    response = embedding(
+        model="azure_ai/text-embedding-3-small",
+        input=["hello world"],
+        api_base="https://my-foundry.services.ai.azure.com",
+        api_key="fake-key",
+    )
+
+    assert route.called
+    assert response.data is not None
+    assert len(response.data) == 1

From c493fc855c85041e331f77fc53ad39950089e8f2 Mon Sep 17 00:00:00 2001
From: ryan-crabbe-berri <ryan@berri.ai>
Date: Tue, 1 Sep 2026 15:46:14 -0700
Subject: [PATCH 12/42] fix(ui): show MCP servers and agents inherited from
 access groups on the team overview

The team Overview and Settings tabs fed only object_permission into the Object
Permissions card, so a team whose access group grants MCP servers or agents read
"MCP Servers 0" and "Agents 0" while the Models card next to it already listed the
inherited models. /team/info has returned access_group_mcp_server_ids and
access_group_agent_ids for a while, nothing in the dashboard read them.

ObjectPermissionsView now accepts the inherited ids and MCPServerPermissions /
AgentPermissions merge them into their lists with an Inherited tag, deduped
against direct grants, so an admin can tell a group grant from a direct one.

Claude-Session: https://claude.ai/code/session_01QvQzYztinxj8ZuD5YxbVdL
---
 .../components/object_permissions_view.tsx    | 12 ++++-
 .../permissions/AgentPermissions.test.tsx     | 50 +++++++++++++++++++
 .../permissions/AgentPermissions.tsx          | 34 ++++++++++---
 .../permissions/MCPServerPermissions.test.tsx | 43 ++++++++++++++++
 .../permissions/MCPServerPermissions.tsx      | 34 ++++++++++---
 .../src/components/team/TeamInfo.test.tsx     | 28 +++++++++++
 .../src/components/team/TeamInfo.tsx          | 10 +++-
 7 files changed, 194 insertions(+), 17 deletions(-)
 create mode 100644 ui/litellm-dashboard/src/components/permissions/AgentPermissions.test.tsx

diff --git a/ui/litellm-dashboard/src/components/object_permissions_view.tsx b/ui/litellm-dashboard/src/components/object_permissions_view.tsx
index 011791fa18d..7d1c4ce4ac9 100644
--- a/ui/litellm-dashboard/src/components/object_permissions_view.tsx
+++ b/ui/litellm-dashboard/src/components/object_permissions_view.tsx
@@ -6,6 +6,8 @@ import type { ObjectPermission } from "./object_permission_types";
 
 interface ObjectPermissionsViewProps {
   objectPermission?: ObjectPermission | null;
+  inheritedMcpServerIds?: string[];
+  inheritedAgentIds?: string[];
   variant?: "card" | "inline";
   className?: string;
   accessToken?: string | null;
@@ -13,6 +15,8 @@ interface ObjectPermissionsViewProps {
 
 export function ObjectPermissionsView({
   objectPermission,
+  inheritedMcpServerIds = [],
+  inheritedAgentIds = [],
   variant = "card",
   className = "",
   accessToken,
@@ -34,9 +38,15 @@ export function ObjectPermissionsView({
         mcpAccessGroups={mcpAccessGroups}
         mcpToolPermissions={mcpToolPermissions}
         mcpToolsets={mcpToolsets}
+        inheritedMcpServers={inheritedMcpServerIds}
+        accessToken={accessToken}
+      />
+      <AgentPermissions
+        agents={agents}
+        agentAccessGroups={agentAccessGroups}
+        inheritedAgents={inheritedAgentIds}
         accessToken={accessToken}
       />
-      <AgentPermissions agents={agents} agentAccessGroups={agentAccessGroups} accessToken={accessToken} />
       <div className="min-w-0 rounded-md border border-border p-4">
         <p className="text-sm font-medium text-foreground">Search tools</p>
         {searchTools.length === 0 ? (
diff --git a/ui/litellm-dashboard/src/components/permissions/AgentPermissions.test.tsx b/ui/litellm-dashboard/src/components/permissions/AgentPermissions.test.tsx
new file mode 100644
index 00000000000..14fe6177ac5
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/permissions/AgentPermissions.test.tsx
@@ -0,0 +1,50 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+import { render, screen } from "@testing-library/react";
+import AgentPermissions from "./AgentPermissions";
+import * as networking from "../networking";
+
+vi.mock("../networking");
+
+describe("AgentPermissions", () => {
+  const accessToken = "test-token";
+  const agentId = "90337622-756e-4f25-98f0-01fc8174aa24";
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("lists agents inherited from access groups with an Inherited tag and counts them", async () => {
+    vi.mocked(networking.getAgentsList).mockResolvedValue({
+      agents: [{ agent_id: agentId, agent_name: "support_agent" }],
+    });
+
+    render(<AgentPermissions agents={[]} inheritedAgents={[agentId]} accessToken={accessToken} />);
+
+    expect(await screen.findByText(/support_agent/)).toBeInTheDocument();
+    expect(screen.getByText("Inherited")).toBeInTheDocument();
+    expect(screen.getByText("1")).toBeInTheDocument();
+    expect(screen.queryByText("No agents or access groups configured")).not.toBeInTheDocument();
+    expect(networking.getAgentsList).toHaveBeenCalledWith(accessToken);
+  });
+
+  it("does not double-list an agent that is both granted directly and inherited", async () => {
+    vi.mocked(networking.getAgentsList).mockResolvedValue({
+      agents: [{ agent_id: agentId, agent_name: "support_agent" }],
+    });
+
+    render(<AgentPermissions agents={[agentId]} inheritedAgents={[agentId]} accessToken={accessToken} />);
+
+    expect(await screen.findByText(/support_agent/)).toBeInTheDocument();
+    expect(screen.getAllByText(/support_agent/)).toHaveLength(1);
+    expect(screen.queryByText("Inherited")).not.toBeInTheDocument();
+    expect(screen.getByText("1")).toBeInTheDocument();
+  });
+
+  it("shows the empty state when nothing is granted directly or inherited", () => {
+    render(<AgentPermissions agents={[]} inheritedAgents={[]} accessToken={accessToken} />);
+
+    expect(screen.getByText("No agents or access groups configured")).toBeInTheDocument();
+    expect(screen.getByText("0")).toBeInTheDocument();
+    expect(networking.getAgentsList).not.toHaveBeenCalled();
+  });
+});
diff --git a/ui/litellm-dashboard/src/components/permissions/AgentPermissions.tsx b/ui/litellm-dashboard/src/components/permissions/AgentPermissions.tsx
index d84bd1834f8..21bacc23ce1 100644
--- a/ui/litellm-dashboard/src/components/permissions/AgentPermissions.tsx
+++ b/ui/litellm-dashboard/src/components/permissions/AgentPermissions.tsx
@@ -14,16 +14,26 @@ interface Agent {
 interface AgentPermissionsProps {
   agents: string[];
   agentAccessGroups?: string[];
+  inheritedAgents?: string[];
   accessToken?: string | null;
 }
 
-export function AgentPermissions({ agents, agentAccessGroups = [], accessToken }: AgentPermissionsProps) {
+const INHERITED_AGENT_TOOLTIP = "Granted through one of the team's access groups";
+
+export function AgentPermissions({
+  agents,
+  agentAccessGroups = [],
+  inheritedAgents = [],
+  accessToken,
+}: AgentPermissionsProps) {
   const [agentDetails, setAgentDetails] = useState<Agent[]>([]);
+  const inheritedOnlyAgents = inheritedAgents.filter((agent) => !agents.includes(agent));
+  const agentIdCount = agents.length + inheritedOnlyAgents.length;
 
   // Fetch agent details when component mounts
   useEffect(() => {
     const fetchAgentDetails = async () => {
-      if (accessToken && agents.length > 0) {
+      if (accessToken && agentIdCount > 0) {
         try {
           const response = await getAgentsList(accessToken);
           if (response && response.agents && Array.isArray(response.agents)) {
@@ -35,7 +45,7 @@ export function AgentPermissions({ agents, agentAccessGroups = [], accessToken }
       }
     };
     fetchAgentDetails();
-  }, [accessToken, agents.length]);
+  }, [accessToken, agentIdCount]);
 
   // Function to get display name for agent
   const getAgentDisplayName = (agentId: string) => {
@@ -47,10 +57,11 @@ export function AgentPermissions({ agents, agentAccessGroups = [], accessToken }
     return agentId;
   };
 
-  // Merge agents and access groups into one list
+  // Merge agents, inherited agents and access groups into one list
   const mergedItems = [
-    ...agents.map((agent) => ({ type: "agent", value: agent })),
-    ...agentAccessGroups.map((group) => ({ type: "accessGroup", value: group })),
+    ...agents.map((agent) => ({ type: "agent", value: agent, inherited: false })),
+    ...inheritedOnlyAgents.map((agent) => ({ type: "agent", value: agent, inherited: true })),
+    ...agentAccessGroups.map((group) => ({ type: "accessGroup", value: group, inherited: false })),
   ];
   const totalCount = mergedItems.length;
 
@@ -76,8 +87,17 @@ export function AgentPermissions({ agents, agentAccessGroups = [], accessToken }
                           <span className="text-sm font-medium text-foreground truncate">
                             {getAgentDisplayName(item.value)}
                           </span>
+                          {item.inherited && (
+                            <span className="ml-1 px-1.5 py-0.5 text-[9px] font-semibold text-purple-600 bg-purple-50 border border-purple-200 rounded-sm uppercase tracking-wide shrink-0 dark:text-purple-300 dark:bg-purple-950 dark:border-purple-800">
+                              Inherited
+                            </span>
+                          )}
                         </TooltipTrigger>
-                        <TooltipContent>{`Full ID: ${item.value}`}</TooltipContent>
+                        <TooltipContent>
+                          {item.inherited
+                            ? `${INHERITED_AGENT_TOOLTIP}. Full ID: ${item.value}`
+                            : `Full ID: ${item.value}`}
+                        </TooltipContent>
                       </Tooltip>
                     </TooltipProvider>
                   ) : (
diff --git a/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.test.tsx b/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.test.tsx
index c2df945f367..fe7e3a7b971 100644
--- a/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.test.tsx
+++ b/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.test.tsx
@@ -406,4 +406,47 @@ describe("MCPServerPermissions", () => {
     );
     await waitFor(() => expect(screen.getByText("Blocked")).toHaveAttribute("data-variant", "destructive"));
   });
+
+  it("lists servers inherited from access groups with an Inherited tag and counts them", async () => {
+    vi.mocked(networking.fetchMCPServers).mockResolvedValue([
+      { server_id: mockServerId1, server_name: mockServerName1, alias: mockServerName1 },
+    ]);
+
+    render(
+      <MCPServerPermissions
+        mcpServers={[]}
+        mcpAccessGroups={[]}
+        mcpToolPermissions={{}}
+        inheritedMcpServers={[mockServerId1]}
+        accessToken={mockAccessToken}
+      />,
+    );
+
+    expect(await screen.findByText(/DW_MCP/)).toBeInTheDocument();
+    expect(screen.getByText("Inherited")).toBeInTheDocument();
+    expect(screen.getByText("1")).toBeInTheDocument();
+    expect(screen.queryByText("No MCP servers, access groups, or toolsets configured")).not.toBeInTheDocument();
+    expect(networking.fetchMCPServers).toHaveBeenCalledWith(mockAccessToken);
+  });
+
+  it("does not double-list a server that is both granted directly and inherited", async () => {
+    vi.mocked(networking.fetchMCPServers).mockResolvedValue([
+      { server_id: mockServerId2, server_name: mockServerName2, alias: mockServerName2 },
+    ]);
+
+    render(
+      <MCPServerPermissions
+        mcpServers={[mockServerId2]}
+        mcpAccessGroups={[]}
+        mcpToolPermissions={{}}
+        inheritedMcpServers={[mockServerId2]}
+        accessToken={mockAccessToken}
+      />,
+    );
+
+    expect(await screen.findByText(/Test Server/)).toBeInTheDocument();
+    expect(screen.getAllByText(/Test Server/)).toHaveLength(1);
+    expect(screen.queryByText("Inherited")).not.toBeInTheDocument();
+    expect(screen.getByText("1")).toBeInTheDocument();
+  });
 });
diff --git a/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.tsx b/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.tsx
index f11bc6b9627..742409f3f8d 100644
--- a/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.tsx
+++ b/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.tsx
@@ -11,14 +11,18 @@ interface MCPServerPermissionsProps {
   mcpAccessGroups?: string[];
   mcpToolPermissions?: Record<string, string[]>;
   mcpToolsets?: string[];
+  inheritedMcpServers?: string[];
   accessToken?: string | null;
 }
 
+const INHERITED_MCP_SERVER_TOOLTIP = "Granted through one of the team's access groups";
+
 export function MCPServerPermissions({
   mcpServers,
   mcpAccessGroups = [],
   mcpToolPermissions = {},
   mcpToolsets = [],
+  inheritedMcpServers = [],
   accessToken,
 }: MCPServerPermissionsProps) {
   const [mcpServerDetails, setMCPServerDetails] = useState<MCPServer[]>([]);
@@ -50,10 +54,16 @@ export function MCPServerPermissions({
     });
   };
 
+  const directServerIds = mcpServers.filter(
+    (server) => server !== NO_MCP_SERVERS_SENTINEL && server !== ALL_PROXY_MCP_SERVERS_SENTINEL,
+  );
+  const inheritedOnlyServerIds = inheritedMcpServers.filter((server) => !mcpServers.includes(server));
+  const serverIdCount = directServerIds.length + inheritedOnlyServerIds.length;
+
   // Fetch MCP server details when component mounts
   useEffect(() => {
     const fetchMCPServerDetails = async () => {
-      if (accessToken && mcpServers.length > 0) {
+      if (accessToken && serverIdCount > 0) {
         try {
           const response = await fetchMCPServers(accessToken);
           if (response && Array.isArray(response)) {
@@ -67,7 +77,7 @@ export function MCPServerPermissions({
       }
     };
     fetchMCPServerDetails();
-  }, [accessToken, mcpServers.length]);
+  }, [accessToken, serverIdCount]);
 
   // Fetch toolset details
   useEffect(() => {
@@ -98,12 +108,11 @@ export function MCPServerPermissions({
   const blocksAllMcpServers = mcpServers.includes(NO_MCP_SERVERS_SENTINEL);
   const grantsAllProxyMcpServers = mcpServers.includes(ALL_PROXY_MCP_SERVERS_SENTINEL);
 
-  // Merge servers and access groups into one list
+  // Merge servers, inherited servers and access groups into one list
   const mergedItems = [
-    ...mcpServers
-      .filter((server) => server !== NO_MCP_SERVERS_SENTINEL && server !== ALL_PROXY_MCP_SERVERS_SENTINEL)
-      .map((server) => ({ type: "server", value: server })),
-    ...mcpAccessGroups.map((group) => ({ type: "accessGroup", value: group })),
+    ...directServerIds.map((server) => ({ type: "server", value: server, inherited: false })),
+    ...inheritedOnlyServerIds.map((server) => ({ type: "server", value: server, inherited: true })),
+    ...mcpAccessGroups.map((group) => ({ type: "accessGroup", value: group, inherited: false })),
   ];
   const totalCount = mergedItems.length + mcpToolsets.length;
 
@@ -152,8 +161,17 @@ export function MCPServerPermissions({
                           <span className="text-sm font-medium text-foreground truncate">
                             {getMCPServerDisplayName(item.value)}
                           </span>
+                          {item.inherited && (
+                            <span className="ml-1 px-1.5 py-0.5 text-[9px] font-semibold text-info bg-info/10 border border-info/20 rounded-sm uppercase tracking-wide shrink-0">
+                              Inherited
+                            </span>
+                          )}
                         </TooltipTrigger>
-                        <TooltipContent>{`Full ID: ${item.value}`}</TooltipContent>
+                        <TooltipContent>
+                          {item.inherited
+                            ? `${INHERITED_MCP_SERVER_TOOLTIP}. Full ID: ${item.value}`
+                            : `Full ID: ${item.value}`}
+                        </TooltipContent>
                       </Tooltip>
                     ) : (
                       <div className="inline-flex items-center gap-2 min-w-0">
diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx
index ca1e0413dcb..40a3d2ada1a 100644
--- a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx
+++ b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx
@@ -38,6 +38,9 @@ vi.mock("@/components/networking", () => ({
   organizationInfoCall: vi.fn(),
   getRouterSettingsCall: vi.fn().mockResolvedValue({ fields: [] }),
   getPassThroughEndpointsCall: vi.fn(),
+  fetchMCPServers: vi.fn().mockResolvedValue([]),
+  fetchMCPToolsets: vi.fn().mockResolvedValue([]),
+  getAgentsList: vi.fn().mockResolvedValue({ agents: [] }),
 }));
 
 const can = vi.fn();
@@ -302,6 +305,31 @@ describe("TeamInfoView", () => {
       );
     });
 
+    it("shows MCP servers and agents inherited from access groups in the Object Permissions card", async () => {
+      vi.mocked(networking.fetchMCPServers).mockResolvedValue([
+        { server_id: "mcp-github-1234", server_name: "github", alias: "github" },
+      ]);
+      vi.mocked(networking.getAgentsList).mockResolvedValue({
+        agents: [{ agent_id: "agent-support-5678", agent_name: "support_agent" }],
+      });
+      vi.mocked(networking.teamInfoCall).mockResolvedValue(
+        createMockTeamData({
+          object_permission: null,
+          access_group_ids: ["ag-1"],
+          access_group_mcp_server_ids: ["mcp-github-1234"],
+          access_group_agent_ids: ["agent-support-5678"],
+        }),
+      );
+
+      renderWithProviders(<TeamInfoView {...defaultProps} />);
+
+      expect(await screen.findByText(/github \(mcp\.\.\.1234\)/)).toBeInTheDocument();
+      expect(await screen.findByText(/support_agent \(age\.\.\.5678\)/)).toBeInTheDocument();
+      expect(screen.getAllByText("Inherited")).toHaveLength(2);
+      expect(screen.queryByText("No MCP servers, access groups, or toolsets configured")).not.toBeInTheDocument();
+      expect(screen.queryByText("No agents or access groups configured")).not.toBeInTheDocument();
+    });
+
     it("keeps the all-proxy-models badge non-clickable", async () => {
       vi.mocked(networking.teamInfoCall).mockResolvedValue(createMockTeamData({ models: ["all-proxy-models"] }));
 
diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx
index 44f0d420fd6..7ebef0691e6 100644
--- a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx
+++ b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx
@@ -1033,7 +1033,13 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({
             </div>
           </Card>
 
-          <ObjectPermissionsView objectPermission={info.object_permission} variant="card" accessToken={accessToken} />
+          <ObjectPermissionsView
+            objectPermission={info.object_permission}
+            inheritedMcpServerIds={info.access_group_mcp_server_ids}
+            inheritedAgentIds={info.access_group_agent_ids}
+            variant="card"
+            accessToken={accessToken}
+          />
 
           <Card className="block p-6">
             <GuardrailSettingsView
@@ -1883,6 +1889,8 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({
 
               <ObjectPermissionsView
                 objectPermission={info.object_permission}
+                inheritedMcpServerIds={info.access_group_mcp_server_ids}
+                inheritedAgentIds={info.access_group_agent_ids}
                 variant="inline"
                 className="pt-4 border-t border-border"
                 accessToken={accessToken}

From 944d018c72cae0930fd8c13c61ebe702ffca85d3 Mon Sep 17 00:00:00 2001
From: ryan-crabbe-berri <ryan@berri.ai>
Date: Tue, 1 Sep 2026 16:25:15 -0700
Subject: [PATCH 13/42] test(ui): query the screen in models page tests and
 drop restating comments

The merge of #38872 into staging kept the destructured render queries
in the models-and-endpoints page test, which pushes
testing-library/prefer-screen-queries to 21 against a budget of 18 and
fails frontend-lint for every PR on top of it. Two comments that only
restated the list merge below them are gone as well.

Claude-Session: https://claude.ai/code/session_01QvQzYztinxj8ZuD5YxbVdL
---
 .../app/(dashboard)/models-and-endpoints/page.test.tsx | 10 +++++-----
 .../src/components/permissions/AgentPermissions.tsx    |  1 -
 .../components/permissions/MCPServerPermissions.tsx    |  1 -
 3 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.test.tsx
index 1199b66621f..84a05113177 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/page.test.tsx
@@ -111,17 +111,17 @@ describe("ModelsAndEndpointsPage", () => {
   // POST /model/new 403s a proxy_admin_viewer, so the form's tab must not render for one.
   it("hides the Add Model tab for a view-only admin session", () => {
     mockUseAuthorized.mockReturnValue(VIEW_ONLY_ADMIN);
-    const { getByRole, queryByRole } = renderPage();
-    expect(queryByRole("tab", { name: "Add Model" })).not.toBeInTheDocument();
-    expect(getByRole("tab", { name: "All Models" })).toBeInTheDocument();
+    renderPage();
+    expect(screen.queryByRole("tab", { name: "Add Model" })).not.toBeInTheDocument();
+    expect(screen.getByRole("tab", { name: "All Models" })).toBeInTheDocument();
   });
 
   // Read parity: the Auto-Routers list stays reachable for a view-only admin; only the
   // create affordance inside it is withheld, which AutoRoutersTabPanel decides.
   it("keeps the Auto-Routers tab for a view-only admin session", () => {
     mockUseAuthorized.mockReturnValue(VIEW_ONLY_ADMIN);
-    const { getByRole } = renderPage();
-    expect(getByRole("tab", { name: /Auto-Routers/ })).toBeInTheDocument();
+    renderPage();
+    expect(screen.getByRole("tab", { name: /Auto-Routers/ })).toBeInTheDocument();
   });
 
   // Auto-routers are excluded from the All Models table, so this tab is their home: the only
diff --git a/ui/litellm-dashboard/src/components/permissions/AgentPermissions.tsx b/ui/litellm-dashboard/src/components/permissions/AgentPermissions.tsx
index 21bacc23ce1..11f6c414922 100644
--- a/ui/litellm-dashboard/src/components/permissions/AgentPermissions.tsx
+++ b/ui/litellm-dashboard/src/components/permissions/AgentPermissions.tsx
@@ -57,7 +57,6 @@ export function AgentPermissions({
     return agentId;
   };
 
-  // Merge agents, inherited agents and access groups into one list
   const mergedItems = [
     ...agents.map((agent) => ({ type: "agent", value: agent, inherited: false })),
     ...inheritedOnlyAgents.map((agent) => ({ type: "agent", value: agent, inherited: true })),
diff --git a/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.tsx b/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.tsx
index 742409f3f8d..52199dab3c7 100644
--- a/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.tsx
+++ b/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.tsx
@@ -108,7 +108,6 @@ export function MCPServerPermissions({
   const blocksAllMcpServers = mcpServers.includes(NO_MCP_SERVERS_SENTINEL);
   const grantsAllProxyMcpServers = mcpServers.includes(ALL_PROXY_MCP_SERVERS_SENTINEL);
 
-  // Merge servers, inherited servers and access groups into one list
   const mergedItems = [
     ...directServerIds.map((server) => ({ type: "server", value: server, inherited: false })),
     ...inheritedOnlyServerIds.map((server) => ({ type: "server", value: server, inherited: true })),

From 7b942fd983af2b033c2767f783777ae86df6cec8 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Wed, 2 Sep 2026 11:05:38 -0700
Subject: [PATCH 14/42] fix(azure_ai): route audio and realtime calls on
 Foundry hosts through the Azure OpenAI handlers

---
 litellm/constants.py                         |  1 +
 litellm/main.py                              | 10 +++--
 litellm/realtime_api/main.py                 |  3 +-
 tests/test_litellm/realtime_api/test_main.py | 37 ++++++++++++++++++
 tests/test_litellm/test_main.py              | 40 ++++++++++++++++++++
 5 files changed, 87 insertions(+), 4 deletions(-)

diff --git a/litellm/constants.py b/litellm/constants.py
index 1bd977dd9a9..041a83f53e6 100644
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -7,6 +7,7 @@ from litellm.litellm_core_utils.env_utils import get_env_int, get_env_int_in_ran
 
 DEFAULT_HEALTH_CHECK_PROMPT: Final = str(os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm"))
 AZURE_DEFAULT_RESPONSES_API_VERSION: Final = str(os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview"))
+AZURE_OPENAI_AUDIO_PROVIDERS: Final = frozenset({"azure", "azure_ai"})
 ROUTER_MAX_FALLBACKS: Final = int(os.getenv("ROUTER_MAX_FALLBACKS", 5))
 ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS: Final = 2000
 DEFAULT_BATCH_SIZE: Final = int(os.getenv("DEFAULT_BATCH_SIZE", 512))
diff --git a/litellm/main.py b/litellm/main.py
index c4c5bbefc4f..f6b6453ab45 100644
--- a/litellm/main.py
+++ b/litellm/main.py
@@ -60,6 +60,7 @@ if TYPE_CHECKING:
     from litellm.types.utils import TokenCountResponse
 
 from litellm.constants import (
+    AZURE_OPENAI_AUDIO_PROVIDERS,
     DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT,
     DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT,
 )
@@ -7770,7 +7771,7 @@ def transcription(
         provider=LlmProviders(custom_llm_provider),
     )
 
-    if custom_llm_provider == "azure" and provider_config is None:
+    if custom_llm_provider in AZURE_OPENAI_AUDIO_PROVIDERS and provider_config is None:
         # azure configs
         api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
 
@@ -8057,7 +8058,10 @@ def speech(
         custom_llm_provider=custom_llm_provider,
     )
     response: HttpxBinaryResponseContent | Coroutine[object, object, HttpxBinaryResponseContent] | None = None
-    if custom_llm_provider == "openai" or custom_llm_provider in litellm.openai_compatible_providers:
+    if custom_llm_provider == "openai" or (
+        custom_llm_provider in litellm.openai_compatible_providers
+        and custom_llm_provider not in AZURE_OPENAI_AUDIO_PROVIDERS
+    ):
         if voice is None or not (isinstance(voice, str)):
             raise litellm.BadRequestError(
                 message="'voice' is required to be passed as a string for OpenAI TTS",
@@ -8111,7 +8115,7 @@ def speech(
             aspeech=aspeech,
             shared_session=shared_session,
         )
-    elif custom_llm_provider == "azure":
+    elif custom_llm_provider in AZURE_OPENAI_AUDIO_PROVIDERS:
         # Check if this is Azure Speech Service (Cognitive Services TTS)
         if model.startswith("speech/"):
             from litellm.llms.azure.text_to_speech.transformation import (
diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py
index d4b9f4e8cce..3862aec445f 100644
--- a/litellm/realtime_api/main.py
+++ b/litellm/realtime_api/main.py
@@ -8,6 +8,7 @@ from typing import Any, Final, Literal, cast
 
 import litellm
 from litellm.constants import (
+    AZURE_OPENAI_AUDIO_PROVIDERS,
     REALTIME_CREDENTIAL_RESOLUTION_TIMEOUT_SECONDS,
     REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES,
     request_timeout,
@@ -400,7 +401,7 @@ async def _arealtime(
             litellm_metadata=_build_litellm_metadata(kwargs),
             query_params=query_params,
         )
-    elif _custom_llm_provider == "azure":
+    elif _custom_llm_provider in AZURE_OPENAI_AUDIO_PROVIDERS:
         api_base = dynamic_api_base or litellm_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
         # set API KEY
         api_key = dynamic_api_key or litellm.api_key or litellm.openai_key or get_secret_str("AZURE_API_KEY")
diff --git a/tests/test_litellm/realtime_api/test_main.py b/tests/test_litellm/realtime_api/test_main.py
index a3dd5688ad1..761e87ac764 100644
--- a/tests/test_litellm/realtime_api/test_main.py
+++ b/tests/test_litellm/realtime_api/test_main.py
@@ -1,6 +1,7 @@
 import asyncio
 import time
 from types import TracebackType
+from typing import Final
 from unittest.mock import MagicMock, patch
 
 
@@ -294,3 +295,39 @@ async def test_azure_health_check_honors_deployment_realtime_protocol():
             model_params={"realtime_protocol": "GA"},
         )
     assert connect.url == "wss://my-endpoint.openai.azure.com/openai/v1/realtime?model=gpt-4o-realtime-preview"
+
+
+class _ConnectThatStopsAfterCapturingTheUrl:
+    url: str | None = None
+
+    def __call__(self, url: str, **kwargs: object) -> "_ConnectThatStopsAfterCapturingTheUrl":
+        self.url = url
+        return self
+
+    async def __aenter__(self) -> None:
+        raise RuntimeError("backend url captured, nothing to bridge")
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        return None
+
+
+@pytest.mark.asyncio
+async def test_arealtime_azure_ai_on_a_foundry_host_connects_to_the_azure_openai_realtime_route():
+    connect: Final = _ConnectThatStopsAfterCapturingTheUrl()
+    with patch("websockets.connect", connect):
+        await realtime_main._arealtime.__wrapped__(
+            model="azure_ai/gpt-realtime-mini",
+            websocket=MagicMock(),
+            api_base="https://my-project.services.ai.azure.com",
+            api_key="fake-key",
+            litellm_logging_obj=FakeLogging(),
+        )
+    assert connect.url == (
+        "wss://my-project.services.ai.azure.com/openai/realtime"
+        "?api-version=2024-10-01-preview&deployment=gpt-realtime-mini"
+    )
diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py
index 8cf878d05d9..30975033b7d 100644
--- a/tests/test_litellm/test_main.py
+++ b/tests/test_litellm/test_main.py
@@ -3181,3 +3181,43 @@ def test_stream_chunk_builder_defers_cost_to_logging_obj_when_usage_cost_absent(
 
     assert response is not None
     assert response._hidden_params.get("response_cost") is None
+
+
+FOUNDRY_HOST: Final = "https://my-project.services.ai.azure.com"
+
+
+def test_azure_ai_transcription_on_a_foundry_host_uses_the_azure_openai_deployment_route(
+    respx_mock: respx.MockRouter,
+):
+    route: Final = respx_mock.post(
+        url__regex=r"https://my-project\.services\.ai\.azure\.com/openai/deployments/whisper-1/audio/transcriptions\?api-version=.+"
+    ).mock(return_value=httpx.Response(200, json={"text": "hello"}))
+
+    response: Final = litellm.transcription(
+        model="azure_ai/whisper-1",
+        file=("tone.wav", b"RIFF\x00\x00\x00\x00WAVE", "audio/wav"),
+        api_base=FOUNDRY_HOST,
+        api_key="fake-key",
+    )
+
+    assert route.called
+    assert response.text == "hello"
+
+
+def test_azure_ai_speech_on_a_foundry_host_uses_the_azure_openai_deployment_route(
+    respx_mock: respx.MockRouter,
+):
+    route: Final = respx_mock.post(
+        url__regex=r"https://my-project\.services\.ai\.azure\.com/openai/deployments/tts-1/audio/speech\?api-version=.+"
+    ).mock(return_value=httpx.Response(200, content=b"mp3-bytes"))
+
+    response: Final = litellm.speech(
+        model="azure_ai/tts-1",
+        input="hello",
+        voice="alloy",
+        api_base=FOUNDRY_HOST,
+        api_key="fake-key",
+    )
+
+    assert route.called
+    assert response.content == b"mp3-bytes"

From de643c028fce8d7634191550968eae7c4c991f04 Mon Sep 17 00:00:00 2001
From: yassin <yassin@berri.ai>
Date: Thu, 3 Sep 2026 00:25:50 +0000
Subject: [PATCH 15/42] style: format handler after merge

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../llms/openai/responses/guardrail_translation/handler.py    | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py
index 0c19ecf909e..59ca601d8be 100644
--- a/litellm/llms/openai/responses/guardrail_translation/handler.py
+++ b/litellm/llms/openai/responses/guardrail_translation/handler.py
@@ -374,7 +374,9 @@ class OpenAIResponsesHandler(BaseTranslation):
             input_type="request",
             logging_obj=litellm_logging_obj,
         )
-        self._apply_guardrailed_tools_to_data(data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools"))
+        self._apply_guardrailed_tools_to_data(
+            data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools")
+        )
         written_back: Final = self._written_back_request_fields(data, structured_messages, guardrailed_inputs)
         if written_back is not None:
             data["input"] = list(written_back.input)  # mutable-ok: JSON body

From 45f44fea6019e3e9afd75e7157992899f0846bd0 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Wed, 2 Sep 2026 17:53:39 -0700
Subject: [PATCH 16/42] fix(headroom): leave background responses requests
 uncompressed and unconverted

---
 .../guardrail_hooks/headroom/headroom.py      |  6 ++-
 .../guardrail_hooks/test_headroom.py          | 43 +++++++++++++++++++
 2 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py
index 5fbf98d0aae..9d993384461 100644
--- a/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py
+++ b/litellm/proxy/guardrails/guardrail_hooks/headroom/headroom.py
@@ -728,6 +728,10 @@ class HeadroomGuardrail(CustomGuardrail):
             verbose_proxy_logger.debug("Headroom: %s header set; skipping compression", BYPASS_HEADER)
             return inputs
 
+        if request_data.get("background"):
+            verbose_proxy_logger.debug("Headroom: background request; skipping compression")
+            return inputs
+
         structured_messages: Final = inputs.get("structured_messages")
         if not _is_object_list(structured_messages) or not structured_messages:
             return inputs
@@ -831,7 +835,7 @@ class HeadroomGuardrail(CustomGuardrail):
         effective: Final = base_result if base_result is not None else kwargs
         if call_type not in _STREAM_CONVERTIBLE_CALL_TYPES:
             return base_result
-        if not effective.get("stream"):
+        if not effective.get("stream") or effective.get("background"):
             return base_result
         if not has_headroom_retrieve_tool(effective.get("tools")):
             return base_result
diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py
index a940e4c3394..400eaf8ab3f 100644
--- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py
+++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_headroom.py
@@ -193,6 +193,33 @@ async def test_apply_guardrail_compresses_and_returns_structured_messages(
     assert "headroom" in _applied_guardrails(request_data)
 
 
+@pytest.mark.asyncio
+async def test_apply_guardrail_leaves_background_requests_uncompressed(
+    guardrail: HeadroomGuardrail,
+):
+    inputs = GenericGuardrailAPIInputs(
+        texts=["A" * 5000],
+        structured_messages=ORIGINAL_MESSAGES,
+    )
+    request_data = {"model": "gpt-4o", "background": True}
+
+    with patch.object(
+        guardrail.async_handler,
+        "post",
+        new_callable=AsyncMock,
+        return_value=_make_compress_response(COMPRESSED_MESSAGES),
+    ) as post:
+        result = await guardrail.apply_guardrail(
+            inputs=inputs,
+            request_data=request_data,
+            input_type="request",
+        )
+
+    assert result is inputs
+    post.assert_not_awaited()
+    assert _recorded_guardrail_entries(request_data) == []
+
+
 def _recorded_guardrail_response(request_data: dict) -> dict:
     entries = request_data["metadata"]["standard_logging_guardrail_information"]
     assert len(entries) == 1
@@ -2074,6 +2101,22 @@ async def test_pre_call_deployment_hook_converts_stream_only_for_ccr_chat_comple
     assert kwargs["stream"] is True
 
 
+@pytest.mark.asyncio
+async def test_pre_call_deployment_hook_leaves_background_streams_alone(guardrail: HeadroomGuardrail):
+    kwargs = {
+        "model": "gpt-4o",
+        "stream": True,
+        "background": True,
+        "tools": [_responses_retrieve_tool_definition()],
+    }
+
+    result = await guardrail.async_pre_call_deployment_hook(kwargs=kwargs, call_type=CallTypes.aresponses)
+
+    assert result is kwargs
+    assert HEADROOM_CONVERTED_STREAM_KEY not in kwargs
+    assert kwargs["stream"] is True
+
+
 @pytest.mark.asyncio
 async def test_pre_call_deployment_hook_still_compresses_for_deployment_level_configs(
     guardrail: HeadroomGuardrail,

From e1b2d9de3c85bdeb8b38dab33fcef28ec42ecac5 Mon Sep 17 00:00:00 2001
From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Thu, 3 Sep 2026 08:45:23 +0000
Subject: [PATCH 17/42] fix(images): forward gpt-image supported params like
 background to OpenAI and Azure

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/types/utils.py                       |  1 +
 litellm/utils.py                             | 11 ++++---
 tests/test_litellm/test_utils.py             | 31 ++++++++++++++++++++
 tests/test_litellm/types/test_types_utils.py |  9 ++++++
 4 files changed, 48 insertions(+), 4 deletions(-)

diff --git a/litellm/types/utils.py b/litellm/types/utils.py
index 569fce4f7b8..0bdbb83fbb4 100644
--- a/litellm/types/utils.py
+++ b/litellm/types/utils.py
@@ -2543,6 +2543,7 @@ class ImageResponse(OpenAIImageResponse, BaseLiteLLMOpenAIResponseObject):
         )
         super().__init__(created=created, data=_data, usage=_usage)
 
+        self.background = kwargs.get("background", None)
         self.quality = kwargs.get("quality", None)
         self.output_format = kwargs.get("output_format", None)
         self.size = kwargs.get("size", None)
diff --git a/litellm/utils.py b/litellm/utils.py
index ba456fc353b..239673a20d9 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -3338,6 +3338,9 @@ def get_optional_params_image_gen(
             continue
         passed_params[k] = v
 
+    provider_supported_params: Final[tuple[str, ...]] = (
+        tuple(provider_config.get_supported_openai_params(model=model or "")) if provider_config is not None else ()
+    )
     default_params: Final = {
         "n": None,
         "quality": None,
@@ -3348,6 +3351,7 @@ def get_optional_params_image_gen(
         "imageConfig": None,
         "tools": None,
         "web_search_options": None,
+        **{k: None for k in provider_supported_params},
     }
 
     non_default_params: Final = _get_non_default_params(
@@ -3407,10 +3411,9 @@ def get_optional_params_image_gen(
         if size is not None:
             optional_params["aspectRatio"] = _map_openai_size_to_vertex_ai_aspect_ratio(size)
 
-    openai_params: list[str] = list(default_params.keys())
-    if provider_config is not None:
-        supported_params = provider_config.get_supported_openai_params(model=model or "")
-        openai_params = list(supported_params)
+    openai_params: Final[list[str]] = (
+        list(provider_supported_params) if provider_config is not None else list(default_params.keys())
+    )
 
     optional_params = add_provider_specific_params_to_optional_params(
         optional_params=optional_params,
diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py
index 200cfd02197..99c69d6bc31 100644
--- a/tests/test_litellm/test_utils.py
+++ b/tests/test_litellm/test_utils.py
@@ -330,6 +330,37 @@ def test_get_optional_params_image_gen():
     assert optional_params["n"] == 3
 
 
+@pytest.mark.parametrize("custom_llm_provider", ["openai", "azure"])
+def test_get_optional_params_image_gen_keeps_gpt_image_supported_params(custom_llm_provider):
+    """https://github.com/BerriAI/litellm/issues/38649"""
+    from litellm.types.utils import LlmProviders
+
+    provider_config = ProviderConfigManager.get_provider_image_generation_config(
+        model="gpt-image-2", provider=LlmProviders(custom_llm_provider)
+    )
+    optional_params = get_optional_params_image_gen(
+        model="gpt-image-2",
+        n=1,
+        size="1024x1024",
+        custom_llm_provider=custom_llm_provider,
+        provider_config=provider_config,
+        background="transparent",
+        output_format="png",
+        moderation="low",
+        output_compression=50,
+        unknown_param="kept-in-extra-body",
+    )
+    assert optional_params == {
+        "n": 1,
+        "size": "1024x1024",
+        "background": "transparent",
+        "output_format": "png",
+        "moderation": "low",
+        "output_compression": 50,
+        "extra_body": {"unknown_param": "kept-in-extra-body"},
+    }
+
+
 def test_get_optional_params_image_gen_vertex_ai_size():
     """Test that Vertex AI image generation properly handles size parameter and maps it to aspectRatio"""
     # Test with various size parameters
diff --git a/tests/test_litellm/types/test_types_utils.py b/tests/test_litellm/types/test_types_utils.py
index c081b9e8e0d..554604ab200 100644
--- a/tests/test_litellm/types/test_types_utils.py
+++ b/tests/test_litellm/types/test_types_utils.py
@@ -759,3 +759,12 @@ def test_delta_function_tool_call_unchanged_by_custom_support():
     delta = Delta(tool_calls=[{"index": 0, "id": "c2", "type": "function", "function": {"name": "g", "arguments": ""}}])
     assert isinstance(delta.tool_calls[0], ChatCompletionDeltaToolCall)
     assert "custom" not in delta.model_dump()["tool_calls"][0]
+
+
+def test_image_response_keeps_background():
+    """https://github.com/BerriAI/litellm/issues/38649"""
+    from litellm.types.utils import ImageResponse
+
+    response = ImageResponse(created=1, data=[{"b64_json": "aGk="}], background="transparent", output_format="png")
+    assert response.background == "transparent"
+    assert response.model_dump()["background"] == "transparent"

From 7a5e4b9ba4f6d7d61122dd32b7e9c923cbd18bcf Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Thu, 3 Sep 2026 10:04:18 -0700
Subject: [PATCH 18/42] fix(openai): bridge gpt-5.4+ tool calls to
 /v1/responses on every api.openai.com host

The auto-bridge that moves gpt-5.4+ requests carrying function tools and no
reasoning_effort onto /v1/responses only fired when the resolved api_base was
the literal https://api.openai.com/v1, so a deployment pointed at an OpenAI
PrivateLink hostname (<region>.privatelink.api.openai.com) or a port-qualified
or trailing-slash default stayed on Chat Completions and got OpenAI's 400 back.
Gate on the resolved URL's hostname instead: api.openai.com or any subdomain of
it bridges, every other custom base still stays on chat
---
 litellm/main.py                 | 26 +++++++----
 tests/test_litellm/test_main.py | 76 +++++++++++++++++++++++++++++++++
 2 files changed, 93 insertions(+), 9 deletions(-)

diff --git a/litellm/main.py b/litellm/main.py
index 0bca4a7350e..6f8310a9d8a 100644
--- a/litellm/main.py
+++ b/litellm/main.py
@@ -26,6 +26,7 @@ from copy import deepcopy
 from functools import partial
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args
+from urllib.parse import urlsplit
 
 from litellm._logging import _redact_string
 from litellm._uuid import uuid
@@ -984,6 +985,12 @@ def mock_completion(
 
 
 _OPENAI_DEFAULT_API_BASE: Final = "https://api.openai.com/v1"
+_OPENAI_API_HOST: Final = "api.openai.com"
+
+
+def _is_openai_backed_api_base(api_base: str) -> bool:
+    hostname: Final = urlsplit(api_base).hostname
+    return hostname is not None and (hostname == _OPENAI_API_HOST or hostname.endswith(f".{_OPENAI_API_HOST}"))
 
 
 def _resolve_openai_api_base(api_base: str | None) -> str:
@@ -1053,7 +1060,7 @@ def responses_api_bridge_check(
     #   natively by Chat Completions with reasoning on, so custom-only requests stay on
     #   chat and keep their native custom tool_call response shape.
     # - The UNSET-effort arm only fires against endpoints known to enforce that
-    #   constraint (the default OpenAI endpoint, or Azure OpenAI where api_base is
+    #   constraint (any api.openai.com host, or Azure OpenAI where api_base is
     #   always set): chat-only OpenAI-compatible backends registered under the openai
     #   provider with a custom api_base and gpt-5.4+ model names serve tools without
     #   reasoning fine and have no /responses route, so they keep pre-existing
@@ -1068,14 +1075,15 @@ def responses_api_bridge_check(
         reasoning_active = reasoning_effort.get("effort") != "none" or reasoning_effort.get("summary") is not None
     else:
         reasoning_active = reasoning_effort != "none"
-    # The reasoning+tools constraint is enforced only by the real OpenAI endpoint (and Azure OpenAI).
-    # Resolve the effective base arg>global>env>default exactly as the chat handler does, so a custom
-    # base set via litellm.api_base or OPENAI_BASE_URL/OPENAI_API_BASE isn't misread as the default and
-    # bridged to a /responses route it lacks. A whitespace-only base collapses to the default too.
-    resolved_api_base: Final = _resolve_openai_api_base(api_base)
-    on_constraint_enforcing_endpoint: Final = custom_llm_provider == "azure" or resolved_api_base.strip() in (
-        "",
-        _OPENAI_DEFAULT_API_BASE,
+    # The reasoning+tools constraint is enforced by the real OpenAI backend behind any api.openai.com
+    # host (the default URL or a PrivateLink hostname such as <region>.privatelink.api.openai.com) and
+    # by Azure OpenAI. Resolve the effective base arg>global>env>default exactly as the chat handler
+    # does, so a custom base set via litellm.api_base or OPENAI_BASE_URL/OPENAI_API_BASE isn't misread
+    # as the default and bridged to a /responses route it lacks. A whitespace-only base collapses to
+    # the default too.
+    resolved_api_base: Final = _resolve_openai_api_base(api_base).strip()
+    on_constraint_enforcing_endpoint: Final = (
+        custom_llm_provider == "azure" or resolved_api_base == "" or _is_openai_backed_api_base(resolved_api_base)
     )
     if (
         custom_llm_provider in ("openai", "azure")
diff --git a/tests/test_litellm/test_main.py b/tests/test_litellm/test_main.py
index 3c8bf142835..4cba5048e5d 100644
--- a/tests/test_litellm/test_main.py
+++ b/tests/test_litellm/test_main.py
@@ -1133,6 +1133,82 @@ def test_responses_api_bridge_check_custom_api_base_via_env_with_unset_effort_st
     assert model_info.get("mode") != "responses"
 
 
+@pytest.mark.parametrize(
+    "api_base",
+    [
+        "https://southcentralus.privatelink.api.openai.com/v1",
+        "https://privatelink.corp.api.openai.com/v1",
+        "https://api.openai.com:443/v1",
+        "https://api.openai.com/v1/",
+        "HTTPS://API.OPENAI.COM/v1",
+    ],
+)
+def test_responses_api_bridge_check_openai_backed_custom_api_base_with_unset_effort_routes_to_responses(api_base):
+    """
+    A custom api_base whose host is api.openai.com or a subdomain of it (a PrivateLink hostname, a
+    port-qualified or trailing-slash default) still reaches the real OpenAI backend, which rejects
+    function tools with reasoning on Chat Completions, so the unset-effort arm must bridge exactly as
+    it does for the literal default URL. Regression guard for GH #39353.
+    """
+    from litellm.main import responses_api_bridge_check
+
+    model_info, model = responses_api_bridge_check(
+        model="gpt-5.6",
+        custom_llm_provider="openai",
+        tools=[{"type": "function", "function": {"name": "get_capital"}}],
+        reasoning_effort=None,
+        api_base=api_base,
+        )
+
+    assert model == "gpt-5.6"
+    assert model_info.get("mode") == "responses"
+
+
+@pytest.mark.parametrize(
+    "api_base",
+    [
+        "https://api.openai.com.evil.example/v1",
+        "https://notapi.openai.com/v1",
+        "https://gateway.example/v1?upstream=api.openai.com",
+        "https://openai.internal.example/api.openai.com/v1",
+    ],
+)
+def test_responses_api_bridge_check_lookalike_custom_api_base_with_unset_effort_stays_chat(api_base):
+    """Only the host decides: api.openai.com appearing elsewhere in the URL is still a foreign backend."""
+    from litellm.main import responses_api_bridge_check
+
+    model_info, model = responses_api_bridge_check(
+        model="gpt-5.6",
+        custom_llm_provider="openai",
+        tools=[{"type": "function", "function": {"name": "get_capital"}}],
+        reasoning_effort=None,
+        api_base=api_base,
+        )
+
+    assert model == "gpt-5.6"
+    assert model_info.get("mode") != "responses"
+
+
+def test_responses_api_bridge_check_privatelink_api_base_via_env_with_unset_effort_routes_to_responses(monkeypatch):
+    """A PrivateLink base set through OPENAI_BASE_URL resolves the way the chat handler's does and still bridges."""
+    import litellm
+    from litellm.main import responses_api_bridge_check
+
+    monkeypatch.setattr(litellm, "api_base", None)
+    monkeypatch.delenv("OPENAI_API_BASE", raising=False)
+    monkeypatch.setenv("OPENAI_BASE_URL", "https://southcentralus.privatelink.api.openai.com/v1")
+    model_info, model = responses_api_bridge_check(
+        model="gpt-5.6",
+        custom_llm_provider="openai",
+        tools=[{"type": "function", "function": {"name": "get_capital"}}],
+        reasoning_effort=None,
+        api_base=None,
+        )
+
+    assert model == "gpt-5.6"
+    assert model_info.get("mode") == "responses"
+
+
 def test_responses_api_bridge_check_custom_api_base_with_explicit_effort_still_routes():
     """Explicit reasoning_effort keeps its pre-existing bridging behavior on any api_base."""
     from litellm.main import responses_api_bridge_check

From 2fd6e190517b7ce050c603489556efdb9870137a Mon Sep 17 00:00:00 2001
From: ryan-crabbe-berri <ryan@berri.ai>
Date: Thu, 3 Sep 2026 10:20:10 -0700
Subject: [PATCH 19/42] fix(ui): name the granting access group on hover
 instead of an Inherited tag

`/team/info` access_group_details now carries mcp_server_ids and agent_ids per
group next to models, so the dashboard can say which group granted a server or
agent. The Object Permissions rows drop the Inherited badge and the row tooltip
reads "Granted via access group <name>. Full ID: <id>", listing every group
when more than one grants the same id and falling back to "an access group"
when the proxy did not say.

Claude-Session: https://claude.ai/code/session_01QvQzYztinxj8ZuD5YxbVdL
---
 litellm/proxy/_types.py                       |  2 +
 .../management_endpoints/team_endpoints.py    |  2 +
 .../test_team_endpoints.py                    |  6 +-
 .../components/object_permissions_view.tsx    | 13 ++--
 .../permissions/AgentPermissions.test.tsx     | 35 ++++++++---
 .../permissions/AgentPermissions.tsx          | 24 +++-----
 .../permissions/MCPServerPermissions.test.tsx | 22 ++++---
 .../permissions/MCPServerPermissions.tsx      | 30 ++++-----
 .../permissions/inheritedGrants.test.ts       | 61 +++++++++++++++++++
 .../components/permissions/inheritedGrants.ts | 26 ++++++++
 .../src/components/team/TeamInfo.test.tsx     | 26 ++++++--
 .../src/components/team/TeamInfo.tsx          | 20 ++++--
 .../src/components/team/teamModelAccess.ts    |  4 +-
 13 files changed, 204 insertions(+), 67 deletions(-)
 create mode 100644 ui/litellm-dashboard/src/components/permissions/inheritedGrants.test.ts
 create mode 100644 ui/litellm-dashboard/src/components/permissions/inheritedGrants.ts

diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index e0a2097919b..57f6f31d4d9 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -4172,6 +4172,8 @@ class TeamAccessGroupModelGrant(LiteLLMPydanticObjectBase):
     access_group_id: str
     access_group_name: str
     models: tuple[str, ...]
+    mcp_server_ids: tuple[str, ...] = ()
+    agent_ids: tuple[str, ...] = ()
 
 
 class TeamInfoResponseObjectTeamTable(LiteLLM_TeamTable):
diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py
index 714cf252e69..72cdc75c29e 100644
--- a/litellm/proxy/management_endpoints/team_endpoints.py
+++ b/litellm/proxy/management_endpoints/team_endpoints.py
@@ -4318,6 +4318,8 @@ async def _resolve_team_access_group_resources(
                     access_group_id=group.access_group_id,
                     access_group_name=group.access_group_name,
                     models=tuple(group.access_model_names or ()),
+                    mcp_server_ids=tuple(group.access_mcp_server_ids or ()),
+                    agent_ids=tuple(group.access_agent_ids or ()),
                 )
                 for group in resolved_groups
             ),
diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py
index 30b2ab86b9a..5bb7ceb8fb1 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py
@@ -9897,11 +9897,11 @@ class TestResolveTeamAccessGroupResources:
         assert resolved.access_group_mcp_server_ids == ["mcp-1"]
         assert resolved.access_group_agent_ids == ["agent-1"]
         assert [
-            (d.access_group_id, d.access_group_name, d.models)
+            (d.access_group_id, d.access_group_name, d.models, d.mcp_server_ids, d.agent_ids)
             for d in (resolved.access_group_details or [])
         ] == [
-            ("ag-1", "shared-models", ("gpt-4", "claude-3")),
-            ("ag-2", "extra-models", ("claude-3", "gemini")),
+            ("ag-1", "shared-models", ("gpt-4", "claude-3"), ("mcp-1",), ()),
+            ("ag-2", "extra-models", ("claude-3", "gemini"), (), ("agent-1",)),
         ]
 
     @pytest.mark.asyncio
diff --git a/ui/litellm-dashboard/src/components/object_permissions_view.tsx b/ui/litellm-dashboard/src/components/object_permissions_view.tsx
index 7d1c4ce4ac9..e90645a9e9f 100644
--- a/ui/litellm-dashboard/src/components/object_permissions_view.tsx
+++ b/ui/litellm-dashboard/src/components/object_permissions_view.tsx
@@ -3,11 +3,12 @@ import VectorStorePermissions from "./permissions/VectorStorePermissions";
 import MCPServerPermissions from "./permissions/MCPServerPermissions";
 import AgentPermissions from "./permissions/AgentPermissions";
 import type { ObjectPermission } from "./object_permission_types";
+import type { InheritedGrant } from "./permissions/inheritedGrants";
 
 interface ObjectPermissionsViewProps {
   objectPermission?: ObjectPermission | null;
-  inheritedMcpServerIds?: string[];
-  inheritedAgentIds?: string[];
+  inheritedMcpServers?: InheritedGrant[];
+  inheritedAgents?: InheritedGrant[];
   variant?: "card" | "inline";
   className?: string;
   accessToken?: string | null;
@@ -15,8 +16,8 @@ interface ObjectPermissionsViewProps {
 
 export function ObjectPermissionsView({
   objectPermission,
-  inheritedMcpServerIds = [],
-  inheritedAgentIds = [],
+  inheritedMcpServers = [],
+  inheritedAgents = [],
   variant = "card",
   className = "",
   accessToken,
@@ -38,13 +39,13 @@ export function ObjectPermissionsView({
         mcpAccessGroups={mcpAccessGroups}
         mcpToolPermissions={mcpToolPermissions}
         mcpToolsets={mcpToolsets}
-        inheritedMcpServers={inheritedMcpServerIds}
+        inheritedMcpServers={inheritedMcpServers}
         accessToken={accessToken}
       />
       <AgentPermissions
         agents={agents}
         agentAccessGroups={agentAccessGroups}
-        inheritedAgents={inheritedAgentIds}
+        inheritedAgents={inheritedAgents}
         accessToken={accessToken}
       />
       <div className="min-w-0 rounded-md border border-border p-4">
diff --git a/ui/litellm-dashboard/src/components/permissions/AgentPermissions.test.tsx b/ui/litellm-dashboard/src/components/permissions/AgentPermissions.test.tsx
index 14fe6177ac5..01c7b60b8a5 100644
--- a/ui/litellm-dashboard/src/components/permissions/AgentPermissions.test.tsx
+++ b/ui/litellm-dashboard/src/components/permissions/AgentPermissions.test.tsx
@@ -1,5 +1,6 @@
 import { describe, it, expect, vi, beforeEach } from "vitest";
 import { render, screen } from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
 import AgentPermissions from "./AgentPermissions";
 import * as networking from "../networking";
 
@@ -13,31 +14,51 @@ describe("AgentPermissions", () => {
     vi.clearAllMocks();
   });
 
-  it("lists agents inherited from access groups with an Inherited tag and counts them", async () => {
+  it("lists agents inherited from access groups, counts them, and names the groups on hover", async () => {
+    const user = userEvent.setup();
     vi.mocked(networking.getAgentsList).mockResolvedValue({
       agents: [{ agent_id: agentId, agent_name: "support_agent" }],
     });
 
-    render(<AgentPermissions agents={[]} inheritedAgents={[agentId]} accessToken={accessToken} />);
+    render(
+      <AgentPermissions
+        agents={[]}
+        inheritedAgents={[{ id: agentId, accessGroupNames: ["platform-tools", "support"] }]}
+        accessToken={accessToken}
+      />,
+    );
 
-    expect(await screen.findByText(/support_agent/)).toBeInTheDocument();
-    expect(screen.getByText("Inherited")).toBeInTheDocument();
+    const row = await screen.findByText(/support_agent/);
     expect(screen.getByText("1")).toBeInTheDocument();
     expect(screen.queryByText("No agents or access groups configured")).not.toBeInTheDocument();
     expect(networking.getAgentsList).toHaveBeenCalledWith(accessToken);
+
+    await user.hover(row);
+    expect(
+      await screen.findByText(`Granted via access groups platform-tools, support. Full ID: ${agentId}`),
+    ).toBeInTheDocument();
   });
 
   it("does not double-list an agent that is both granted directly and inherited", async () => {
+    const user = userEvent.setup();
     vi.mocked(networking.getAgentsList).mockResolvedValue({
       agents: [{ agent_id: agentId, agent_name: "support_agent" }],
     });
 
-    render(<AgentPermissions agents={[agentId]} inheritedAgents={[agentId]} accessToken={accessToken} />);
+    render(
+      <AgentPermissions
+        agents={[agentId]}
+        inheritedAgents={[{ id: agentId, accessGroupNames: ["support"] }]}
+        accessToken={accessToken}
+      />,
+    );
 
-    expect(await screen.findByText(/support_agent/)).toBeInTheDocument();
+    const row = await screen.findByText(/support_agent/);
     expect(screen.getAllByText(/support_agent/)).toHaveLength(1);
-    expect(screen.queryByText("Inherited")).not.toBeInTheDocument();
     expect(screen.getByText("1")).toBeInTheDocument();
+
+    await user.hover(row);
+    expect(await screen.findByText(`Full ID: ${agentId}`)).toBeInTheDocument();
   });
 
   it("shows the empty state when nothing is granted directly or inherited", () => {
diff --git a/ui/litellm-dashboard/src/components/permissions/AgentPermissions.tsx b/ui/litellm-dashboard/src/components/permissions/AgentPermissions.tsx
index 11f6c414922..d1ca25c975b 100644
--- a/ui/litellm-dashboard/src/components/permissions/AgentPermissions.tsx
+++ b/ui/litellm-dashboard/src/components/permissions/AgentPermissions.tsx
@@ -3,6 +3,7 @@ import { UserGroupIcon } from "@heroicons/react/outline";
 import { Badge } from "@/components/ui/badge";
 import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip";
 import { getAgentsList } from "../networking";
+import { InheritedGrant, inheritedGrantTooltip } from "./inheritedGrants";
 
 interface Agent {
   agent_id: string;
@@ -14,12 +15,10 @@ interface Agent {
 interface AgentPermissionsProps {
   agents: string[];
   agentAccessGroups?: string[];
-  inheritedAgents?: string[];
+  inheritedAgents?: InheritedGrant[];
   accessToken?: string | null;
 }
 
-const INHERITED_AGENT_TOOLTIP = "Granted through one of the team's access groups";
-
 export function AgentPermissions({
   agents,
   agentAccessGroups = [],
@@ -27,7 +26,7 @@ export function AgentPermissions({
   accessToken,
 }: AgentPermissionsProps) {
   const [agentDetails, setAgentDetails] = useState<Agent[]>([]);
-  const inheritedOnlyAgents = inheritedAgents.filter((agent) => !agents.includes(agent));
+  const inheritedOnlyAgents = inheritedAgents.filter((grant) => !agents.includes(grant.id));
   const agentIdCount = agents.length + inheritedOnlyAgents.length;
 
   // Fetch agent details when component mounts
@@ -58,9 +57,9 @@ export function AgentPermissions({
   };
 
   const mergedItems = [
-    ...agents.map((agent) => ({ type: "agent", value: agent, inherited: false })),
-    ...inheritedOnlyAgents.map((agent) => ({ type: "agent", value: agent, inherited: true })),
-    ...agentAccessGroups.map((group) => ({ type: "accessGroup", value: group, inherited: false })),
+    ...agents.map((agent) => ({ type: "agent", value: agent, tooltip: `Full ID: ${agent}` })),
+    ...inheritedOnlyAgents.map((grant) => ({ type: "agent", value: grant.id, tooltip: inheritedGrantTooltip(grant) })),
+    ...agentAccessGroups.map((group) => ({ type: "accessGroup", value: group, tooltip: "" })),
   ];
   const totalCount = mergedItems.length;
 
@@ -86,17 +85,8 @@ export function AgentPermissions({
                           <span className="text-sm font-medium text-foreground truncate">
                             {getAgentDisplayName(item.value)}
                           </span>
-                          {item.inherited && (
-                            <span className="ml-1 px-1.5 py-0.5 text-[9px] font-semibold text-purple-600 bg-purple-50 border border-purple-200 rounded-sm uppercase tracking-wide shrink-0 dark:text-purple-300 dark:bg-purple-950 dark:border-purple-800">
-                              Inherited
-                            </span>
-                          )}
                         </TooltipTrigger>
-                        <TooltipContent>
-                          {item.inherited
-                            ? `${INHERITED_AGENT_TOOLTIP}. Full ID: ${item.value}`
-                            : `Full ID: ${item.value}`}
-                        </TooltipContent>
+                        <TooltipContent>{item.tooltip}</TooltipContent>
                       </Tooltip>
                     </TooltipProvider>
                   ) : (
diff --git a/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.test.tsx b/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.test.tsx
index fe7e3a7b971..78ede9865a4 100644
--- a/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.test.tsx
+++ b/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.test.tsx
@@ -407,7 +407,8 @@ describe("MCPServerPermissions", () => {
     await waitFor(() => expect(screen.getByText("Blocked")).toHaveAttribute("data-variant", "destructive"));
   });
 
-  it("lists servers inherited from access groups with an Inherited tag and counts them", async () => {
+  it("lists servers inherited from access groups, counts them, and names the group on hover", async () => {
+    const user = userEvent.setup();
     vi.mocked(networking.fetchMCPServers).mockResolvedValue([
       { server_id: mockServerId1, server_name: mockServerName1, alias: mockServerName1 },
     ]);
@@ -417,19 +418,24 @@ describe("MCPServerPermissions", () => {
         mcpServers={[]}
         mcpAccessGroups={[]}
         mcpToolPermissions={{}}
-        inheritedMcpServers={[mockServerId1]}
+        inheritedMcpServers={[{ id: mockServerId1, accessGroupNames: ["platform-tools"] }]}
         accessToken={mockAccessToken}
       />,
     );
 
-    expect(await screen.findByText(/DW_MCP/)).toBeInTheDocument();
-    expect(screen.getByText("Inherited")).toBeInTheDocument();
+    const row = await screen.findByText(/DW_MCP/);
     expect(screen.getByText("1")).toBeInTheDocument();
     expect(screen.queryByText("No MCP servers, access groups, or toolsets configured")).not.toBeInTheDocument();
     expect(networking.fetchMCPServers).toHaveBeenCalledWith(mockAccessToken);
+
+    await user.hover(row);
+    expect(
+      await screen.findByText(`Granted via access group platform-tools. Full ID: ${mockServerId1}`),
+    ).toBeInTheDocument();
   });
 
   it("does not double-list a server that is both granted directly and inherited", async () => {
+    const user = userEvent.setup();
     vi.mocked(networking.fetchMCPServers).mockResolvedValue([
       { server_id: mockServerId2, server_name: mockServerName2, alias: mockServerName2 },
     ]);
@@ -439,14 +445,16 @@ describe("MCPServerPermissions", () => {
         mcpServers={[mockServerId2]}
         mcpAccessGroups={[]}
         mcpToolPermissions={{}}
-        inheritedMcpServers={[mockServerId2]}
+        inheritedMcpServers={[{ id: mockServerId2, accessGroupNames: ["platform-tools"] }]}
         accessToken={mockAccessToken}
       />,
     );
 
-    expect(await screen.findByText(/Test Server/)).toBeInTheDocument();
+    const row = await screen.findByText(/Test Server/);
     expect(screen.getAllByText(/Test Server/)).toHaveLength(1);
-    expect(screen.queryByText("Inherited")).not.toBeInTheDocument();
     expect(screen.getByText("1")).toBeInTheDocument();
+
+    await user.hover(row);
+    expect(await screen.findByText(`Full ID: ${mockServerId2}`)).toBeInTheDocument();
   });
 });
diff --git a/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.tsx b/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.tsx
index 52199dab3c7..65f210addfa 100644
--- a/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.tsx
+++ b/ui/litellm-dashboard/src/components/permissions/MCPServerPermissions.tsx
@@ -5,18 +5,17 @@ import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip
 import { fetchMCPServers, fetchMCPToolsets } from "../networking";
 import { MCPServer, MCPToolset } from "../mcp_tools/types";
 import { ALL_PROXY_MCP_SERVERS_SENTINEL, NO_MCP_SERVERS_SENTINEL } from "../mcp_tools/constants";
+import { InheritedGrant, inheritedGrantTooltip } from "./inheritedGrants";
 
 interface MCPServerPermissionsProps {
   mcpServers: string[];
   mcpAccessGroups?: string[];
   mcpToolPermissions?: Record<string, string[]>;
   mcpToolsets?: string[];
-  inheritedMcpServers?: string[];
+  inheritedMcpServers?: InheritedGrant[];
   accessToken?: string | null;
 }
 
-const INHERITED_MCP_SERVER_TOOLTIP = "Granted through one of the team's access groups";
-
 export function MCPServerPermissions({
   mcpServers,
   mcpAccessGroups = [],
@@ -57,8 +56,8 @@ export function MCPServerPermissions({
   const directServerIds = mcpServers.filter(
     (server) => server !== NO_MCP_SERVERS_SENTINEL && server !== ALL_PROXY_MCP_SERVERS_SENTINEL,
   );
-  const inheritedOnlyServerIds = inheritedMcpServers.filter((server) => !mcpServers.includes(server));
-  const serverIdCount = directServerIds.length + inheritedOnlyServerIds.length;
+  const inheritedOnlyServers = inheritedMcpServers.filter((grant) => !mcpServers.includes(grant.id));
+  const serverIdCount = directServerIds.length + inheritedOnlyServers.length;
 
   // Fetch MCP server details when component mounts
   useEffect(() => {
@@ -109,9 +108,13 @@ export function MCPServerPermissions({
   const grantsAllProxyMcpServers = mcpServers.includes(ALL_PROXY_MCP_SERVERS_SENTINEL);
 
   const mergedItems = [
-    ...directServerIds.map((server) => ({ type: "server", value: server, inherited: false })),
-    ...inheritedOnlyServerIds.map((server) => ({ type: "server", value: server, inherited: true })),
-    ...mcpAccessGroups.map((group) => ({ type: "accessGroup", value: group, inherited: false })),
+    ...directServerIds.map((server) => ({ type: "server", value: server, tooltip: `Full ID: ${server}` })),
+    ...inheritedOnlyServers.map((grant) => ({
+      type: "server",
+      value: grant.id,
+      tooltip: inheritedGrantTooltip(grant),
+    })),
+    ...mcpAccessGroups.map((group) => ({ type: "accessGroup", value: group, tooltip: "" })),
   ];
   const totalCount = mergedItems.length + mcpToolsets.length;
 
@@ -160,17 +163,8 @@ export function MCPServerPermissions({
                           <span className="text-sm font-medium text-foreground truncate">
                             {getMCPServerDisplayName(item.value)}
                           </span>
-                          {item.inherited && (
-                            <span className="ml-1 px-1.5 py-0.5 text-[9px] font-semibold text-info bg-info/10 border border-info/20 rounded-sm uppercase tracking-wide shrink-0">
-                              Inherited
-                            </span>
-                          )}
                         </TooltipTrigger>
-                        <TooltipContent>
-                          {item.inherited
-                            ? `${INHERITED_MCP_SERVER_TOOLTIP}. Full ID: ${item.value}`
-                            : `Full ID: ${item.value}`}
-                        </TooltipContent>
+                        <TooltipContent>{item.tooltip}</TooltipContent>
                       </Tooltip>
                     ) : (
                       <div className="inline-flex items-center gap-2 min-w-0">
diff --git a/ui/litellm-dashboard/src/components/permissions/inheritedGrants.test.ts b/ui/litellm-dashboard/src/components/permissions/inheritedGrants.test.ts
new file mode 100644
index 00000000000..766f8f23f0b
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/permissions/inheritedGrants.test.ts
@@ -0,0 +1,61 @@
+import { describe, expect, it } from "vitest";
+import { computeInheritedGrants, inheritedGrantTooltip } from "./inheritedGrants";
+import { TeamAccessGroupModelGrant } from "../team/teamModelAccess";
+
+const GRANTS: TeamAccessGroupModelGrant[] = [
+  { access_group_id: "ag-1", access_group_name: "platform-tools", models: [], mcp_server_ids: ["mcp-1", "mcp-2"] },
+  {
+    access_group_id: "ag-2",
+    access_group_name: "support",
+    models: [],
+    mcp_server_ids: ["mcp-2"],
+    agent_ids: ["agent-1"],
+  },
+];
+
+describe("computeInheritedGrants", () => {
+  it("attributes each id to every group that grants it, in group order", () => {
+    expect(computeInheritedGrants(["mcp-1", "mcp-2"], GRANTS, (g) => g.mcp_server_ids)).toEqual([
+      { id: "mcp-1", accessGroupNames: ["platform-tools"] },
+      { id: "mcp-2", accessGroupNames: ["platform-tools", "support"] },
+    ]);
+  });
+
+  it("keeps ids the flat list carries but no group detail explains, with no group names", () => {
+    expect(computeInheritedGrants(["agent-1", "agent-legacy"], GRANTS, (g) => g.agent_ids)).toEqual([
+      { id: "agent-1", accessGroupNames: ["support"] },
+      { id: "agent-legacy", accessGroupNames: [] },
+    ]);
+  });
+
+  it("falls back to the group details when the flat list is missing, without duplicates", () => {
+    expect(computeInheritedGrants(undefined, GRANTS, (g) => g.mcp_server_ids).map((g) => g.id)).toEqual([
+      "mcp-1",
+      "mcp-2",
+    ]);
+  });
+
+  it("returns nothing when neither source has ids", () => {
+    expect(computeInheritedGrants(undefined, undefined, (g) => g.agent_ids)).toEqual([]);
+  });
+});
+
+describe("inheritedGrantTooltip", () => {
+  it("names a single group", () => {
+    expect(inheritedGrantTooltip({ id: "mcp-1", accessGroupNames: ["platform-tools"] })).toBe(
+      "Granted via access group platform-tools. Full ID: mcp-1",
+    );
+  });
+
+  it("lists several groups", () => {
+    expect(inheritedGrantTooltip({ id: "mcp-2", accessGroupNames: ["platform-tools", "support"] })).toBe(
+      "Granted via access groups platform-tools, support. Full ID: mcp-2",
+    );
+  });
+
+  it("stays generic when the proxy did not say which group granted it", () => {
+    expect(inheritedGrantTooltip({ id: "agent-legacy", accessGroupNames: [] })).toBe(
+      "Granted via an access group. Full ID: agent-legacy",
+    );
+  });
+});
diff --git a/ui/litellm-dashboard/src/components/permissions/inheritedGrants.ts b/ui/litellm-dashboard/src/components/permissions/inheritedGrants.ts
new file mode 100644
index 00000000000..fb78de5ed77
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/permissions/inheritedGrants.ts
@@ -0,0 +1,26 @@
+import { describeGroups, TeamAccessGroupModelGrant } from "../team/teamModelAccess";
+
+export interface InheritedGrant {
+  id: string;
+  accessGroupNames: string[];
+}
+
+export function computeInheritedGrants(
+  ids: string[] | undefined,
+  grants: TeamAccessGroupModelGrant[] | undefined,
+  idsOf: (grant: TeamAccessGroupModelGrant) => string[] | undefined,
+): InheritedGrant[] {
+  const known = grants ?? [];
+  const allIds = [...new Set([...(ids ?? []), ...known.flatMap((grant) => idsOf(grant) ?? [])])];
+  return allIds.map((id) => ({
+    id,
+    accessGroupNames: known
+      .filter((grant) => (idsOf(grant) ?? []).includes(id))
+      .map((grant) => grant.access_group_name),
+  }));
+}
+
+export const inheritedGrantTooltip = (grant: InheritedGrant): string => {
+  const source = grant.accessGroupNames.length > 0 ? describeGroups(grant.accessGroupNames) : "an access group";
+  return `Granted via ${source}. Full ID: ${grant.id}`;
+};
diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx
index 40a3d2ada1a..ae78ac06a9c 100644
--- a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx
+++ b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx
@@ -305,7 +305,8 @@ describe("TeamInfoView", () => {
       );
     });
 
-    it("shows MCP servers and agents inherited from access groups in the Object Permissions card", async () => {
+    it("shows MCP servers and agents inherited from access groups in the Object Permissions card, naming the group on hover", async () => {
+      const user = userEvent.setup();
       vi.mocked(networking.fetchMCPServers).mockResolvedValue([
         { server_id: "mcp-github-1234", server_name: "github", alias: "github" },
       ]);
@@ -318,16 +319,33 @@ describe("TeamInfoView", () => {
           access_group_ids: ["ag-1"],
           access_group_mcp_server_ids: ["mcp-github-1234"],
           access_group_agent_ids: ["agent-support-5678"],
+          access_group_details: [
+            {
+              access_group_id: "ag-1",
+              access_group_name: "platform-tools",
+              models: [],
+              mcp_server_ids: ["mcp-github-1234"],
+              agent_ids: ["agent-support-5678"],
+            },
+          ],
         }),
       );
 
       renderWithProviders(<TeamInfoView {...defaultProps} />);
 
-      expect(await screen.findByText(/github \(mcp\.\.\.1234\)/)).toBeInTheDocument();
-      expect(await screen.findByText(/support_agent \(age\.\.\.5678\)/)).toBeInTheDocument();
-      expect(screen.getAllByText("Inherited")).toHaveLength(2);
+      const serverRow = await screen.findByText(/github \(mcp\.\.\.1234\)/);
+      const agentRow = await screen.findByText(/support_agent \(age\.\.\.5678\)/);
       expect(screen.queryByText("No MCP servers, access groups, or toolsets configured")).not.toBeInTheDocument();
       expect(screen.queryByText("No agents or access groups configured")).not.toBeInTheDocument();
+
+      await user.hover(serverRow);
+      expect(
+        await screen.findByText("Granted via access group platform-tools. Full ID: mcp-github-1234"),
+      ).toBeInTheDocument();
+      await user.hover(agentRow);
+      expect(
+        await screen.findByText("Granted via access group platform-tools. Full ID: agent-support-5678"),
+      ).toBeInTheDocument();
     });
 
     it("keeps the all-proxy-models badge non-clickable", async () => {
diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx
index 7ebef0691e6..3f6d6a96972 100644
--- a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx
+++ b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx
@@ -58,6 +58,7 @@ import {
   TeamModelBadge,
   TeamModelBadgeKind,
 } from "./teamModelAccess";
+import { computeInheritedGrants } from "../permissions/inheritedGrants";
 import MetadataKeyValueFields, {
   metadataObjectToPairs,
   metadataPairsSchema,
@@ -936,6 +937,17 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({
 
   const { team_info: info } = teamData;
 
+  const inheritedMcpServers = computeInheritedGrants(
+    info.access_group_mcp_server_ids,
+    info.access_group_details,
+    (grant) => grant.mcp_server_ids,
+  );
+  const inheritedAgents = computeInheritedGrants(
+    info.access_group_agent_ids,
+    info.access_group_details,
+    (grant) => grant.agent_ids,
+  );
+
   const initialKillSwitchOn = info.metadata?.disable_global_guardrails === true;
 
   const allGuardrails: GuardrailListItem[] = guardrailsData?.guardrails ?? [];
@@ -1035,8 +1047,8 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({
 
           <ObjectPermissionsView
             objectPermission={info.object_permission}
-            inheritedMcpServerIds={info.access_group_mcp_server_ids}
-            inheritedAgentIds={info.access_group_agent_ids}
+            inheritedMcpServers={inheritedMcpServers}
+            inheritedAgents={inheritedAgents}
             variant="card"
             accessToken={accessToken}
           />
@@ -1889,8 +1901,8 @@ const TeamInfoView: React.FC<TeamInfoProps> = ({
 
               <ObjectPermissionsView
                 objectPermission={info.object_permission}
-                inheritedMcpServerIds={info.access_group_mcp_server_ids}
-                inheritedAgentIds={info.access_group_agent_ids}
+                inheritedMcpServers={inheritedMcpServers}
+                inheritedAgents={inheritedAgents}
                 variant="inline"
                 className="pt-4 border-t border-border"
                 accessToken={accessToken}
diff --git a/ui/litellm-dashboard/src/components/team/teamModelAccess.ts b/ui/litellm-dashboard/src/components/team/teamModelAccess.ts
index 91ddf0f4045..bc8022592bd 100644
--- a/ui/litellm-dashboard/src/components/team/teamModelAccess.ts
+++ b/ui/litellm-dashboard/src/components/team/teamModelAccess.ts
@@ -5,6 +5,8 @@ export interface TeamAccessGroupModelGrant {
   access_group_id: string;
   access_group_name: string;
   models: string[];
+  mcp_server_ids?: string[];
+  agent_ids?: string[];
 }
 
 export type TeamModelBadgeKind = "all-proxy" | "no-default" | "direct" | "access-group";
@@ -19,7 +21,7 @@ export function normalizeTeamModelSelection(models: string[] | undefined): strin
   return models && models.length > 0 ? models : [NO_DEFAULT_MODELS];
 }
 
-const describeGroups = (names: string[]): string =>
+export const describeGroups = (names: string[]): string =>
   names.length > 1 ? `access groups ${names.join(", ")}` : `access group ${names[0]}`;
 
 export function computeTeamModelBadges(

From b9e030ddd662cb7abb98b8fa5139efb14d57cb6c Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Thu, 3 Sep 2026 10:38:25 -0700
Subject: [PATCH 20/42] fix(cost): apply off_peak_pricing in the dashscope cost
 calculator

---
 .../litellm_core_utils/llm_cost_calc/utils.py |   4 +-
 litellm/llms/dashscope/cost_calculator.py     | 120 ++++++++-------
 .../test_dashscope_cost_calculator.py         | 138 +++++++++++++++++-
 3 files changed, 209 insertions(+), 53 deletions(-)

diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py
index b34c416cd40..21587af73aa 100644
--- a/litellm/litellm_core_utils/llm_cost_calc/utils.py
+++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py
@@ -428,7 +428,7 @@ def _coerce_off_peak_rate(value: object, default: float) -> float:
     return default
 
 
-def _apply_off_peak_pricing(
+def apply_off_peak_pricing(
     model_info: ModelInfo,
     current_time: datetime | None,
     prompt_base_cost: float,
@@ -462,7 +462,7 @@ def _apply_off_peak_to_base_costs(
     has no field for them.
     """
     prompt, completion, cache_creation, cache_creation_above_1hr, cache_read = base_costs
-    off_peak_prompt, off_peak_completion, off_peak_cache_read = _apply_off_peak_pricing(
+    off_peak_prompt, off_peak_completion, off_peak_cache_read = apply_off_peak_pricing(
         model_info, current_time, prompt, completion, cache_read
     )
     return (off_peak_prompt, off_peak_completion, cache_creation, cache_creation_above_1hr, off_peak_cache_read)
diff --git a/litellm/llms/dashscope/cost_calculator.py b/litellm/llms/dashscope/cost_calculator.py
index dd5bee1fe8b..d8eb1f9f8d7 100644
--- a/litellm/llms/dashscope/cost_calculator.py
+++ b/litellm/llms/dashscope/cost_calculator.py
@@ -7,11 +7,13 @@ cached, cache-creation, output, reasoning) is billed at that one tier's rate.
 See https://help.aliyun.com/zh/model-studio/billing-for-model-studio
 """
 
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
+from datetime import datetime
 from typing import Final
 
 from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import select_tier_for_input, tier_rate
 from litellm.litellm_core_utils.llm_cost_calc.utils import (
+    apply_off_peak_pricing,
     parse_completion_tokens_details,
     parse_prompt_tokens_details,
 )
@@ -32,6 +34,19 @@ class TokenBreakdown:
         return self.text_tokens + self.cached_tokens + self.cache_creation_tokens
 
 
+@dataclass(frozen=True, slots=True)
+class TokenRates:
+    input_rate: float
+    cache_read_rate: float
+    cache_creation_rate: float
+    output_rate: float
+    reasoning_rate: float | None
+
+    @property
+    def billed_reasoning_rate(self) -> float:
+        return self.output_rate if self.reasoning_rate is None else self.reasoning_rate
+
+
 def _extract_token_breakdown(usage: Usage) -> TokenBreakdown:
     prompt_details: Final = parse_prompt_tokens_details(usage)
     cached_tokens: Final = prompt_details["cache_hit_tokens"]
@@ -57,69 +72,75 @@ def _flat_rate(model_info: ModelInfo, cost_key: str, fallback_cost_key: str) ->
     return float(value)
 
 
-def _calculate_prompt_cost(
-    breakdown: TokenBreakdown,
-    model_info: ModelInfo,
-    tier: dict | None,
-) -> float:
-    if tier is not None:
-        return (
-            (breakdown.text_tokens * tier_rate(tier, "input_cost_per_token"))
-            + (breakdown.cached_tokens * tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token"))
-            + (
-                breakdown.cache_creation_tokens
-                * tier_rate(tier, "cache_creation_input_token_cost", "input_cost_per_token")
-            )
-        )
-
-    input_cost: Final = float(model_info.get("input_cost_per_token") or 0.0)
-    cache_read_cost: Final = _flat_rate(model_info, "cache_read_input_token_cost", "input_cost_per_token")
-    cache_creation_cost: Final = _flat_rate(model_info, "cache_creation_input_token_cost", "input_cost_per_token")
-
-    return (
-        (breakdown.text_tokens * input_cost)
-        + (breakdown.cached_tokens * cache_read_cost)
-        + (breakdown.cache_creation_tokens * cache_creation_cost)
+def _flat_rates(model_info: ModelInfo) -> TokenRates:
+    reasoning_rate: Final = model_info.get("output_cost_per_reasoning_token")
+    return TokenRates(
+        input_rate=float(model_info.get("input_cost_per_token") or 0.0),
+        cache_read_rate=_flat_rate(model_info, "cache_read_input_token_cost", "input_cost_per_token"),
+        cache_creation_rate=_flat_rate(model_info, "cache_creation_input_token_cost", "input_cost_per_token"),
+        output_rate=float(model_info.get("output_cost_per_token") or 0.0),
+        reasoning_rate=None if reasoning_rate is None else float(reasoning_rate),
     )
 
 
-def _calculate_completion_cost(
-    breakdown: TokenBreakdown,
-    model_info: ModelInfo,
-    tier: dict | None,
-) -> float:
+def _tier_rates(model_info: ModelInfo, tier: dict) -> TokenRates:
     # A tier that declares output rates keeps the request on them, all-or-nothing. A tier table
     # spelling out only input rates would serve every completion for free, so there the model's
     # own output rates stand in
-    tier_declares_output: Final = tier is not None and "output_cost_per_token" in tier
-    output_cost: Final = (
-        tier_rate(tier, "output_cost_per_token")
-        if tier_declares_output
-        else float(model_info.get("output_cost_per_token") or 0.0)
-    )
-    tier_declares_reasoning: Final = tier is not None and "output_cost_per_reasoning_token" in tier
-    model_reasoning_rate: Final = None if tier_declares_output else model_info.get("output_cost_per_reasoning_token")
-    reasoning_cost: Final = (
-        tier_rate(tier, "output_cost_per_reasoning_token", "output_cost_per_token")
-        if tier_declares_reasoning
-        else float(model_reasoning_rate)
-        if model_reasoning_rate is not None
-        else output_cost
+    flat_rates: Final = _flat_rates(model_info)
+    tier_declares_output: Final = "output_cost_per_token" in tier
+    tier_declares_reasoning: Final = "output_cost_per_reasoning_token" in tier
+    return TokenRates(
+        input_rate=tier_rate(tier, "input_cost_per_token"),
+        cache_read_rate=tier_rate(tier, "cache_read_input_token_cost", "input_cost_per_token"),
+        cache_creation_rate=tier_rate(tier, "cache_creation_input_token_cost", "input_cost_per_token"),
+        output_rate=tier_rate(tier, "output_cost_per_token") if tier_declares_output else flat_rates.output_rate,
+        reasoning_rate=(
+            tier_rate(tier, "output_cost_per_reasoning_token")
+            if tier_declares_reasoning
+            else None
+            if tier_declares_output
+            else flat_rates.reasoning_rate
+        ),
     )
 
-    return (breakdown.completion_tokens * output_cost) + (breakdown.reasoning_tokens * reasoning_cost)
+
+def _off_peak_rates(model_info: ModelInfo, current_time: datetime | None, rates: TokenRates) -> TokenRates:
+    input_rate, output_rate, cache_read_rate = apply_off_peak_pricing(
+        model_info, current_time, rates.input_rate, rates.output_rate, rates.cache_read_rate
+    )
+    return replace(rates, input_rate=input_rate, output_rate=output_rate, cache_read_rate=cache_read_rate)
 
 
-def cost_per_token(model: str, usage: Usage, custom_llm_provider: str = "dashscope") -> tuple[float, float]:
+def _bill(breakdown: TokenBreakdown, rates: TokenRates) -> tuple[float, float]:
+    prompt_cost: Final = (
+        (breakdown.text_tokens * rates.input_rate)
+        + (breakdown.cached_tokens * rates.cache_read_rate)
+        + (breakdown.cache_creation_tokens * rates.cache_creation_rate)
+    )
+    completion_cost: Final = (breakdown.completion_tokens * rates.output_rate) + (
+        breakdown.reasoning_tokens * rates.billed_reasoning_rate
+    )
+    return prompt_cost, completion_cost
+
+
+def cost_per_token(
+    model: str,
+    usage: Usage,
+    custom_llm_provider: str = "dashscope",
+    current_time: datetime | None = None,
+) -> tuple[float, float]:
     """
     Calculate cost per token for Dashscope models.
 
-    Supports both tiered and flat pricing with cached and reasoning tokens.
+    Supports both tiered and flat pricing with cached and reasoning tokens, and swaps in the
+    model's off_peak_pricing rates while one of its windows is open.
 
     Args:
         model: Model name without provider prefix
         usage: LiteLLM Usage block
         custom_llm_provider: The provider id the request resolved to; dashscope or one of its brand aliases
+        current_time: The moment the request is billed at; defaults to now, UTC
 
     Returns:
         Tuple[float, float] - (prompt_cost_in_usd, completion_cost_in_usd)
@@ -133,8 +154,7 @@ def cost_per_token(model: str, usage: Usage, custom_llm_provider: str = "dashsco
         if tiered_pricing
         else None
     )
+    standard_rates: Final = _flat_rates(model_info) if tier is None else _tier_rates(model_info, tier)
+    rates: Final = _off_peak_rates(model_info, current_time, standard_rates)
 
-    prompt_cost: Final = _calculate_prompt_cost(breakdown=breakdown, model_info=model_info, tier=tier)
-    completion_cost: Final = _calculate_completion_cost(breakdown=breakdown, model_info=model_info, tier=tier)
-
-    return prompt_cost, completion_cost
+    return _bill(breakdown, rates)
diff --git a/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py b/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py
index 8dc4620dd1b..b6281834f24 100644
--- a/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py
+++ b/tests/test_litellm/llms/dashscope/test_dashscope_cost_calculator.py
@@ -10,11 +10,11 @@ Tests the cost calculation for Dashscope models including:
 
 import math
 import os
+from datetime import datetime, timezone
 
 import pytest
 
 # Add the project root to Python path
-
 import litellm
 from litellm.llms.dashscope.cost_calculator import (
     cost_per_token as dashscope_cost_per_token,
@@ -526,3 +526,139 @@ class TestDashscopeCostCalculator:
 
         assert prompt_cost == 0.0
         assert math.isclose(completion_cost, 500 * 1.6e-06, rel_tol=1e-10)
+
+    OFF_PEAK_WINDOW = "14:00-00:00"
+    INSIDE_WINDOW = datetime(2026, 9, 3, 17, 25, tzinfo=timezone.utc)
+    OUTSIDE_WINDOW = datetime(2026, 9, 3, 9, 0, tzinfo=timezone.utc)
+
+    def _register_off_peak_flat_model(self, model_key: str, off_peak_pricing: dict) -> None:
+        litellm.model_cost[model_key] = {
+            "litellm_provider": "dashscope",
+            "mode": "chat",
+            "input_cost_per_token": 2.4e-06,
+            "output_cost_per_token": 4.8e-06,
+            "cache_read_input_token_cost": 2e-07,
+            "cache_creation_input_token_cost": 3e-06,
+            "off_peak_pricing": off_peak_pricing,
+        }
+
+    def test_dashscope_off_peak_window_swaps_in_the_off_peak_rates(self):
+        """
+        Regression (LIT-6782): a deployment configured with off_peak_pricing kept billing the
+        standard dashscope rates inside its window, while the same block on a deepseek
+        deployment billed the off-peak rates.
+        """
+        self._register_off_peak_flat_model(
+            "dashscope/deepseek-off-peak-test",
+            {
+                "hours_utc": self.OFF_PEAK_WINDOW,
+                "input_cost_per_token": 1.2e-06,
+                "output_cost_per_token": 2.4e-06,
+                "cache_read_input_token_cost": 1e-07,
+            },
+        )
+        usage = Usage(
+            prompt_tokens=1000,
+            completion_tokens=200,
+            prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=300, cache_creation_tokens=100),
+        )
+
+        prompt_cost, completion_cost = dashscope_cost_per_token(
+            model="deepseek-off-peak-test", usage=usage, current_time=self.INSIDE_WINDOW
+        )
+
+        assert math.isclose(prompt_cost, (600 * 1.2e-06) + (300 * 1e-07) + (100 * 3e-06), rel_tol=1e-10)
+        assert math.isclose(completion_cost, 200 * 2.4e-06, rel_tol=1e-10)
+
+        peak_prompt_cost, peak_completion_cost = dashscope_cost_per_token(
+            model="deepseek-off-peak-test", usage=usage, current_time=self.OUTSIDE_WINDOW
+        )
+
+        assert math.isclose(peak_prompt_cost, (600 * 2.4e-06) + (300 * 2e-07) + (100 * 3e-06), rel_tol=1e-10)
+        assert math.isclose(peak_completion_cost, 200 * 4.8e-06, rel_tol=1e-10)
+
+    def test_dashscope_off_peak_window_overrides_the_selected_tier(self):
+        """An open off-peak window bills the whole request at the flat off-peak rates, whichever tier
+        the input volume selected."""
+        self._register_tiered_model(
+            "dashscope/qwen-tiered-off-peak-test",
+            [
+                {"range": [0, 1000], "input_cost_per_token": 4e-07, "output_cost_per_token": 1.6e-06},
+                {"range": [1000, 2000], "input_cost_per_token": 8e-07, "output_cost_per_token": 3.2e-06},
+            ],
+        )
+        litellm.model_cost["dashscope/qwen-tiered-off-peak-test"]["off_peak_pricing"] = {
+            "hours_utc": self.OFF_PEAK_WINDOW,
+            "input_cost_per_token": 1e-07,
+            "output_cost_per_token": 4e-07,
+        }
+        usage = Usage(prompt_tokens=1500, completion_tokens=300)
+
+        prompt_cost, completion_cost = dashscope_cost_per_token(
+            model="qwen-tiered-off-peak-test", usage=usage, current_time=self.INSIDE_WINDOW
+        )
+
+        assert math.isclose(prompt_cost, 1500 * 1e-07, rel_tol=1e-10)
+        assert math.isclose(completion_cost, 300 * 4e-07, rel_tol=1e-10)
+
+        peak_prompt_cost, peak_completion_cost = dashscope_cost_per_token(
+            model="qwen-tiered-off-peak-test", usage=usage, current_time=self.OUTSIDE_WINDOW
+        )
+
+        assert math.isclose(peak_prompt_cost, 1500 * 8e-07, rel_tol=1e-10)
+        assert math.isclose(peak_completion_cost, 300 * 3.2e-06, rel_tol=1e-10)
+
+    def test_dashscope_off_peak_rates_left_unset_keep_the_standard_rates(self):
+        """A block that only overrides the input rate leaves output and cache reads on the standard
+        rates, and an explicit reasoning rate is never swapped out."""
+        self._register_off_peak_flat_model(
+            "dashscope/qwen-partial-off-peak-test",
+            {"hours_utc": self.OFF_PEAK_WINDOW, "input_cost_per_token": 1.2e-06},
+        )
+        litellm.model_cost["dashscope/qwen-partial-off-peak-test"]["output_cost_per_reasoning_token"] = 9e-06
+        usage = Usage(
+            prompt_tokens=1000,
+            completion_tokens=200,
+            prompt_tokens_details=PromptTokensDetailsWrapper(cached_tokens=300),
+            completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=50),
+        )
+
+        prompt_cost, completion_cost = dashscope_cost_per_token(
+            model="qwen-partial-off-peak-test", usage=usage, current_time=self.INSIDE_WINDOW
+        )
+
+        assert math.isclose(prompt_cost, (700 * 1.2e-06) + (300 * 2e-07), rel_tol=1e-10)
+        assert math.isclose(completion_cost, (150 * 4.8e-06) + (50 * 9e-06), rel_tol=1e-10)
+
+    def test_dashscope_off_peak_output_rate_covers_reasoning_without_a_dedicated_rate(self):
+        """Reasoning tokens on a model with no dedicated reasoning rate follow the off-peak output
+        rate, the same way they follow the standard output rate outside the window."""
+        self._register_off_peak_flat_model(
+            "dashscope/qwen-reasoning-off-peak-test",
+            {"hours_utc": self.OFF_PEAK_WINDOW, "output_cost_per_token": 2.4e-06},
+        )
+        usage = Usage(
+            prompt_tokens=100,
+            completion_tokens=200,
+            completion_tokens_details=CompletionTokensDetailsWrapper(reasoning_tokens=50),
+        )
+
+        _, completion_cost = dashscope_cost_per_token(
+            model="qwen-reasoning-off-peak-test", usage=usage, current_time=self.INSIDE_WINDOW
+        )
+
+        assert math.isclose(completion_cost, 200 * 2.4e-06, rel_tol=1e-10)
+
+    def test_dashscope_off_peak_defaults_to_the_current_time(self):
+        """The proxy's cost dispatch passes no clock, so an all-day window has to apply on the
+        default current time."""
+        self._register_off_peak_flat_model(
+            "dashscope/qwen-all-day-off-peak-test",
+            {"hours_utc": "00:00-00:00", "input_cost_per_token": 1.2e-06, "output_cost_per_token": 2.4e-06},
+        )
+        usage = Usage(prompt_tokens=1000, completion_tokens=200)
+
+        prompt_cost, completion_cost = dashscope_cost_per_token(model="qwen-all-day-off-peak-test", usage=usage)
+
+        assert math.isclose(prompt_cost, 1000 * 1.2e-06, rel_tol=1e-10)
+        assert math.isclose(completion_cost, 200 * 2.4e-06, rel_tol=1e-10)

From 68ffa1db230ee2a03e851f567ce03ec48ce83e9b Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Thu, 3 Sep 2026 10:43:41 -0700
Subject: [PATCH 21/42] fix(router): pin JWT-authenticated callers by user id
 in deployment_affinity

---
 .../deployment_affinity_check.py              |  36 +--
 .../test_deployment_affinity_check.py         | 207 ++++++++++++++++++
 2 files changed, 225 insertions(+), 18 deletions(-)

diff --git a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py
index 39d3e25aacb..ea450864604 100644
--- a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py
+++ b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py
@@ -82,6 +82,7 @@ class DeploymentAffinityCheck(CustomLogger):
     """
 
     CACHE_KEY_PREFIX = "deployment_affinity:v1"
+    USER_ID_AFFINITY_PREFIX: Final = "user_id:"
 
     def __init__(
         self,
@@ -253,15 +254,6 @@ class DeploymentAffinityCheck(CustomLogger):
         hashed_user_key: Final = cls._hash_user_key(user_key) if user_key is not None else "unscoped"
         return f"{cls.CACHE_KEY_PREFIX}:session:{model_group}:{hashed_user_key}:{session_id}"
 
-    @staticmethod
-    def _get_user_key_from_metadata_dict(metadata: dict) -> str | None:
-        # NOTE: affinity is keyed on the *API key hash* provided by the proxy (not the
-        # OpenAI `user` parameter, which is an end-user identifier).
-        user_key: Final = metadata.get("user_api_key_hash")
-        if user_key is None:
-            return None
-        return str(user_key)
-
     @staticmethod
     def _get_session_id_from_metadata_dict(metadata: dict) -> str | None:
         session_id: Final = metadata.get("session_id")
@@ -285,22 +277,30 @@ class DeploymentAffinityCheck(CustomLogger):
         return metadata_dicts
 
     @staticmethod
-    def _get_user_key_from_request_kwargs(request_kwargs: dict) -> str | None:
+    def _first_metadata_value(metadata_dicts: Sequence[dict], key: str) -> str | None:
+        value: Final = next((metadata[key] for metadata in metadata_dicts if metadata.get(key) is not None), None)
+        return None if value is None else str(value)
+
+    @classmethod
+    def _get_user_key_from_request_kwargs(cls, request_kwargs: dict) -> str | None:
         """
         Extract a stable affinity key from request kwargs.
 
-        Source (proxy): `metadata.user_api_key_hash`
+        Source (proxy): `metadata.user_api_key_hash` for virtual-key callers. JWT-authenticated
+        callers carry no key hash, so their `metadata.user_api_key_user_id` stands in for it,
+        namespaced under `USER_ID_AFFINITY_PREFIX` so a user id can never alias a key hash.
 
         Note: the OpenAI `user` parameter is an end-user identifier and is intentionally
         not used for deployment affinity.
         """
-        # Check metadata dicts (Proxy usage)
-        for metadata in DeploymentAffinityCheck._iter_metadata_dicts(request_kwargs):
-            user_key = DeploymentAffinityCheck._get_user_key_from_metadata_dict(metadata=metadata)
-            if user_key is not None:
-                return user_key
-
-        return None
+        metadata_dicts: Final = cls._iter_metadata_dicts(request_kwargs)
+        user_api_key_hash: Final = cls._first_metadata_value(metadata_dicts, "user_api_key_hash")
+        if user_api_key_hash is not None:
+            return user_api_key_hash
+        user_id: Final = cls._first_metadata_value(metadata_dicts, "user_api_key_user_id")
+        if user_id is None:
+            return None
+        return f"{cls.USER_ID_AFFINITY_PREFIX}{user_id}"
 
     @staticmethod
     def _get_session_id_from_request_kwargs(request_kwargs: dict) -> str | None:
diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_deployment_affinity_check.py b/tests/test_litellm/router_utils/pre_call_checks/test_deployment_affinity_check.py
index 60433921de6..1852d641f0a 100644
--- a/tests/test_litellm/router_utils/pre_call_checks/test_deployment_affinity_check.py
+++ b/tests/test_litellm/router_utils/pre_call_checks/test_deployment_affinity_check.py
@@ -998,3 +998,210 @@ async def test_model_group_affinity_config_overrides_global():
     )
     # All deployments returned (user-key affinity disabled for this group)
     assert len(filtered) == 2
+
+
+def _jwt_metadata(user_id: str) -> dict:
+    return {"user_api_key_hash": None, "user_api_key_user_id": user_id}
+
+
+def _two_deployments(model_group: str) -> list[dict]:
+    return [
+        {
+            "model_name": model_group,
+            "litellm_params": {"model": "openai/gpt-5.4-mini"},
+            "model_info": {"id": "openai-deployment-a"},
+        },
+        {
+            "model_name": model_group,
+            "litellm_params": {"model": "openai/gpt-5.4-mini"},
+            "model_info": {"id": "openai-deployment-b"},
+        },
+    ]
+
+
+@pytest.mark.asyncio
+async def test_async_jwt_user_affinity_routes_to_same_deployment():
+    """
+    JWT-authenticated proxy requests carry no `user_api_key_hash`, only `user_api_key_user_id`.
+    They must still pin to one deployment per user.
+    """
+    model_group = "gpt-5.4-mini"
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": model_group,
+                "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "mock-api-key-a"},
+                "model_info": {"id": "openai-deployment-a"},
+            },
+            {
+                "model_name": model_group,
+                "litellm_params": {"model": "openai/gpt-5.4-mini", "api_key": "mock-api-key-b"},
+                "model_info": {"id": "openai-deployment-b"},
+            },
+        ],
+        optional_pre_call_checks=["deployment_affinity"],
+    )
+
+    choice_calls = {"count": 0}
+
+    def deterministic_choice(seq):
+        choice_calls["count"] += 1
+        if choice_calls["count"] == 1:
+            return seq[0]
+        return seq[1] if len(seq) > 1 else seq[0]
+
+    with patch(  # test-quality-ok: simple-shuffle has no injectable RNG; forcing the other pick is what proves the pin overrides the strategy
+        "litellm.router_strategy.simple_shuffle.random.choice",
+        side_effect=deterministic_choice,
+    ):
+        first_response = await router.acompletion(
+            model=model_group,
+            messages=[{"role": "user", "content": "Reply with the single word ok"}],
+            mock_response="ok",
+            metadata=_jwt_metadata("jwt-user-alice"),
+        )
+        second_response = await router.acompletion(
+            model=model_group,
+            messages=[{"role": "user", "content": "Reply with the single word ok"}],
+            mock_response="ok",
+            metadata=_jwt_metadata("jwt-user-alice"),
+        )
+
+    first_model_id = first_response._hidden_params["model_id"]
+    assert first_model_id in ("openai-deployment-a", "openai-deployment-b")
+    assert second_response._hidden_params["model_id"] == first_model_id
+
+
+@pytest.mark.asyncio
+async def test_proxy_jwt_auth_metadata_pins_per_user():
+    """
+    The metadata the proxy stamps for a JWT caller (`UserAPIKeyAuth(api_key=None, user_id=<sub>)`)
+    must claim a pin and be read back by the filter, and another JWT user must not inherit it.
+    """
+    from litellm.proxy._types import UserAPIKeyAuth
+    from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
+
+    model_group = "gpt-5.4-mini"
+    healthy_deployments = _two_deployments(model_group)
+    callback = DeploymentAffinityCheck(
+        cache=DualCache(),
+        ttl_seconds=60,
+        enable_user_key_affinity=True,
+        enable_responses_api_affinity=False,
+    )
+
+    def proxy_request(user_id: str) -> dict:
+        return LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata(
+            data={"model": model_group, "messages": [{"role": "user", "content": "hi"}], "metadata": {}},
+            user_api_key_dict=UserAPIKeyAuth(api_key=None, user_id=user_id),
+            _metadata_variable_name="metadata",
+        )
+
+    alice_request = proxy_request("jwt-user-alice")
+    assert alice_request["metadata"]["user_api_key_hash"] is None
+
+    await callback.async_pre_call_deployment_hook(
+        kwargs={
+            **alice_request,
+            "metadata": {**alice_request["metadata"], "deployment_model_name": model_group},
+            "model_info": {"id": "openai-deployment-b"},
+        },
+        call_type=None,
+    )
+
+    alice_pinned = await callback.async_filter_deployments(
+        model=model_group,
+        healthy_deployments=healthy_deployments,
+        messages=None,
+        request_kwargs=alice_request,
+        parent_otel_span=None,
+    )
+    assert [deployment["model_info"]["id"] for deployment in alice_pinned] == ["openai-deployment-b"]
+
+    bob_filtered = await callback.async_filter_deployments(
+        model=model_group,
+        healthy_deployments=healthy_deployments,
+        messages=None,
+        request_kwargs=proxy_request("jwt-user-bob"),
+        parent_otel_span=None,
+    )
+    assert bob_filtered == healthy_deployments
+
+
+@pytest.mark.asyncio
+async def test_jwt_user_id_never_reads_a_virtual_key_pin():
+    """
+    A JWT user id that happens to equal a virtual key's 64-hex hash must not read that key's pin.
+    """
+    model_group = "gpt-5.4-mini"
+    healthy_deployments = _two_deployments(model_group)
+    callback = DeploymentAffinityCheck(
+        cache=DualCache(),
+        ttl_seconds=60,
+        enable_user_key_affinity=True,
+        enable_responses_api_affinity=False,
+    )
+    key_hash = "a" * 64
+
+    await callback.async_pre_call_deployment_hook(
+        kwargs={
+            "metadata": {"user_api_key_hash": key_hash, "deployment_model_name": model_group},
+            "model_info": {"id": "openai-deployment-b"},
+        },
+        call_type=None,
+    )
+
+    key_pinned = await callback.async_filter_deployments(
+        model=model_group,
+        healthy_deployments=healthy_deployments,
+        messages=None,
+        request_kwargs={"metadata": {"user_api_key_hash": key_hash}},
+        parent_otel_span=None,
+    )
+    assert [deployment["model_info"]["id"] for deployment in key_pinned] == ["openai-deployment-b"]
+
+    lookalike_jwt_user = await callback.async_filter_deployments(
+        model=model_group,
+        healthy_deployments=healthy_deployments,
+        messages=None,
+        request_kwargs={"metadata": _jwt_metadata(key_hash)},
+        parent_otel_span=None,
+    )
+    assert lookalike_jwt_user == healthy_deployments
+
+
+@pytest.mark.asyncio
+async def test_virtual_key_hash_wins_over_user_id_for_affinity():
+    """
+    A virtual-key caller with a user id pins on the key hash, so two keys owned by one user
+    keep independent pins.
+    """
+    model_group = "gpt-5.4-mini"
+    healthy_deployments = _two_deployments(model_group)
+    callback = DeploymentAffinityCheck(
+        cache=DualCache(),
+        ttl_seconds=60,
+        enable_user_key_affinity=True,
+        enable_responses_api_affinity=False,
+    )
+
+    await callback.async_pre_call_deployment_hook(
+        kwargs={
+            "metadata": {
+                "user_api_key_hash": "key-one",
+                "user_api_key_user_id": "shared-user",
+                "deployment_model_name": model_group,
+            },
+            "model_info": {"id": "openai-deployment-b"},
+        },
+        call_type=None,
+    )
+
+    other_key_same_user = await callback.async_filter_deployments(
+        model=model_group,
+        healthy_deployments=healthy_deployments,
+        messages=None,
+        request_kwargs={"metadata": {"user_api_key_hash": "key-two", "user_api_key_user_id": "shared-user"}},
+        parent_otel_span=None,
+    )
+    assert other_key_same_user == healthy_deployments

From 1b42b81f4e048db9403c967365f15165f07a0dd4 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Thu, 3 Sep 2026 10:50:21 -0700
Subject: [PATCH 22/42] fix(router): log the hashed affinity key so JWT callers
 stay distinguishable

---
 .../pre_call_checks/deployment_affinity_check.py            | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py
index ea450864604..6f3ea8eb78a 100644
--- a/litellm/router_utils/pre_call_checks/deployment_affinity_check.py
+++ b/litellm/router_utils/pre_call_checks/deployment_affinity_check.py
@@ -533,9 +533,9 @@ class DeploymentAffinityCheck(CustomLogger):
             return typed_healthy_deployments
 
         verbose_router_logger.debug(
-            "DeploymentAffinityCheck: api-key affinity hit -> deployment=%s user_key=%s",
+            "DeploymentAffinityCheck: caller affinity hit -> deployment=%s user_key=%s",
             model_id,
-            self._shorten_for_logs(user_key),
+            self._shorten_for_logs(self._hash_user_key(user_key)),
         )
         return [deployment]
 
@@ -626,7 +626,7 @@ class DeploymentAffinityCheck(CustomLogger):
                         deployment_model_name,
                         model_id,
                         self.ttl_seconds,
-                        self._shorten_for_logs(user_key),
+                        self._shorten_for_logs(self._hash_user_key(user_key)),
                     )
                 else:
                     verbose_router_logger.debug(

From 4d3c1998affa9249fbfdcd0c8157acaa991f9186 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Thu, 3 Sep 2026 10:50:23 -0700
Subject: [PATCH 23/42] fix(image_gen): report the requested output_format on
 gpt-image responses

---
 .../image_generation/gpt_transformation.py    |  2 +-
 .../test_gpt_transformation.py                | 38 +++++++++++++++++++
 2 files changed, 39 insertions(+), 1 deletion(-)
 create mode 100644 tests/test_litellm/llms/openai/image_generation/test_gpt_transformation.py

diff --git a/litellm/llms/openai/image_generation/gpt_transformation.py b/litellm/llms/openai/image_generation/gpt_transformation.py
index 05494c497ca..64244cfaeda 100644
--- a/litellm/llms/openai/image_generation/gpt_transformation.py
+++ b/litellm/llms/openai/image_generation/gpt_transformation.py
@@ -84,6 +84,6 @@ class GPTImageGenerationConfig(BaseImageGenerationConfig):
         # set optional params
         image_response.size = optional_params.get("size", "1024x1024")  # default is always 1024x1024
         image_response.quality = optional_params.get("quality", "high")  # always hd for dall-e-3
-        image_response.output_format = optional_params.get("response_format", "png")  # always png for dall-e-3
+        image_response.output_format = optional_params.get("output_format", "png")
 
         return image_response
diff --git a/tests/test_litellm/llms/openai/image_generation/test_gpt_transformation.py b/tests/test_litellm/llms/openai/image_generation/test_gpt_transformation.py
new file mode 100644
index 00000000000..de54713a570
--- /dev/null
+++ b/tests/test_litellm/llms/openai/image_generation/test_gpt_transformation.py
@@ -0,0 +1,38 @@
+from unittest.mock import MagicMock
+
+import httpx
+import pytest
+
+from litellm.llms.azure.image_generation.gpt_transformation import AzureGPTImageGenerationConfig
+from litellm.llms.openai.image_generation.gpt_transformation import GPTImageGenerationConfig
+from litellm.types.utils import ImageResponse
+
+
+@pytest.mark.parametrize("config", [GPTImageGenerationConfig(), AzureGPTImageGenerationConfig()])
+def test_transform_image_generation_response_reports_requested_output_format(config):
+    raw_response = httpx.Response(
+        status_code=200,
+        json={
+            "created": 1788457009,
+            "data": [{"b64_json": "/9j/4AAQSkZJRg=="}],
+            "output_format": "jpeg",
+            "background": "opaque",
+            "quality": "low",
+            "size": "1024x1024",
+        },
+        request=httpx.Request("POST", "https://api.openai.com/v1/images/generations"),
+    )
+
+    image_response = config.transform_image_generation_response(
+        model="gpt-image-2",
+        raw_response=raw_response,
+        model_response=ImageResponse(),
+        logging_obj=MagicMock(),
+        request_data={"prompt": "a red apple", "output_format": "jpeg"},
+        optional_params={"output_format": "jpeg"},
+        litellm_params={},
+        encoding=None,
+    )
+
+    assert image_response.output_format == "jpeg"
+    assert image_response.background == "opaque"

From ba9bb752980ea18d8b92830b11c7040bc48b5675 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Thu, 3 Sep 2026 10:53:50 -0700
Subject: [PATCH 24/42] bump: litellm-enterprise 0.1.63 -> 0.1.64,
 litellm-proxy-extras 0.4.92 -> 0.4.93

---
 enterprise/pyproject.toml           | 4 ++--
 litellm-proxy-extras/pyproject.toml | 4 ++--
 pyproject.toml                      | 4 ++--
 uv.lock                             | 6 +++---
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml
index 8360c0a077d..b6f482ccd86 100644
--- a/enterprise/pyproject.toml
+++ b/enterprise/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.63"
+version = "0.1.64"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.63"
+version = "0.1.64"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",
diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml
index 0944f99ad54..97e9eb66bf2 100644
--- a/litellm-proxy-extras/pyproject.toml
+++ b/litellm-proxy-extras/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.92"
+version = "0.4.93"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.92"
+version = "0.4.93"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",
diff --git a/pyproject.toml b/pyproject.toml
index 161994635b8..d3038a60c42 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -67,8 +67,8 @@ proxy = [
     "azure-identity>=1.25.2,<2.0",
     "azure-storage-blob>=12.28.0,<13.0",
     "mcp>=1.28.1,<2.0",
-    "litellm-proxy-extras==0.4.92",
-    "litellm-enterprise==0.1.63",
+    "litellm-proxy-extras==0.4.93",
+    "litellm-enterprise==0.1.64",
     "RestrictedPython>=8.5,<9.0",
     "rich>=13.9.4,<14.0",
     "InquirerPy>=0.3.4,<1.0",
diff --git a/uv.lock b/uv.lock
index de3181edd5a..dfa77c66dfe 100644
--- a/uv.lock
+++ b/uv.lock
@@ -10,7 +10,7 @@ resolution-markers = [
 ]
 
 [options]
-exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values.
+exclude-newer = "2026-08-31T17:52:45.782441Z"
 exclude-newer-span = "P3D"
 
 [manifest]
@@ -4765,12 +4765,12 @@ proxy-dev = [
 
 [[package]]
 name = "litellm-enterprise"
-version = "0.1.63"
+version = "0.1.64"
 source = { editable = "enterprise" }
 
 [[package]]
 name = "litellm-proxy-extras"
-version = "0.4.92"
+version = "0.4.93"
 source = { editable = "litellm-proxy-extras" }
 
 [[package]]

From ec2e35b6796b75231f8ff54686baa1604e7f3697 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Thu, 3 Sep 2026 11:07:08 -0700
Subject: [PATCH 25/42] fix(image_gen): keep the provider's echoed size,
 quality, and output_format on gpt-image responses

---
 litellm/llms/openai/image_generation/gpt_transformation.py  | 6 +++---
 .../llms/openai/image_generation/test_gpt_transformation.py | 3 ++-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/litellm/llms/openai/image_generation/gpt_transformation.py b/litellm/llms/openai/image_generation/gpt_transformation.py
index 64244cfaeda..090b2eba387 100644
--- a/litellm/llms/openai/image_generation/gpt_transformation.py
+++ b/litellm/llms/openai/image_generation/gpt_transformation.py
@@ -82,8 +82,8 @@ class GPTImageGenerationConfig(BaseImageGenerationConfig):
         )
 
         # set optional params
-        image_response.size = optional_params.get("size", "1024x1024")  # default is always 1024x1024
-        image_response.quality = optional_params.get("quality", "high")  # always hd for dall-e-3
-        image_response.output_format = optional_params.get("output_format", "png")
+        image_response.size = image_response.size or optional_params.get("size", "1024x1024")
+        image_response.quality = image_response.quality or optional_params.get("quality", "high")
+        image_response.output_format = image_response.output_format or optional_params.get("output_format", "png")
 
         return image_response
diff --git a/tests/test_litellm/llms/openai/image_generation/test_gpt_transformation.py b/tests/test_litellm/llms/openai/image_generation/test_gpt_transformation.py
index de54713a570..d9b87627b58 100644
--- a/tests/test_litellm/llms/openai/image_generation/test_gpt_transformation.py
+++ b/tests/test_litellm/llms/openai/image_generation/test_gpt_transformation.py
@@ -9,7 +9,7 @@ from litellm.types.utils import ImageResponse
 
 
 @pytest.mark.parametrize("config", [GPTImageGenerationConfig(), AzureGPTImageGenerationConfig()])
-def test_transform_image_generation_response_reports_requested_output_format(config):
+def test_transform_image_generation_response_keeps_provider_echo(config):
     raw_response = httpx.Response(
         status_code=200,
         json={
@@ -35,4 +35,5 @@ def test_transform_image_generation_response_reports_requested_output_format(con
     )
 
     assert image_response.output_format == "jpeg"
+    assert image_response.quality == "low"
     assert image_response.background == "opaque"

From aaef5d219aa3ed1ba262302705df8805570a6a45 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Thu, 3 Sep 2026 11:07:29 -0700
Subject: [PATCH 26/42] fix(proxy): drop anthropic-beta on the Vertex
 passthrough count-tokens route

---
 .../llm_passthrough_endpoints.py              | 12 ++-
 .../test_vertex_passthrough_load_balancing.py | 88 +++++++++++++++++++
 2 files changed, 99 insertions(+), 1 deletion(-)

diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
index b48b8d81494..29f216fd450 100644
--- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
+++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
@@ -1730,6 +1730,16 @@ def get_vertex_ai_allowed_incoming_headers(request: Request) -> dict:
     return headers
 
 
+def _is_vertex_anthropic_count_tokens_route(endpoint: str) -> bool:
+    return endpoint.rsplit("/", 1)[-1].split(":", 1)[0] == "count-tokens"
+
+
+def _upstream_headers_for_vertex_route(endpoint: str, headers: Mapping[str, str]) -> Mapping[str, str]:
+    if not _is_vertex_anthropic_count_tokens_route(endpoint):
+        return headers
+    return MappingProxyType({name: value for name, value in headers.items() if name.lower() != "anthropic-beta"})
+
+
 def get_vertex_pass_through_handler(
     call_type: Literal["discovery", "aiplatform"],  # noqa: UP037  # ruff reports quoted Literal values here
 ) -> BaseVertexAIPassThroughHandler:
@@ -2128,7 +2138,7 @@ async def _base_vertex_proxy_route(
     endpoint_func: Final = create_pass_through_route(
         endpoint=endpoint,
         target=target,
-        custom_headers=headers,
+        custom_headers=_upstream_headers_for_vertex_route(endpoint, headers),
         is_streaming_request=is_streaming_request,
     )  # dynamically construct pass-through endpoint based on incoming path
 
diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py b/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py
index 961479c0393..6735f2a3780 100644
--- a/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py
+++ b/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py
@@ -5,6 +5,7 @@ import pytest
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
     _base_vertex_proxy_route,
+    _upstream_headers_for_vertex_route,
 )
 from litellm.types.router import DeploymentTypedDict
 
@@ -348,6 +349,93 @@ async def test_vertex_passthrough_forwards_anthropic_beta_header():
         assert headers_passed_through is False
 
 
+VERTEX_ANTHROPIC_MODELS_PREFIX = "v1/projects/test-project/locations/global/publishers/anthropic/models/"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("model_segment", "expects_anthropic_beta"),
+    [
+        ("count-tokens:rawPredict", False),
+        ("claude-sonnet-4-6:streamRawPredict", True),
+    ],
+)
+async def test_vertex_passthrough_drops_anthropic_beta_only_on_count_tokens(
+    model_segment: str, expects_anthropic_beta: bool
+):
+    with (
+        patch(  # test-quality-ok: the route reads this proxy global at call time, nothing injects it
+            "litellm.proxy.proxy_server.llm_router", None
+        ),
+        patch(  # test-quality-ok: the route reads this proxy global at call time, nothing injects it
+            "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router"
+        ) as mock_pt_router,
+        patch(  # test-quality-ok: the route offers no injection point for its header preparation
+            "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints._prepare_vertex_auth_headers",
+            new_callable=AsyncMock,
+        ) as mock_prep_headers,
+        patch(  # test-quality-ok: the upstream call is captured here, the route offers no injection point
+            "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route"
+        ) as mock_create_route,
+        patch(  # test-quality-ok: the route calls auth directly rather than through Depends
+            "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.user_api_key_auth",
+            new_callable=AsyncMock,
+        ) as mock_auth,
+        patch(  # test-quality-ok: the route reads the request body for this, a MagicMock request has none
+            "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.is_streaming_request_fn",
+            new_callable=AsyncMock,
+            return_value=False,
+        ),
+    ):
+        mock_pt_router.get_vertex_credentials.return_value = MagicMock()
+        mock_prep_headers.return_value = (
+            {
+                "anthropic-beta": "tool-search-tool-2025-10-19,web-search-2025-03-05",
+                "content-type": "application/json",
+                "Authorization": "Bearer vertex-access-token",
+            },
+            "https://aiplatform.googleapis.com",
+            False,
+            "test-project",
+            "global",
+        )
+        mock_create_route.return_value = AsyncMock()
+        mock_auth.return_value = UserAPIKeyAuth(api_key="sk-litellm-secret-key")
+
+        await _base_vertex_proxy_route(
+            endpoint=f"{VERTEX_ANTHROPIC_MODELS_PREFIX}{model_segment}",
+            request=MagicMock(),
+            fastapi_response=MagicMock(),
+            get_vertex_pass_through_handler=MagicMock(),
+        )
+
+        upstream_headers = mock_create_route.call_args.kwargs["custom_headers"]
+        assert ("anthropic-beta" in upstream_headers) is expects_anthropic_beta
+        assert upstream_headers["Authorization"] == "Bearer vertex-access-token"
+        assert upstream_headers["content-type"] == "application/json"
+
+
+def test_upstream_headers_for_vertex_route_filters_anthropic_beta_by_route():
+    headers = {
+        "Anthropic-Beta": "effort-2025-11-24",
+        "content-type": "application/json",
+        "Authorization": "Bearer vertex-access-token",
+    }
+
+    count_tokens_headers = _upstream_headers_for_vertex_route(
+        f"{VERTEX_ANTHROPIC_MODELS_PREFIX}count-tokens:rawPredict", headers
+    )
+    model_headers = _upstream_headers_for_vertex_route(
+        f"{VERTEX_ANTHROPIC_MODELS_PREFIX}claude-sonnet-4-6:rawPredict", headers
+    )
+
+    assert dict(count_tokens_headers) == {
+        "content-type": "application/json",
+        "Authorization": "Bearer vertex-access-token",
+    }
+    assert dict(model_headers) == headers
+
+
 @pytest.mark.asyncio
 async def test_vertex_passthrough_does_not_forward_litellm_auth_token():
     """

From b3325750ae4d5b462c7162084c5f74261f458c42 Mon Sep 17 00:00:00 2001
From: ryan-crabbe-berri <ryan@berri.ai>
Date: Thu, 3 Sep 2026 11:09:44 -0700
Subject: [PATCH 27/42] fix(ui): aggregate session token usage in the logs
 table

The logs table already rolled up cost per session but the Tokens column only showed the representative call's usage. The per-session aggregate query now also sums prompt, completion and total tokens, and the Tokens cell switches to those sums for multi-call sessions the same way the Cost cell does.

Claude-Session: https://claude.ai/code/session_01CNasFqyjnLN3Rqman25vde
---
 .../spend_management_endpoints.py             | 21 ++++-
 .../test_spend_management_endpoints.py        | 85 +++++++++++++++++++
 .../RequestLogsTableColumns.test.tsx          | 43 +++++++++-
 .../view_logs/RequestLogsTableColumns.tsx     | 17 ++--
 .../src/components/view_logs/columns.tsx      |  3 +
 5 files changed, 160 insertions(+), 9 deletions(-)

diff --git a/litellm/proxy/spend_tracking/spend_management_endpoints.py b/litellm/proxy/spend_tracking/spend_management_endpoints.py
index 295faaa980a..2a50d5170f0 100644
--- a/litellm/proxy/spend_tracking/spend_management_endpoints.py
+++ b/litellm/proxy/spend_tracking/spend_management_endpoints.py
@@ -173,6 +173,9 @@ class _SessionSpendRow(TypedDict):
     session_cache_hit_count: ReadOnly[int]
     session_llm_count: ReadOnly[int]
     session_agent_count: ReadOnly[int]
+    session_total_prompt_tokens: ReadOnly[int]
+    session_total_completion_tokens: ReadOnly[int]
+    session_total_tokens: ReadOnly[int]
     session_models: ReadOnly[Sequence[str]]
 
 
@@ -188,6 +191,9 @@ class _SessionSpendStats(NamedTuple):
     session_cache_hit_count: int
     session_llm_count: int
     session_agent_count: int
+    session_total_prompt_tokens: int
+    session_total_completion_tokens: int
+    session_total_tokens: int
     session_models: Sequence[str]
     session_models_truncated: bool
 
@@ -4287,8 +4293,8 @@ async def _build_ui_spend_logs_response(
     Build the paginated response for the UI spend-logs endpoint.
 
     When ``enrich_session_counts`` is ``True`` (the default for the v1/UI
-    endpoint), each row is enriched with ``session_total_count`` plus spend
-    and call-type aggregates so the frontend knows which sessions are
+    endpoint), each row is enriched with ``session_total_count`` plus spend,
+    token and call-type aggregates so the frontend knows which sessions are
     expandable (multi-call sessions).  One ``GROUP BY (session_id, api_key)``
     query serves every referenced session, keyed per api key so two callers
     reusing a session id never see each other's totals.  Rows without a
@@ -4356,7 +4362,10 @@ async def _build_ui_spend_logs_response(
                            COUNT(*) FILTER (
                                WHERE call_type NOT IN {_MCP_CALL_TYPES_SQL} AND call_type != {_AGENT_CALL_TYPE_SQL}
                            )::int AS session_llm_count,
-                           COUNT(*) FILTER (WHERE call_type = {_AGENT_CALL_TYPE_SQL})::int AS session_agent_count
+                           COUNT(*) FILTER (WHERE call_type = {_AGENT_CALL_TYPE_SQL})::int AS session_agent_count,
+                           COALESCE(SUM(prompt_tokens), 0)::bigint AS session_total_prompt_tokens,
+                           COALESCE(SUM(completion_tokens), 0)::bigint AS session_total_completion_tokens,
+                           COALESCE(SUM(total_tokens), 0)::bigint AS session_total_tokens
                     FROM "LiteLLM_SpendLogs"
                     WHERE session_id = ANY($1::text[])
                       AND api_key = ANY($2::text[])
@@ -4389,6 +4398,9 @@ async def _build_ui_spend_logs_response(
                     session_cache_hit_count=int(row.get("session_cache_hit_count") or 0),
                     session_llm_count=int(row.get("session_llm_count") or 0),
                     session_agent_count=int(row.get("session_agent_count") or 0),
+                    session_total_prompt_tokens=int(row.get("session_total_prompt_tokens") or 0),
+                    session_total_completion_tokens=int(row.get("session_total_completion_tokens") or 0),
+                    session_total_tokens=int(row.get("session_total_tokens") or 0),
                     session_models=models[:_SESSION_MODELS_LIMIT],
                     session_models_truncated=len(models) > _SESSION_MODELS_LIMIT,
                 )
@@ -4418,6 +4430,9 @@ async def _build_ui_spend_logs_response(
                 row_dict["session_cache_hit_count"] = session_stats.session_cache_hit_count
                 row_dict["session_llm_count"] = session_stats.session_llm_count
                 row_dict["session_agent_count"] = session_stats.session_agent_count
+                row_dict["session_total_prompt_tokens"] = session_stats.session_total_prompt_tokens
+                row_dict["session_total_completion_tokens"] = session_stats.session_total_completion_tokens
+                row_dict["session_total_tokens"] = session_stats.session_total_tokens
                 row_dict["session_models"] = session_stats.session_models
                 row_dict["session_models_truncated"] = session_stats.session_models_truncated
             enriched.append(row_dict)
diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py
index 60a32102946..f4cd8814bc1 100644
--- a/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py
+++ b/tests/test_litellm/proxy/spend_tracking/test_spend_management_endpoints.py
@@ -4234,6 +4234,91 @@ async def test_build_ui_spend_logs_response_sums_multi_round_session_spend():
     assert call_args[2] == [api_key]
 
 
+@pytest.mark.asyncio
+async def test_build_ui_spend_logs_response_sums_multi_round_session_tokens():
+    """
+    Regression test for LIT-4929: the logs table showed the summed session cost but
+    only the last call's token usage.  Every row of a multi-round session must carry
+    the session-wide prompt, completion and total token sums from the aggregate
+    query, while rows outside a session carry none of them.
+    """
+    from litellm.proxy.spend_tracking.spend_management_endpoints import (
+        _build_ui_spend_logs_response,
+    )
+
+    session_id = "sess-multi-round-tokens"
+    api_key = "hashed-key-xyz"
+    dict_rows = [
+        {
+            "request_id": "req-1",
+            "session_id": session_id,
+            "call_type": "completion",
+            "api_key": api_key,
+            "total_tokens": 10,
+            "prompt_tokens": 7,
+            "completion_tokens": 3,
+        },
+        {
+            "request_id": "req-2",
+            "session_id": session_id,
+            "call_type": "completion",
+            "api_key": api_key,
+            "total_tokens": 50,
+            "prompt_tokens": 35,
+            "completion_tokens": 15,
+        },
+        {
+            "request_id": "req-3",
+            "session_id": None,
+            "call_type": "completion",
+            "api_key": api_key,
+            "total_tokens": 5,
+            "prompt_tokens": 4,
+            "completion_tokens": 1,
+        },
+    ]
+
+    mock_prisma = MagicMock()
+    mock_prisma.db.query_raw = AsyncMock(
+        return_value=[
+            {
+                "session_id": session_id,
+                "api_key": api_key,
+                "session_total_count": 2,
+                "session_total_spend": 0.06,
+                "mcp_tool_call_count": 0,
+                "mcp_tool_call_spend": 0.0,
+                "session_total_prompt_tokens": 42,
+                "session_total_completion_tokens": 18,
+                "session_total_tokens": 60,
+            }
+        ]
+    )
+
+    result = await _build_ui_spend_logs_response(
+        prisma_client=mock_prisma,
+        data=dict_rows,
+        total_records=3,
+        page=1,
+        page_size=50,
+        total_pages=1,
+        enrich_session_counts=True,
+    )
+
+    rows = result["data"]
+    session_rows = rows[:2]
+    assert [row["session_total_tokens"] for row in session_rows] == [60, 60]
+    assert [row["session_total_prompt_tokens"] for row in session_rows] == [42, 42]
+    assert [row["session_total_completion_tokens"] for row in session_rows] == [18, 18]
+    assert [(row["total_tokens"], row["prompt_tokens"], row["completion_tokens"]) for row in session_rows] == [
+        (10, 7, 3),
+        (50, 35, 15),
+    ]
+
+    token_keys = ("session_total_tokens", "session_total_prompt_tokens", "session_total_completion_tokens")
+    assert all(key not in rows[2] for key in token_keys)
+
+
 @pytest.mark.asyncio
 async def test_build_ui_spend_logs_response_session_cache_hit_count():
     """
diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx
index e3bacc0908a..d2c84173fd1 100644
--- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx
@@ -1,4 +1,4 @@
-import { render, screen } from "@testing-library/react";
+import { render, screen, within } from "@testing-library/react";
 import userEvent from "@testing-library/user-event";
 import { describe, expect, it, vi } from "vitest";
 
@@ -75,6 +75,47 @@ describe("Cost column", () => {
   });
 });
 
+describe("Tokens column", () => {
+  it("shows the summed session token usage, not the representative call's tokens, for a multi-round session", () => {
+    renderRows([
+      logEntry({
+        request_id: "req-session-tokens",
+        total_tokens: 10,
+        prompt_tokens: 7,
+        completion_tokens: 3,
+        session_id: "sess-1",
+        session_total_count: 3,
+        session_total_tokens: 60,
+        session_total_prompt_tokens: 42,
+        session_total_completion_tokens: 18,
+      }),
+    ]);
+
+    const tokensCell = screen.getByText("60").closest("td")!;
+    expect(within(tokensCell).getByText("(42+18)")).toBeInTheDocument();
+    expect(within(tokensCell).getByText("session total")).toBeInTheDocument();
+    expect(screen.queryByText("10")).not.toBeInTheDocument();
+    expect(screen.queryByText("(7+3)")).not.toBeInTheDocument();
+  });
+
+  it("falls back to the call's own tokens with no session label when the backend sent no session token sums", () => {
+    renderRows([
+      logEntry({
+        request_id: "req-no-token-aggregate",
+        total_tokens: 10,
+        prompt_tokens: 7,
+        completion_tokens: 3,
+        session_id: "sess-2",
+        session_total_count: 3,
+      }),
+    ]);
+
+    const tokensCell = screen.getByText("10").closest("td")!;
+    expect(within(tokensCell).getByText("(7+3)")).toBeInTheDocument();
+    expect(within(tokensCell).queryByText("session total")).not.toBeInTheDocument();
+  });
+});
+
 describe("Type column", () => {
   it("shows the conversation badge and composition even when an MCP call represents the conversation", async () => {
     const user = userEvent.setup();
diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx
index 8db0b106851..9d4dc4f7898 100644
--- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx
@@ -263,13 +263,20 @@ export const getRequestLogsTableColumns = ({
     meta: { numeric: true },
     cell: ({ row }) => {
       const log = row.original;
+      const showSessionTotal = (log.session_total_count || 1) > 1 && log.session_total_tokens != null;
+      const total = showSessionTotal ? log.session_total_tokens : log.total_tokens;
+      const prompt = showSessionTotal ? log.session_total_prompt_tokens : log.prompt_tokens;
+      const completion = showSessionTotal ? log.session_total_completion_tokens : log.completion_tokens;
       return (
-        <span className="text-sm">
-          {String(log.total_tokens || "0")}
-          <span className="text-muted-foreground text-xs ml-1">
-            ({String(log.prompt_tokens || "0")}+{String(log.completion_tokens || "0")})
+        <div className="flex flex-col items-end">
+          <span className="text-sm">
+            {String(total || "0")}
+            <span className="text-muted-foreground text-xs ml-1">
+              ({String(prompt || "0")}+{String(completion || "0")})
+            </span>
           </span>
-        </span>
+          {showSessionTotal && <span className="text-[10px] text-muted-foreground">session total</span>}
+        </div>
       );
     },
   },
diff --git a/ui/litellm-dashboard/src/components/view_logs/columns.tsx b/ui/litellm-dashboard/src/components/view_logs/columns.tsx
index 21e09faf454..b2e29c3a0c1 100644
--- a/ui/litellm-dashboard/src/components/view_logs/columns.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/columns.tsx
@@ -42,6 +42,9 @@ export type LogEntry = {
   request_duration_ms?: number;
   session_total_count?: number;
   session_total_spend?: number;
+  session_total_tokens?: number;
+  session_total_prompt_tokens?: number;
+  session_total_completion_tokens?: number;
   session_cache_hit_count?: number;
   mcp_tool_call_count?: number;
   mcp_tool_call_spend?: number;

From 3b13a5fda968620d699d5f8cc15d7485f118bc5e Mon Sep 17 00:00:00 2001
From: ryan <ryan@berri.ai>
Date: Thu, 3 Sep 2026 18:31:43 +0000
Subject: [PATCH 28/42] test(ui): query the tokens cell by role instead of
 walking the DOM

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../RequestLogsTableColumns.test.tsx          | 56 +++++++++----------
 1 file changed, 25 insertions(+), 31 deletions(-)

diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx
index d2c84173fd1..3c9e6543c1c 100644
--- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx
@@ -1,4 +1,4 @@
-import { render, screen, within } from "@testing-library/react";
+import { render, screen } from "@testing-library/react";
 import userEvent from "@testing-library/user-event";
 import { describe, expect, it, vi } from "vitest";
 
@@ -76,43 +76,37 @@ describe("Cost column", () => {
 });
 
 describe("Tokens column", () => {
-  it("shows the summed session token usage, not the representative call's tokens, for a multi-round session", () => {
-    renderRows([
-      logEntry({
-        request_id: "req-session-tokens",
-        total_tokens: 10,
-        prompt_tokens: 7,
-        completion_tokens: 3,
-        session_id: "sess-1",
-        session_total_count: 3,
-        session_total_tokens: 60,
-        session_total_prompt_tokens: 42,
-        session_total_completion_tokens: 18,
-      }),
-    ]);
+  const sessionRow: Partial<LogEntry> = {
+    request_id: "req-session-tokens",
+    total_tokens: 10,
+    prompt_tokens: 7,
+    completion_tokens: 3,
+    session_id: "sess-1",
+    session_total_count: 3,
+  };
 
-    const tokensCell = screen.getByText("60").closest("td")!;
-    expect(within(tokensCell).getByText("(42+18)")).toBeInTheDocument();
-    expect(within(tokensCell).getByText("session total")).toBeInTheDocument();
+  it("shows the summed session token usage, not the representative call's tokens, for a multi-round session", () => {
+    const aggregatedRow: Partial<LogEntry> = {
+      ...sessionRow,
+      session_total_tokens: 60,
+      session_total_prompt_tokens: 42,
+      session_total_completion_tokens: 18,
+    };
+    renderRows([logEntry(aggregatedRow)]);
+
+    const tokensCell = screen.getByRole("cell", { name: /\(42\+18\)/ });
+    expect(tokensCell).toHaveTextContent("60");
+    expect(tokensCell).toHaveTextContent("session total");
     expect(screen.queryByText("10")).not.toBeInTheDocument();
     expect(screen.queryByText("(7+3)")).not.toBeInTheDocument();
   });
 
   it("falls back to the call's own tokens with no session label when the backend sent no session token sums", () => {
-    renderRows([
-      logEntry({
-        request_id: "req-no-token-aggregate",
-        total_tokens: 10,
-        prompt_tokens: 7,
-        completion_tokens: 3,
-        session_id: "sess-2",
-        session_total_count: 3,
-      }),
-    ]);
+    renderRows([logEntry(sessionRow)]);
 
-    const tokensCell = screen.getByText("10").closest("td")!;
-    expect(within(tokensCell).getByText("(7+3)")).toBeInTheDocument();
-    expect(within(tokensCell).queryByText("session total")).not.toBeInTheDocument();
+    const tokensCell = screen.getByRole("cell", { name: /\(7\+3\)/ });
+    expect(tokensCell).toHaveTextContent("10");
+    expect(tokensCell).not.toHaveTextContent("session total");
   });
 });
 

From 4d2ffe2e8e8dd1769a8fd2f1a3ebc857a4a3701c Mon Sep 17 00:00:00 2001
From: ryan <ryan@berri.ai>
Date: Thu, 3 Sep 2026 18:38:22 +0000
Subject: [PATCH 29/42] test(ui): hoist inherited-grant fixture out of the
 inline createMockTeamData arg

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../src/components/team/TeamInfo.test.tsx     | 32 +++++++++----------
 1 file changed, 15 insertions(+), 17 deletions(-)

diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx
index ae78ac06a9c..a9c1077e96b 100644
--- a/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx
+++ b/ui/litellm-dashboard/src/components/team/TeamInfo.test.tsx
@@ -313,23 +313,21 @@ describe("TeamInfoView", () => {
       vi.mocked(networking.getAgentsList).mockResolvedValue({
         agents: [{ agent_id: "agent-support-5678", agent_name: "support_agent" }],
       });
-      vi.mocked(networking.teamInfoCall).mockResolvedValue(
-        createMockTeamData({
-          object_permission: null,
-          access_group_ids: ["ag-1"],
-          access_group_mcp_server_ids: ["mcp-github-1234"],
-          access_group_agent_ids: ["agent-support-5678"],
-          access_group_details: [
-            {
-              access_group_id: "ag-1",
-              access_group_name: "platform-tools",
-              models: [],
-              mcp_server_ids: ["mcp-github-1234"],
-              agent_ids: ["agent-support-5678"],
-            },
-          ],
-        }),
-      );
+      const platformToolsGroup = {
+        access_group_id: "ag-1",
+        access_group_name: "platform-tools",
+        models: [],
+        mcp_server_ids: ["mcp-github-1234"],
+        agent_ids: ["agent-support-5678"],
+      };
+      const inheritedGrants = {
+        object_permission: null,
+        access_group_ids: ["ag-1"],
+        access_group_mcp_server_ids: ["mcp-github-1234"],
+        access_group_agent_ids: ["agent-support-5678"],
+        access_group_details: [platformToolsGroup],
+      };
+      vi.mocked(networking.teamInfoCall).mockResolvedValue(createMockTeamData(inheritedGrants));
 
       renderWithProviders(<TeamInfoView {...defaultProps} />);
 

From f87b9097eaffdde471df332c323a1a59be9c2014 Mon Sep 17 00:00:00 2001
From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com>
Date: Thu, 3 Sep 2026 11:50:51 -0700
Subject: [PATCH 30/42] test(bedrock): drop EOL cohere.command-r-plus-v1:0 from
 local_testing

Bedrock retired cohere.command-r-plus-v1:0 on 2026-08-19 and lists no
Cohere command chat model anymore, so the three local_testing cases that
pinned it fail with a 404 end-of-life error on every pipeline. Drop the
case from test_completion_bedrock_httpx_models and move the
parallel-streaming Bedrock entry to mistral.mistral-7b-instruct-v0:2,
which still takes the invoke route and is ACTIVE in the CI account.
---
 tests/local_testing/test_completion.py | 1 -
 tests/local_testing/test_streaming.py  | 3 +--
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py
index ef8d6c55148..f8f23ea015a 100644
--- a/tests/local_testing/test_completion.py
+++ b/tests/local_testing/test_completion.py
@@ -2879,7 +2879,6 @@ def response_format_tests(response: litellm.ModelResponse):
     "model",
     [
         "bedrock/mistral.mistral-large-2407-v1:0",
-        "bedrock/cohere.command-r-plus-v1:0",
         "us.anthropic.claude-sonnet-4-5-20250929-v1:0",
         "mistral.mistral-7b-instruct-v0:2",
         "meta.llama3-8b-instruct-v1:0",
diff --git a/tests/local_testing/test_streaming.py b/tests/local_testing/test_streaming.py
index 07d693af447..bf39d3155b7 100644
--- a/tests/local_testing/test_streaming.py
+++ b/tests/local_testing/test_streaming.py
@@ -1168,7 +1168,6 @@ async def test_completion_replicate_llama3_streaming(sync_mode):
     "model, region",
     [
         # ["bedrock/ai21.jamba-instruct-v1:0", "us-east-1"],
-        # ["bedrock/cohere.command-r-plus-v1:0", None],
         ["us.anthropic.claude-sonnet-4-5-20250929-v1:0", None],
         # ["mistral.mistral-7b-instruct-v0:2", None],
         # ["meta.llama3-8b-instruct-v1:0", None],
@@ -1271,7 +1270,7 @@ def test_bedrock_claude_3_streaming():
     "model",
     [
         "claude-haiku-4-5-20251001",
-        "cohere.command-r-plus-v1:0",  # bedrock
+        "bedrock/mistral.mistral-7b-instruct-v0:2",
         "gpt-3.5-turbo",
     ],
 )

From c2265b0ef3569b72767359c33516a91aef7db7fb Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Thu, 3 Sep 2026 11:55:59 -0700
Subject: [PATCH 31/42] fix(proxy): return persisted team memberships from
 /user/new so first CLI login gets the default team (#39545)

* fix(proxy): return persisted team memberships from /user/new

new_user attached default teams after building its response from the
pre-membership snapshot, so NewUserResponse.teams was always empty for
users created with default_internal_user_params.teams. The CLI SSO flow
reads that response on a user's first login and minted a teamless JWT,
which skipped the default team's model allowlist.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(proxy): return team ids as a tuple to satisfy LIT001

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../management_endpoints/internal_user_endpoints.py  | 12 ++++++++++++
 .../test_internal_user_endpoints.py                  |  7 +++++++
 2 files changed, 19 insertions(+)

diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py
index 5326074ad3c..93423ca5a1a 100644
--- a/litellm/proxy/management_endpoints/internal_user_endpoints.py
+++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py
@@ -426,6 +426,11 @@ async def add_new_user_to_default_team(
     await asyncio.gather(*tasks, return_exceptions=True)
 
 
+async def _fetch_user_team_ids(user_id: str, prisma_client: "PrismaClient") -> tuple[str, ...]:
+    user_row: Final = await _user_table(prisma_client).find_unique(where={"user_id": user_id})
+    return tuple(user_row.teams) if user_row is not None else ()
+
+
 @router.post(
     "/user/new",
     tags=["Internal User management"],
@@ -580,6 +585,11 @@ async def new_user(
             )
 
         user_id: Final = cast(str | None, response.get("user_id", None))
+        attached_team_ids: Final = (
+            await _fetch_user_team_ids(user_id=user_id, prisma_client=prisma_client)
+            if user_id is not None and (_team_id is not None or teams is not None)
+            else None
+        )
 
         if organization_ids is not None and user_id is not None:
             await _add_user_to_organizations(
@@ -596,6 +606,8 @@ async def new_user(
                 response_dict[key] = value
 
         response_dict["key"] = response.get("token", "")
+        if attached_team_ids is not None:
+            response_dict["teams"] = list(attached_team_ids)
 
         new_user_response: Final = NewUserResponse.model_validate(response_dict)
 
diff --git a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py
index f231eb66a50..022aeff4e20 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py
@@ -1449,6 +1449,11 @@ async def test_new_user_default_teams_flow(mocker):
         return 5  # Low user count, under limit
 
     mock_prisma_client.db.litellm_usertable.count = mock_count
+    persisted_user_row = mocker.MagicMock()
+    persisted_user_row.teams = ["96fed65b-0182-4ff4-8429-2721cd7d42af"]
+    mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(
+        return_value=persisted_user_row
+    )
 
     # Mock duplicate checks to pass
     async def mock_check_duplicate_user_email(*args, **kwargs):
@@ -1477,6 +1482,7 @@ async def test_new_user_default_teams_flow(mocker):
         "token": "sk-test-token-123",
         "expires": None,
         "max_budget": 100,
+        "teams": [],
     }
 
     # Mock _add_user_to_team
@@ -1551,6 +1557,7 @@ async def test_new_user_default_teams_flow(mocker):
         # Verify response structure
         assert response.user_id == "test-user-123"
         assert response.key == "sk-test-token-123"
+        assert response.teams == ["96fed65b-0182-4ff4-8429-2721cd7d42af"]
 
     finally:
         # Restore original default params (always assign, never delattr — the attribute

From e046aee3d52e2308d94399b033893fe77674ee50 Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Thu, 3 Sep 2026 11:57:56 -0700
Subject: [PATCH 32/42] fix(spend_tracking): add missing_session_id: omit to
 leave SpendLogs.session_id null without a client session (#39458)

* fix(spend_tracking): leave SpendLogs.session_id null when no client session id was established

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* chore(lint): ratchet basedpyright budget after session_id fix

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(spend_tracking): ignore trace ids as session ids

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(spend_tracking): gate null SpendLogs.session_id behind missing_session_id: omit

Unset, generate and reject keep the legacy trace id fallback. omit records only
metadata.session_id, the key Langfuse reads, so a trace id copied into
litellm_session_id by get_litellm_params never becomes a session.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(spend_tracking): stamp the omit decision on the request so a config reload cannot fabricate a session

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(spend_tracking): keep omit covering requests the pre-call stamp never reaches

Router-model provider pass-through calls allm_passthrough_route directly and
skips add_litellm_data_to_request, so those requests never run the pre-call
helper and carry no omit stamp. Reading only the stamp made POST
/anthropic/v1/messages write a fabricated uuid into SpendLogs.session_id under
missing_session_id: omit while its Langfuse trace had no session, the exact
divergence the policy exists to remove.

The stamp now only pins omit on, and an unstamped request falls back to the
configured policy, so a config reload still cannot fabricate a session for a
request that was decided pre-call.

* fix(spend_tracking): make the session-omission marker proxy-owned so clients cannot forge it

* fix(spend_tracking): strip the client-sent omission marker from both metadata buckets before they merge

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(spend_tracking): strip the session-omission marker from both metadata buckets

The pre-call policy ran before litellm_metadata is merged into metadata, so a
client that planted the marker in litellm_metadata had it copied back into the
route's own bucket after the strip and still got a null SpendLogs.session_id.

---------

Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
---
 basedpyright-code-budget.json                 |   6 +-
 litellm/constants.py                          |   1 +
 litellm/proxy/_types.py                       |   4 +-
 .../proxy/hooks/proxy_track_cost_callback.py  |   7 +-
 litellm/proxy/litellm_pre_call_utils.py       |  12 +-
 .../pass_through_endpoints.py                 |   6 +-
 .../spend_tracking/spend_tracking_utils.py    |  38 +-
 .../test_pass_through_endpoints.py            |  35 +
 .../test_spend_tracking_utils.py              | 477 ++++++-------
 .../proxy/test_litellm_pre_call_utils.py      | 649 ++++++------------
 ui/litellm-dashboard/src/lib/http/schema.d.ts |   4 +-
 11 files changed, 519 insertions(+), 720 deletions(-)

diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json
index 2967fc2a505..a37cb194757 100644
--- a/basedpyright-code-budget.json
+++ b/basedpyright-code-budget.json
@@ -57,7 +57,7 @@
     "limit": 5601
   },
   "reportMissingTypeArgument": {
-    "limit": 15288
+    "limit": 15287
   },
   "reportMissingTypeStubs": {
     "limit": 40
@@ -105,10 +105,10 @@
     "limit": 109
   },
   "reportUnknownMemberType": {
-    "limit": 38324
+    "limit": 38323
   },
   "reportUnknownParameterType": {
-    "limit": 19625
+    "limit": 19624
   },
   "reportUnknownVariableType": {
     "limit": 29861
diff --git a/litellm/constants.py b/litellm/constants.py
index ef9329b9dfc..be13d9aac5f 100644
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -1450,6 +1450,7 @@ SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: Final = "_session_deployment_affin
 CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags"
 INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
 SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"
+SESSION_ID_OMITTED_METADATA_KEY: Final = "litellm_session_id_omitted"
 LITELLM_TRUNCATED_PAYLOAD_FIELD: Final = "litellm_truncated"
 LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE: Final = (
     "Truncation is a DB storage safeguard. "
diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index 97f5f59d2dc..0aea72be1e2 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -2608,9 +2608,9 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
         None,
         description="When set to True, rejects requests that contain client-side 'metadata.tags' to prevent users from influencing budgets by sending different tags. Tags can only be inherited from the API key metadata.",
     )
-    missing_session_id: Literal["generate", "reject"] | None = Field(
+    missing_session_id: Literal["generate", "reject", "omit"] | None = Field(
         None,
-        description="What to do with LLM API requests that carry no session id (x-litellm-session-id header, metadata.session_id, etc.). 'generate' stamps one id into litellm_session_id, litellm_trace_id and metadata.session_id so SpendLogs and logging callbacks agree; 'reject' returns 400. Unset keeps the legacy behavior where SpendLogs falls back to the trace id while callbacks get no session id.",
+        description="What to do with LLM API requests that carry no session id (x-litellm-session-id header, metadata.session_id, etc.). 'generate' stamps one id into litellm_session_id, litellm_trace_id and metadata.session_id so SpendLogs and logging callbacks agree; 'reject' returns 400; 'omit' leaves SpendLogs.session_id null, matching callbacks such as Langfuse that only record a client-established metadata.session_id. Unset keeps the legacy behavior where SpendLogs falls back to the trace id while callbacks get no session id.",
     )
     enable_public_model_hub: bool = Field(
         default=False,
diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py
index 47aafda2337..7254b05db2e 100644
--- a/litellm/proxy/hooks/proxy_track_cost_callback.py
+++ b/litellm/proxy/hooks/proxy_track_cost_callback.py
@@ -168,11 +168,8 @@ class _ProxyDBLogger(CustomLogger):
                 "custom_llm_provider"
             ) or request_data.get("custom_llm_provider", "")
 
-        # Propagate standard_logging_object and litellm_trace_id from the
-        # Logging instance so that _get_session_id_for_spend_log uses the same
-        # trace_id that Langfuse received (via async_failure_handler).
-        # Without this, the DB session_id would be a random UUID that doesn't
-        # match the Langfuse trace_id, making failed requests unsearchable.
+        # Propagate standard_logging_object and litellm_trace_id from the Logging
+        # instance so the failure row carries the same trace_id Langfuse received.
         _litellm_logging_obj: Final = request_data.get("litellm_logging_obj")
         if _litellm_logging_obj is not None:
             if not request_data.get("standard_logging_object"):
diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py
index 1d440448c2f..f752d7cfa89 100644
--- a/litellm/proxy/litellm_pre_call_utils.py
+++ b/litellm/proxy/litellm_pre_call_utils.py
@@ -25,6 +25,7 @@ from litellm.constants import (
     PRE_CALL_EXECUTED_GUARDRAILS_KEY,
     SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
     SESSION_ID_GENERATED_METADATA_KEY,
+    SESSION_ID_OMITTED_METADATA_KEY,
 )
 from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
 from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
@@ -733,12 +734,18 @@ def apply_missing_session_id_policy(
     general_settings: Mapping[str, object] | None,
     request: Request,
 ) -> None:
+    for metadata_key in ("metadata", "litellm_metadata"):
+        if isinstance(client_metadata := data.get(metadata_key), dict):
+            client_metadata.pop(SESSION_ID_OMITTED_METADATA_KEY, None)
+    metadata: Final = data.get(_metadata_variable_name)
     policy: Final = general_settings.get("missing_session_id") if general_settings else None
     if policy is None or not _is_llm_inference_route(request):
         return
-    metadata: Final = data.get(_metadata_variable_name)
     if not isinstance(metadata, dict):
         return
+    if policy == "omit":
+        metadata[SESSION_ID_OMITTED_METADATA_KEY] = True
+        return
     if data.get("litellm_session_id") or metadata.get("session_id"):
         return
     match policy:
@@ -760,7 +767,8 @@ def apply_missing_session_id_policy(
             )
         case _:
             verbose_proxy_logger.warning(
-                "Ignoring unknown general_settings.missing_session_id=%r; expected 'generate' or 'reject'", policy
+                "Ignoring unknown general_settings.missing_session_id=%r; expected 'generate', 'reject' or 'omit'",
+                policy,
             )
 
 
diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
index 79d5d0a016f..323756bf204 100644
--- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
+++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
@@ -40,6 +40,7 @@ from litellm._logging import verbose_proxy_logger
 from litellm._uuid import uuid
 from litellm.constants import (
     MAXIMUM_TRACEBACK_LINES_TO_LOG,
+    SESSION_ID_OMITTED_METADATA_KEY,
     WEBSOCKET_CLOSE_REASON_MAX_BYTES,
 )
 from litellm.integrations.custom_guardrail import CustomGuardrail
@@ -581,8 +582,9 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
         )
 
         # Set internal keys after merging client-supplied metadata so a request
-        # body that mirrors them cannot clobber the authenticated key or the
-        # real parent span.
+        # body that mirrors them cannot clobber the authenticated key, the real
+        # parent span, or the proxy's own session-id decision.
+        _metadata.pop(SESSION_ID_OMITTED_METADATA_KEY, None)
         _metadata["user_api_key"] = user_api_key_dict.api_key
         _metadata["litellm_parent_otel_span"] = user_api_key_dict.parent_otel_span
         _metadata["user_api_key_budget_reservation"] = user_api_key_dict.budget_reservation
diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py
index 7442d71bd96..a37c3ba4405 100644
--- a/litellm/proxy/spend_tracking/spend_tracking_utils.py
+++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py
@@ -15,6 +15,7 @@ from litellm.constants import (
     LITELLM_TRUNCATED_PAYLOAD_FIELD,
     LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE,
     REDACTED_BY_LITELM_STRING,
+    SESSION_ID_OMITTED_METADATA_KEY,
 )
 from litellm.constants import (
     MAX_STRING_LENGTH_PROMPT_IN_DB as DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB,
@@ -578,7 +579,9 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs
             ),
             session_id=_get_session_id_for_spend_log(
                 kwargs=kwargs,
+                metadata=metadata,
                 standard_logging_payload=standard_logging_payload,
+                omit_when_missing=_omits_session_id_when_missing(metadata),
             ),
             request_duration_ms=_get_request_duration_ms(start_time, end_time),
             status=_get_status_for_spend_log(
@@ -602,26 +605,39 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs
         raise e
 
 
+def _omits_session_id_when_missing(metadata: Mapping[str, object] | None) -> bool:
+    """The pre-call stamp pins `omit` on for the requests that carry it, so a config reload between pre-call and spend
+    logging cannot fabricate a session. `apply_missing_session_id_policy` drops any client-supplied copy of the key
+    from both metadata buckets before stamping, which the merge of `litellm_metadata` into `metadata` makes
+    necessary, so a caller cannot forge it. Requests that never reach the pre-call helper, router-model
+    passthrough among them, carry no stamp, so they fall back to the configured policy and `omit` still covers their
+    spend logs."""
+    if metadata is not None and metadata.get(SESSION_ID_OMITTED_METADATA_KEY):
+        return True
+
+    from litellm.proxy.proxy_server import general_settings
+
+    return general_settings.get("missing_session_id") == "omit"
+
+
 def _get_session_id_for_spend_log(
-    kwargs: dict,
+    kwargs: Mapping[str, object],
+    metadata: Mapping[str, object] | None,
     standard_logging_payload: StandardLoggingPayload | None,
-) -> str:
-    """
-    Get the session id for the spend log.
+    omit_when_missing: bool,
+) -> str | None:
+    """Under `omit` only `metadata.session_id`, the key Langfuse reads, counts as a session; `litellm_session_id` may
+    be a copied trace id."""
+    if omit_when_missing:
+        session_id: Final = metadata.get("session_id") if metadata else None
+        return str(session_id) if session_id else None
 
-    This ensures each spend log is associated with a unique session id.
-
-    """
     from litellm._uuid import uuid
 
     if standard_logging_payload is not None and standard_logging_payload.get("trace_id") is not None:
         return str(standard_logging_payload.get("trace_id"))
-
-    # Users can dynamically set the trace_id for each request by passing `litellm_trace_id` in kwargs
     if kwargs.get("litellm_trace_id") is not None:
         return str(kwargs.get("litellm_trace_id"))
-
-    # Ensure we always have a session id, if none is provided
     return str(uuid.uuid4())
 
 
diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py
index d3f17c73499..91367507247 100644
--- a/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py
+++ b/tests/test_litellm/proxy/pass_through_endpoints/test_pass_through_endpoints.py
@@ -5730,3 +5730,38 @@ async def test_pass_through_request_leaves_cost_router_logger_working():
         verbose_logger.removeHandler(recorder)
 
     assert not raised, f"cost router logger raised on the passthrough logging params: {raised[0].exc_info}"
+
+
+@pytest.mark.parametrize("client_metadata_key", ["litellm_metadata", "metadata"])
+def test_passthrough_client_cannot_forge_session_id_omission(client_metadata_key: str):
+    """The omit marker is proxy-owned: only the pre-call policy may set it. A pass-through body that carries
+    it in its own metadata must not null out SpendLogs.session_id on a request the proxy never omitted."""
+    from litellm.constants import SESSION_ID_OMITTED_METADATA_KEY
+    from litellm.proxy.spend_tracking.spend_tracking_utils import _get_session_id_for_spend_log
+
+    mock_request = MagicMock(spec=Request)
+    mock_request.method = "POST"
+    mock_request.url = "http://0.0.0.0:4000/gemini/v1beta/models/gemini-2.5-flash:generateContent"
+    mock_request.headers = Headers({})
+    mock_request.scope = {}
+
+    kwargs = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint(
+        request=mock_request,
+        user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"),
+        passthrough_logging_payload=MagicMock(),
+        logging_obj=MagicMock(),
+        _parsed_body={client_metadata_key: {SESSION_ID_OMITTED_METADATA_KEY: True}},
+        litellm_call_id="lit-6694-call-id",
+    )
+
+    metadata = kwargs["litellm_params"]["metadata"]
+    assert SESSION_ID_OMITTED_METADATA_KEY not in metadata
+    assert (
+        _get_session_id_for_spend_log(
+            kwargs={},
+            metadata=metadata,
+            standard_logging_payload={"trace_id": "per-call-random-trace-id"},
+            omit_when_missing=bool(metadata.get(SESSION_ID_OMITTED_METADATA_KEY)),
+        )
+        == "per-call-random-trace-id"
+    )
diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
index 9e5917637a8..323930eee60 100644
--- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
+++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
@@ -14,6 +14,7 @@ from litellm.constants import (
     LITELLM_TRUNCATED_PAYLOAD_FIELD,
     LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE,
     REDACTED_BY_LITELM_STRING,
+    SESSION_ID_OMITTED_METADATA_KEY,
 )
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.proxy.spend_tracking.spend_tracking_utils import (
@@ -21,6 +22,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import (
     _get_proxy_server_request_for_spend_logs_payload,
     _get_request_duration_ms,
     _get_response_for_spend_logs_payload,
+    _get_session_id_for_spend_log,
     _get_spend_logs_metadata,
     _get_vector_store_request_for_spend_logs_payload,
     _is_master_key,
@@ -33,6 +35,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import (
     get_logging_payload,
     get_spend_logs_id,
 )
+from litellm.proxy._types import SpendLogsPayload
 from litellm.proxy.utils import hash_token
 from litellm.types.utils import (
     StandardLoggingHiddenParams,
@@ -74,6 +77,110 @@ def test_get_logging_payload_maps_openai_cached_tokens_to_cache_read_input_token
     assert additional_usage_values["prompt_tokens_details"]["cached_tokens"] == 123
 
 
+_TRACE_ONLY_STANDARD_LOGGING: Final = cast(
+    StandardLoggingPayload,
+    {
+        "trace_id": "trace-abc",
+        "session_id": "trace-abc",
+        "metadata": {},
+        "model_map_information": None,
+        "request_tags": [],
+    },
+)
+
+
+def _trace_only_session_id(omit_when_missing: bool) -> str | None:
+    """get_litellm_params copies metadata.trace_id into litellm_session_id, so every field echoes the trace id."""
+    return _get_session_id_for_spend_log(
+        kwargs={"litellm_trace_id": "trace-abc", "litellm_session_id": "trace-abc"},
+        metadata={"trace_id": "trace-abc"},
+        standard_logging_payload=_TRACE_ONLY_STANDARD_LOGGING,
+        omit_when_missing=omit_when_missing,
+    )
+
+
+def test_omit_leaves_session_id_none_when_only_a_trace_id_exists():
+    assert _trace_only_session_id(omit_when_missing=True) is None
+
+
+def test_omit_leaves_session_id_none_without_any_ids():
+    assert (
+        _get_session_id_for_spend_log(kwargs={}, metadata=None, standard_logging_payload=None, omit_when_missing=True)
+        is None
+    )
+
+
+def test_omit_records_metadata_session_id():
+    session_id: Final = _get_session_id_for_spend_log(
+        kwargs={"litellm_session_id": "chain-1"},
+        metadata={"trace_id": "chain-1", "session_id": "chain-1"},
+        standard_logging_payload=_TRACE_ONLY_STANDARD_LOGGING,
+        omit_when_missing=True,
+    )
+    assert session_id == "chain-1"
+
+
+def test_legacy_policy_keeps_trace_id_fallback():
+    assert _trace_only_session_id(omit_when_missing=False) == "trace-abc"
+    generated: Final = _get_session_id_for_spend_log(
+        kwargs={}, metadata=None, standard_logging_payload=None, omit_when_missing=False
+    )
+    assert len(str(generated)) == 36
+
+
+@pytest.mark.parametrize(
+    ("request_metadata", "expected"),
+    [
+        ({"trace_id": "trace-abc"}, "trace-abc"),
+        ({"trace_id": "trace-abc", SESSION_ID_OMITTED_METADATA_KEY: True}, None),
+        ({"trace_id": "trace-abc", "session_id": "chain-1", SESSION_ID_OMITTED_METADATA_KEY: True}, "chain-1"),
+    ],
+)
+def test_get_logging_payload_reads_omit_decision_stamped_on_request(
+    request_metadata: dict[str, object], expected: str | None
+):
+    """The pre-call stamp, not the live general_settings, decides the policy, so a config reload between
+    pre-call and spend logging cannot fabricate a session for a request accepted under `omit`."""
+    with patch(  # test-quality-ok: proves log time ignores proxy config; general_settings is yaml, not an HTTP boundary
+        "litellm.proxy.proxy_server.general_settings", {"missing_session_id": "generate"}
+    ):
+        payload: SpendLogsPayload = get_logging_payload(
+            kwargs={
+                "model": "gpt-4o-mini",
+                "litellm_trace_id": "trace-abc",
+                "litellm_params": {"litellm_session_id": "trace-abc", "metadata": request_metadata},
+                "standard_logging_object": _TRACE_ONLY_STANDARD_LOGGING,
+            },
+            response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]),
+            start_time=datetime.datetime.now(timezone.utc),
+            end_time=datetime.datetime.now(timezone.utc),
+        )
+    assert payload["session_id"] == expected
+
+
+@pytest.mark.parametrize("policy", ["omit", "generate", None])
+def test_get_logging_payload_applies_omit_to_requests_that_carry_no_stamp(policy: str | None):
+    """Router-model passthrough calls `allm_passthrough_route` directly and never reaches the pre-call helper that
+    stamps the omit decision, so an unstamped request falls back to the configured policy. Without that fallback
+    `missing_session_id: omit` would fabricate a uuid session id on every passthrough spend log while its Langfuse
+    trace has none, which is the divergence the policy exists to remove."""
+    with patch(  # test-quality-ok: general_settings is proxy config, loaded from yaml, not an HTTP boundary
+        "litellm.proxy.proxy_server.general_settings", {} if policy is None else {"missing_session_id": policy}
+    ):
+        payload: SpendLogsPayload = get_logging_payload(
+            kwargs={
+                "model": "claude-opus-4",
+                "litellm_trace_id": "trace-abc",
+                "litellm_params": {"litellm_session_id": "trace-abc", "metadata": {"trace_id": "trace-abc"}},
+                "standard_logging_object": _TRACE_ONLY_STANDARD_LOGGING,
+            },
+            response_obj=litellm.ModelResponse(id="chatcmpl-test", choices=[]),
+            start_time=datetime.datetime.now(timezone.utc),
+            end_time=datetime.datetime.now(timezone.utc),
+        )
+    assert payload["session_id"] == (None if policy == "omit" else "trace-abc")
+
+
 def test_get_logging_payload_preserves_anthropic_cache_read_input_tokens():
     additional_usage_values = _get_additional_usage_values_for_usage(
         litellm.Usage(
@@ -277,9 +384,7 @@ def test_sanitize_request_body_for_spend_logs_payload_long_string():
     from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB
 
     # Create a string longer than MAX_STRING_LENGTH_PROMPT_IN_DB (2048)
-    long_string = (
-        "a" * 3000
-    )  # Create a string longer than MAX_STRING_LENGTH_PROMPT_IN_DB
+    long_string = "a" * 3000  # Create a string longer than MAX_STRING_LENGTH_PROMPT_IN_DB
     request_body = {"text": long_string, "normal_text": "short text"}
     sanitized = _sanitize_request_body_for_spend_logs_payload(request_body)
 
@@ -329,9 +434,7 @@ def test_sanitize_request_body_for_spend_logs_payload_nested_list():
 
     # Create a string longer than MAX_STRING_LENGTH_PROMPT_IN_DB
     long_string = "a" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 500)
-    request_body = {
-        "items": [{"text": long_string}, {"text": "short"}, [{"text": long_string}]]
-    }
+    request_body = {"items": [{"text": long_string}, {"text": "short"}, [{"text": long_string}]]}
     sanitized = _sanitize_request_body_for_spend_logs_payload(request_body)
 
     # Calculate expected lengths based on actual MAX_STRING_LENGTH_PROMPT_IN_DB
@@ -415,14 +518,10 @@ def test_sanitize_request_body_for_spend_logs_payload_circular_reference():
 
     # Test that it handles circular reference without infinite recursion
     sanitized = _sanitize_request_body_for_spend_logs_payload(a)
-    assert sanitized == {
-        "b": {"a": {}}
-    }  # Should return empty dict for circular reference
+    assert sanitized == {"b": {"a": {}}}  # Should return empty dict for circular reference
 
 
-@patch(
-    "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs"
-)
+@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs")
 def test_get_vector_store_request_for_spend_logs_payload_store_prompts_true(
     mock_should_store,
 ):
@@ -431,27 +530,16 @@ def test_get_vector_store_request_for_spend_logs_payload_store_prompts_true(
 
     # Sample vector store request metadata
     vector_store_request = [
-        {
-            "vector_store_search_response": {
-                "data": [
-                    {"content": [{"text": "sensitive information", "type": "text"}]}
-                ]
-            }
-        }
+        {"vector_store_search_response": {"data": [{"content": [{"text": "sensitive information", "type": "text"}]}]}}
     ]
 
     # When store_prompts is True, the original data should be returned unchanged
     result = _get_vector_store_request_for_spend_logs_payload(vector_store_request)
     assert result == vector_store_request
-    assert (
-        result[0]["vector_store_search_response"]["data"][0]["content"][0]["text"]
-        == "sensitive information"
-    )
+    assert result[0]["vector_store_search_response"]["data"][0]["content"][0]["text"] == "sensitive information"
 
 
-@patch(
-    "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs"
-)
+@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs")
 def test_get_vector_store_request_for_spend_logs_payload_store_prompts_false(
     mock_should_store,
 ):
@@ -460,32 +548,18 @@ def test_get_vector_store_request_for_spend_logs_payload_store_prompts_false(
 
     # Sample vector store request metadata
     vector_store_request = [
-        {
-            "vector_store_search_response": {
-                "data": [
-                    {"content": [{"text": "sensitive information", "type": "text"}]}
-                ]
-            }
-        }
+        {"vector_store_search_response": {"data": [{"content": [{"text": "sensitive information", "type": "text"}]}]}}
     ]
 
     # When store_prompts is False, text should be redacted
     result = _get_vector_store_request_for_spend_logs_payload(vector_store_request)
     assert result is not None
-    assert (
-        result[0]["vector_store_search_response"]["data"][0]["content"][0]["text"]
-        == REDACTED_BY_LITELM_STRING
-    )
+    assert result[0]["vector_store_search_response"]["data"][0]["content"][0]["text"] == REDACTED_BY_LITELM_STRING
     # Ensure other fields are unchanged
-    assert (
-        result[0]["vector_store_search_response"]["data"][0]["content"][0]["type"]
-        == "text"
-    )
+    assert result[0]["vector_store_search_response"]["data"][0]["content"][0]["type"] == "text"
 
 
-@patch(
-    "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs"
-)
+@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs")
 def test_get_vector_store_request_for_spend_logs_payload_null_input(mock_should_store):
     # When input is None
     mock_should_store.return_value = False
@@ -493,9 +567,7 @@ def test_get_vector_store_request_for_spend_logs_payload_null_input(mock_should_
     assert result is None
 
 
-@patch(
-    "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs"
-)
+@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs")
 def test_get_messages_for_spend_logs_realtime_returns_messages(mock_should_store):
     """
     Test that _get_messages_for_spend_logs_payload returns messages
@@ -522,9 +594,7 @@ def test_get_messages_for_spend_logs_realtime_returns_messages(mock_should_store
     assert parsed[1]["content"] == "What is the weather today?"
 
 
-@patch(
-    "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs"
-)
+@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs")
 def test_get_messages_for_spend_logs_strips_null_bytes(mock_should_store):
     """Regression for PostgreSQL 22P05: NUL bytes must be stripped from messages."""
     mock_should_store.return_value = True
@@ -541,9 +611,7 @@ def test_get_messages_for_spend_logs_strips_null_bytes(mock_should_store):
     assert parsed[0]["content"] == "helloworld"
 
 
-@patch(
-    "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs"
-)
+@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs")
 def test_get_messages_for_spend_logs_realtime_empty_when_disabled(mock_should_store):
     """
     Test that _get_messages_for_spend_logs_payload returns '{}' for realtime calls
@@ -561,9 +629,7 @@ def test_get_messages_for_spend_logs_realtime_empty_when_disabled(mock_should_st
     assert result == "{}"
 
 
-@patch(
-    "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs"
-)
+@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs")
 def test_get_messages_for_spend_logs_non_realtime_returns_empty(mock_should_store):
     """
     Test that _get_messages_for_spend_logs_payload returns '{}' for non-realtime
@@ -581,9 +647,7 @@ def test_get_messages_for_spend_logs_non_realtime_returns_empty(mock_should_stor
     assert result == "{}"
 
 
-@patch(
-    "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs"
-)
+@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs")
 def test_get_response_for_spend_logs_payload_truncates_large_base64(mock_should_store):
     from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB
 
@@ -611,9 +675,7 @@ def test_get_response_for_spend_logs_payload_truncates_large_base64(mock_should_
     assert parsed["data"][0]["other_field"] == "value"
 
 
-@patch(
-    "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs"
-)
+@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs")
 def test_get_response_for_spend_logs_payload_strips_null_bytes(mock_should_store):
     """Regression for PostgreSQL 22P05: NUL bytes must be stripped from response."""
     mock_should_store.return_value = True
@@ -626,18 +688,14 @@ def test_get_response_for_spend_logs_payload_strips_null_bytes(mock_should_store
     assert json.loads(response_json)["content"] == "answerhere"
 
 
-@patch(
-    "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs"
-)
+@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs")
 def test_get_response_for_spend_logs_payload_truncates_large_embedding(
     mock_should_store,
 ):
     from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB
 
     mock_should_store.return_value = True
-    embedding_values = [
-        round(i * 0.0001, 6) for i in range(MAX_STRING_LENGTH_PROMPT_IN_DB + 500)
-    ]
+    embedding_values = [round(i * 0.0001, 6) for i in range(MAX_STRING_LENGTH_PROMPT_IN_DB + 500)]
     large_embedding = json.dumps(embedding_values)
     payload = cast(
         StandardLoggingPayload,
@@ -685,9 +743,7 @@ def test_truncation_includes_db_safeguard_note():
     )
 
 
-@patch(
-    "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs"
-)
+@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs")
 def test_response_truncation_logs_info_message(mock_should_store):
     """
     Test that when response is truncated before DB storage, an info log is emitted
@@ -702,18 +758,14 @@ def test_response_truncation_logs_info_message(mock_should_store):
         {"response": {"data": [{"content": large_text}]}},
     )
 
-    with patch(
-        "litellm.proxy.spend_tracking.spend_tracking_utils.verbose_proxy_logger"
-    ) as mock_logger:
+    with patch("litellm.proxy.spend_tracking.spend_tracking_utils.verbose_proxy_logger") as mock_logger:
         _get_response_for_spend_logs_payload(payload)
         mock_logger.info.assert_called_once()
         log_msg = mock_logger.info.call_args[0][0]
         assert "response was truncated" in log_msg
 
 
-@patch(
-    "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs"
-)
+@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs")
 def test_request_body_truncation_logs_info_message(mock_should_store):
     """
     Test that when request body is truncated before DB storage, an info log is emitted.
@@ -722,18 +774,10 @@ def test_request_body_truncation_logs_info_message(mock_should_store):
 
     mock_should_store.return_value = True
     large_prompt = "C" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 500)
-    litellm_params = {
-        "proxy_server_request": {
-            "body": {"messages": [{"role": "user", "content": large_prompt}]}
-        }
-    }
+    litellm_params = {"proxy_server_request": {"body": {"messages": [{"role": "user", "content": large_prompt}]}}}
 
-    with patch(
-        "litellm.proxy.spend_tracking.spend_tracking_utils.verbose_proxy_logger"
-    ) as mock_logger:
-        _get_proxy_server_request_for_spend_logs_payload(
-            metadata={}, litellm_params=litellm_params, kwargs={}
-        )
+    with patch("litellm.proxy.spend_tracking.spend_tracking_utils.verbose_proxy_logger") as mock_logger:
+        _get_proxy_server_request_for_spend_logs_payload(metadata={}, litellm_params=litellm_params, kwargs={})
         mock_logger.info.assert_called_once()
         log_msg = mock_logger.info.call_args[0][0]
         assert "request body was truncated" in log_msg
@@ -870,14 +914,10 @@ def test_get_logging_payload_api_key_preserved_when_standard_logging_payload_is_
     )
 
     # The api_key should be hashed (not the raw key)
-    assert (
-        payload["api_key"] != test_api_key
-    ), "api_key should be hashed, not the raw key"
+    assert payload["api_key"] != test_api_key, "api_key should be hashed, not the raw key"
 
     # The api_key should be a valid hash (64 character hex string for SHA256)
-    assert (
-        len(payload["api_key"]) == 64
-    ), f"Expected 64 character hash, got {len(payload['api_key'])} characters"
+    assert len(payload["api_key"]) == 64, f"Expected 64 character hash, got {len(payload['api_key'])} characters"
 
     # Verify other fields are set correctly
     assert payload["model"] == "openai/gpt-4.1"
@@ -1019,9 +1059,7 @@ async def test_api_key_preserved_through_failure_hook_to_database():
 
     assert payload_api_key is not None, "🚨 CRITICAL: payload['api_key'] is None!"
 
-    assert (
-        payload_api_key == hashed_key
-    ), f"🚨 CRITICAL: Expected api_key={hashed_key}, got {payload_api_key}"
+    assert payload_api_key == hashed_key, f"🚨 CRITICAL: Expected api_key={hashed_key}, got {payload_api_key}"
 
     # Verify token parameter matches
     assert data["token"] == hashed_key, f"Token parameter should be {hashed_key}"
@@ -1066,9 +1104,7 @@ def test_get_logging_payload_includes_agent_id_from_kwargs():
         end_time=end_time,
     )
 
-    assert (
-        payload["agent_id"] == test_agent_id
-    ), f"Expected agent_id '{test_agent_id}', got '{payload.get('agent_id')}'"
+    assert payload["agent_id"] == test_agent_id, f"Expected agent_id '{test_agent_id}', got '{payload.get('agent_id')}'"
 
 
 @patch("litellm.proxy.proxy_server.master_key", None)
@@ -1093,9 +1129,7 @@ def test_get_logging_payload_includes_overhead_in_spend_logs_metadata():
         startTime=1234567890.0,
         endTime=1234567891.0,
         completionStartTime=None,
-        model_map_information=StandardLoggingModelInformation(
-            model_map_key="gpt-3.5-turbo", model_map_value=None
-        ),
+        model_map_information=StandardLoggingModelInformation(model_map_key="gpt-3.5-turbo", model_map_value=None),
         model="gpt-3.5-turbo",
         model_id="model-123",
         model_group="openai",
@@ -1173,9 +1207,9 @@ def test_get_logging_payload_includes_overhead_in_spend_logs_metadata():
     metadata = json.loads(metadata_json)
 
     # Verify overhead is stored directly in metadata
-    assert (
-        metadata.get("litellm_overhead_time_ms") == test_overhead_ms
-    ), f"Expected overhead '{test_overhead_ms}', got '{metadata.get('litellm_overhead_time_ms')}'"
+    assert metadata.get("litellm_overhead_time_ms") == test_overhead_ms, (
+        f"Expected overhead '{test_overhead_ms}', got '{metadata.get('litellm_overhead_time_ms')}'"
+    )
 
 
 @patch("litellm.proxy.proxy_server.master_key", None)
@@ -1228,9 +1262,7 @@ def test_get_logging_payload_handles_missing_overhead_gracefully():
         startTime=1234567890.0,
         endTime=1234567891.0,
         completionStartTime=None,
-        model_map_information=StandardLoggingModelInformation(
-            model_map_key="gpt-3.5-turbo", model_map_value=None
-        ),
+        model_map_information=StandardLoggingModelInformation(model_map_key="gpt-3.5-turbo", model_map_value=None),
         model="gpt-3.5-turbo",
         model_id="model-123",
         model_group="openai",
@@ -1309,14 +1341,12 @@ def test_get_logging_payload_handles_missing_overhead_gracefully():
     metadata = json.loads(metadata_json)
 
     # When overhead is None, litellm_overhead_time_ms should be None or not present
-    assert (
-        metadata.get("litellm_overhead_time_ms") is None
-    ), "litellm_overhead_time_ms should be None when overhead is not provided"
+    assert metadata.get("litellm_overhead_time_ms") is None, (
+        "litellm_overhead_time_ms should be None when overhead is not provided"
+    )
 
 
-@patch(
-    "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs"
-)
+@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs")
 def test_spend_logs_redacts_request_and_response_when_turn_off_message_logging_enabled(
     mock_should_store,
 ):
@@ -1347,9 +1377,7 @@ def test_spend_logs_redacts_request_and_response_when_turn_off_message_logging_e
     )
 
     parsed_request = json.loads(request_result)
-    assert parsed_request["messages"] == [
-        {"role": "user", "content": "redacted-by-litellm"}
-    ]
+    assert parsed_request["messages"] == [{"role": "user", "content": "redacted-by-litellm"}]
     assert parsed_request["model"] == "gpt-4"
 
     # Test response redaction - use dict response to verify redaction
@@ -1368,9 +1396,7 @@ def test_spend_logs_redacts_request_and_response_when_turn_off_message_logging_e
         {"response": response_dict},
     )
 
-    response_result = _get_response_for_spend_logs_payload(
-        payload=payload, kwargs=kwargs
-    )
+    response_result = _get_response_for_spend_logs_payload(payload=payload, kwargs=kwargs)
 
     # When redaction is enabled and response is a dict (not ModelResponse),
     # perform_redaction redacts content in-place within the choices structure
@@ -1415,30 +1441,22 @@ def test_should_store_prompts_and_responses_in_spend_logs_case_insensitive_strin
             # When env var is True, should return True
             mock_get_secret_bool.return_value = True
             result = _should_store_prompts_and_responses_in_spend_logs()
-            assert (
-                result is True
-            ), f"Expected True (from env var) for '{false_value}', got {result}"
+            assert result is True, f"Expected True (from env var) for '{false_value}', got {result}"
 
             # When env var is False, should return False
             mock_get_secret_bool.return_value = False
             result = _should_store_prompts_and_responses_in_spend_logs()
-            assert (
-                result is False
-            ), f"Expected False (from env var) for '{false_value}', got {result}"
+            assert result is False, f"Expected False (from env var) for '{false_value}', got {result}"
 
     # Test when general_settings doesn't have the key at all
     with patch("litellm.proxy.proxy_server.general_settings", {}):
         mock_get_secret_bool.return_value = True
         result = _should_store_prompts_and_responses_in_spend_logs()
-        assert (
-            result is True
-        ), "Expected True (from env var) when key missing, got False"
+        assert result is True, "Expected True (from env var) when key missing, got False"
 
         mock_get_secret_bool.return_value = False
         result = _should_store_prompts_and_responses_in_spend_logs()
-        assert (
-            result is False
-        ), "Expected False (from env var) when key missing, got True"
+        assert result is False, "Expected False (from env var) when key missing, got True"
 
 
 def test_get_spend_logs_metadata_guardrail_info_fallback_from_metadata():
@@ -1831,9 +1849,7 @@ def test_get_logging_payload_includes_retry_info_in_spend_logs_metadata():
             startTime=1234567890.0,
             endTime=1234567891.0,
             completionStartTime=None,
-            model_map_information=StandardLoggingModelInformation(
-                model_map_key="gpt-3.5-turbo", model_map_value=None
-            ),
+            model_map_information=StandardLoggingModelInformation(model_map_key="gpt-3.5-turbo", model_map_value=None),
             model="gpt-3.5-turbo",
             model_id="model-123",
             model_group="openai",
@@ -1897,12 +1913,10 @@ def test_get_logging_payload_includes_retry_info_in_spend_logs_metadata():
 
     metadata = json.loads(payload["metadata"])
 
-    assert (
-        metadata.get("attempted_retries") == 2
-    ), f"Expected attempted_retries=2, got {metadata.get('attempted_retries')}"
-    assert (
-        metadata.get("max_retries") == 3
-    ), f"Expected max_retries=3, got {metadata.get('max_retries')}"
+    assert metadata.get("attempted_retries") == 2, (
+        f"Expected attempted_retries=2, got {metadata.get('attempted_retries')}"
+    )
+    assert metadata.get("max_retries") == 3, f"Expected max_retries=3, got {metadata.get('max_retries')}"
 
 
 @patch("litellm.proxy.proxy_server.master_key", None)
@@ -1930,9 +1944,7 @@ def test_get_logging_payload_handles_missing_retry_info_gracefully():
             startTime=1234567890.0,
             endTime=1234567891.0,
             completionStartTime=None,
-            model_map_information=StandardLoggingModelInformation(
-                model_map_key="gpt-3.5-turbo", model_map_value=None
-            ),
+            model_map_information=StandardLoggingModelInformation(model_map_key="gpt-3.5-turbo", model_map_value=None),
             model="gpt-3.5-turbo",
             model_id="model-123",
             model_group="openai",
@@ -1996,20 +2008,14 @@ def test_get_logging_payload_handles_missing_retry_info_gracefully():
 
     metadata = json.loads(payload["metadata"])
 
-    assert (
-        metadata.get("attempted_retries") is None
-    ), "attempted_retries should be None when not provided"
-    assert (
-        metadata.get("max_retries") is None
-    ), "max_retries should be None when not provided"
+    assert metadata.get("attempted_retries") is None, "attempted_retries should be None when not provided"
+    assert metadata.get("max_retries") is None, "max_retries should be None when not provided"
 
 
 def test_get_request_duration_ms_normal():
     """Test that request duration is correctly computed in milliseconds."""
     start = datetime.datetime(2025, 1, 1, 0, 0, 0, tzinfo=timezone.utc)
-    end = datetime.datetime(
-        2025, 1, 1, 0, 0, 2, 500000, tzinfo=timezone.utc
-    )  # 2.5s later
+    end = datetime.datetime(2025, 1, 1, 0, 0, 2, 500000, tzinfo=timezone.utc)  # 2.5s later
     result = _get_request_duration_ms(start, end)
     assert result == 2500
 
@@ -2039,9 +2045,7 @@ def test_get_logging_payload_includes_request_duration_ms():
         "litellm_params": {"api_base": "https://api.openai.com"},
         "standard_logging_object": None,
     }
-    response_obj = {
-        "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}
-    }
+    response_obj = {"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}}
 
     with (
         patch("litellm.proxy.proxy_server.master_key", None),
@@ -2107,16 +2111,12 @@ def test_sanitize_request_body_strips_secret_fields():
     }
     sanitized = _sanitize_request_body_for_spend_logs_payload(request_body)
 
-    assert (
-        "secret_fields" not in sanitized
-    ), "secret_fields must be stripped from the sanitized request body"
+    assert "secret_fields" not in sanitized, "secret_fields must be stripped from the sanitized request body"
     assert sanitized["model"] == "gpt-4"
     assert sanitized["messages"] == [{"role": "user", "content": "hi"}]
 
 
-@patch(
-    "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs"
-)
+@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs")
 def test_proxy_server_request_payload_excludes_secret_fields(mock_should_store):
     """
     End-to-end test: when the proxy_server_request body contains
@@ -2140,14 +2140,10 @@ def test_proxy_server_request_payload_excludes_secret_fields(mock_should_store):
         }
     }
 
-    result = _get_proxy_server_request_for_spend_logs_payload(
-        metadata={}, litellm_params=litellm_params, kwargs={}
-    )
+    result = _get_proxy_server_request_for_spend_logs_payload(metadata={}, litellm_params=litellm_params, kwargs={})
     parsed = json.loads(result)
 
-    assert (
-        "secret_fields" not in parsed
-    ), "secret_fields must never appear in the spend-log proxy_server_request column"
+    assert "secret_fields" not in parsed, "secret_fields must never appear in the spend-log proxy_server_request column"
     assert parsed["model"] == "gpt-4"
     assert parsed["messages"] == [{"role": "user", "content": "hello"}]
 
@@ -2176,10 +2172,7 @@ def test_redact_prompt_leaks_strips_input_value_python_repr():
 
 
 def test_redact_prompt_leaks_strips_input_value_json():
-    error_text = (
-        '{"error":{"message":"validation failed",'
-        '"input":[{"role":"user","content":"top-secret-content"}]}}'
-    )
+    error_text = '{"error":{"message":"validation failed","input":[{"role":"user","content":"top-secret-content"}]}}'
     redacted = _redact_prompt_leaks_in_error_string(error_text)
     assert "top-secret-content" not in redacted
     assert REDACTED_BY_LITELM_STRING in redacted
@@ -2203,9 +2196,7 @@ def test_redact_prompt_leaks_empty_string():
     assert _redact_prompt_leaks_in_error_string("") == ""
 
 
-@patch(
-    "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs"
-)
+@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs")
 def test_sanitize_error_information_redacts_when_not_storing_prompts(
     mock_should_store,
 ):
@@ -2233,9 +2224,7 @@ def test_sanitize_error_information_redacts_when_not_storing_prompts(
     assert sanitized["llm_provider"] == "openai"
 
 
-@patch(
-    "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs"
-)
+@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs")
 def test_sanitize_error_information_skips_redaction_when_storing_prompts(
     mock_should_store,
 ):
@@ -2246,9 +2235,7 @@ def test_sanitize_error_information_skips_redaction_when_storing_prompts(
         "error_class": "RateLimitError",
         "llm_provider": "openai",
         "traceback": "",
-        "error_message": (
-            'OpenAIException - {"error":{"input":[{"role":"user","content":"kept"}]}}'
-        ),
+        "error_message": ('OpenAIException - {"error":{"input":[{"role":"user","content":"kept"}]}}'),
     }
 
     sanitized = _sanitize_error_information_for_spend_logs(error_info)
@@ -2259,9 +2246,7 @@ def test_sanitize_error_information_skips_redaction_when_storing_prompts(
     assert REDACTED_BY_LITELM_STRING not in sanitized["error_message"]
 
 
-@patch(
-    "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs"
-)
+@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs")
 def test_sanitize_error_information_caps_size_regardless_of_prompt_flag(
     mock_should_store,
 ):
@@ -2292,9 +2277,7 @@ def test_sanitize_error_information_none_passthrough():
     assert _sanitize_error_information_for_spend_logs(None) is None
 
 
-@patch(
-    "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs"
-)
+@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs")
 def test_sanitize_error_information_reproduces_lit_2992(mock_should_store):
     # Mirrors the reproduced row body from LIT-2992 — a RateLimitError whose
     # message embeds 178 pydantic validation errors, each carrying a full
@@ -2335,10 +2318,7 @@ def test_redact_prompt_leaks_handles_nested_multimodal_content():
     # Multi-modal payload: 'content' is itself a list. The depth-1 regex
     # would stop at the inner '['; the parser-based scanner must walk
     # through balanced nested brackets.
-    error_text = (
-        '{"error":{"messages":[{"role":"user",'
-        '"content":[{"type":"text","text":"top-secret-multimodal"}]}]}}'
-    )
+    error_text = '{"error":{"messages":[{"role":"user","content":[{"type":"text","text":"top-secret-multimodal"}]}]}}'
     redacted = _redact_prompt_leaks_in_error_string(error_text)
     assert "top-secret-multimodal" not in redacted
     assert REDACTED_BY_LITELM_STRING in redacted
@@ -2347,9 +2327,7 @@ def test_redact_prompt_leaks_handles_nested_multimodal_content():
 def test_redact_prompt_leaks_handles_bracket_in_prompt_text():
     # Prompt text contains a literal '[' — the depth-1 regex would close
     # the outer ']' prematurely. The parser must respect string quoting.
-    error_text = (
-        '{"error":{"input":[{"role":"user","content":"secret[123 still secret"}]}}'
-    )
+    error_text = '{"error":{"input":[{"role":"user","content":"secret[123 still secret"}]}}'
     redacted = _redact_prompt_leaks_in_error_string(error_text)
     assert "secret[123" not in redacted
     assert "still secret" not in redacted
@@ -2368,8 +2346,7 @@ def test_redact_prompt_leaks_handles_escaped_quote_in_prompt_text():
 def test_redact_prompt_leaks_handles_nested_input_python_repr():
     # Python dict-repr with nested list inside 'input' — single quotes.
     error_text = (
-        "validation error: {'input': [{'role': 'user', "
-        "'content': [{'type': 'text', 'text': 'leaked-nested-text'}]}]}"
+        "validation error: {'input': [{'role': 'user', 'content': [{'type': 'text', 'text': 'leaked-nested-text'}]}]}"
     )
     redacted = _redact_prompt_leaks_in_error_string(error_text)
     assert "leaked-nested-text" not in redacted
@@ -2385,9 +2362,7 @@ def test_redact_prompt_leaks_handles_unterminated_value():
     assert REDACTED_BY_LITELM_STRING in redacted
 
 
-@patch(
-    "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs"
-)
+@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs")
 def test_sanitize_error_information_redacts_traceback_when_not_storing_prompts(
     mock_should_store,
 ):
@@ -2419,9 +2394,7 @@ def test_sanitize_error_information_redacts_traceback_when_not_storing_prompts(
     assert "ValueError: invalid request" in sanitized["traceback"]
 
 
-@patch(
-    "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs"
-)
+@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs")
 def test_sanitize_error_information_skips_traceback_redaction_when_storing_prompts(
     mock_should_store,
 ):
@@ -2431,9 +2404,7 @@ def test_sanitize_error_information_skips_traceback_redaction_when_storing_promp
         "error_code": "500",
         "error_class": "ValueError",
         "llm_provider": "",
-        "traceback": (
-            'raise ValueError({"input":[{"role":"user","content":"tb-kept"}]})'
-        ),
+        "traceback": ('raise ValueError({"input":[{"role":"user","content":"tb-kept"}]})'),
         "error_message": "invalid request",
     }
 
@@ -2448,20 +2419,14 @@ def test_redact_prompt_leaks_strips_prompt_key_completions_payload():
     # /v1/completions echoes the user input under the top-level 'prompt' key
     # rather than 'messages'. Without 'prompt' coverage the body would survive
     # the redactor when store_prompts_in_spend_logs is False.
-    error_text = (
-        '{"error":{"message":"validation failed",'
-        '"prompt":"super-secret-completion-text"}}'
-    )
+    error_text = '{"error":{"message":"validation failed","prompt":"super-secret-completion-text"}}'
     redacted = _redact_prompt_leaks_in_error_string(error_text)
     assert "super-secret-completion-text" not in redacted
     assert REDACTED_BY_LITELM_STRING in redacted
 
 
 def test_redact_prompt_leaks_strips_prompt_key_python_repr():
-    error_text = (
-        "{'model': 'gpt-3.5-turbo-instruct', "
-        "'prompt': 'leaked-completion-prompt-body'}"
-    )
+    error_text = "{'model': 'gpt-3.5-turbo-instruct', 'prompt': 'leaked-completion-prompt-body'}"
     redacted = _redact_prompt_leaks_in_error_string(error_text)
     assert "leaked-completion-prompt-body" not in redacted
     assert REDACTED_BY_LITELM_STRING in redacted
@@ -2495,11 +2460,7 @@ def test_redact_prompt_leaks_strips_pydantic_input_value_list():
 
 
 def test_redact_prompt_leaks_strips_pydantic_input_value_dict():
-    error_text = (
-        "[type=dict_type, "
-        "input_value={'role': 'user', 'content': 'leaked-dict-content'}, "
-        "input_type=dict]"
-    )
+    error_text = "[type=dict_type, input_value={'role': 'user', 'content': 'leaked-dict-content'}, input_type=dict]"
     redacted = _redact_prompt_leaks_in_error_string(error_text)
     assert "leaked-dict-content" not in redacted
     assert REDACTED_BY_LITELM_STRING in redacted
@@ -2541,9 +2502,7 @@ def test_redact_prompt_leaks_combined_quoted_key_and_pydantic_assignment():
     assert redacted.count(REDACTED_BY_LITELM_STRING) >= 2
 
 
-@patch(
-    "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs"
-)
+@patch("litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs")
 def test_sanitize_error_information_redacts_pydantic_assignment_form(
     mock_should_store,
 ):
@@ -2741,9 +2700,7 @@ def test_get_spend_logs_metadata_non_sk_raw_key_hashed():
 
 def test_get_spend_logs_metadata_already_hashed_unchanged_with_provenance():
     already_hashed = hash_token("sk-some-key")
-    meta = _get_spend_logs_metadata(
-        {"user_api_key": already_hashed, "user_api_key_hash": already_hashed}
-    )
+    meta = _get_spend_logs_metadata({"user_api_key": already_hashed, "user_api_key_hash": already_hashed})
     assert meta["user_api_key"] == already_hashed
     assert hash_token(already_hashed) != meta["user_api_key"]  # no double-hash
 
@@ -2758,9 +2715,7 @@ def test_get_spend_logs_metadata_already_hashed_no_provenance_is_rehashed():
 def test_get_spend_logs_metadata_provenance_bypass_requires_hash_match():
     already_hashed = hash_token("sk-some-key")
     different_hash = hash_token("sk-other-key")
-    meta = _get_spend_logs_metadata(
-        {"user_api_key": already_hashed, "user_api_key_hash": different_hash}
-    )
+    meta = _get_spend_logs_metadata({"user_api_key": already_hashed, "user_api_key_hash": different_hash})
     assert meta["user_api_key"] == hash_token(already_hashed)
 
 
@@ -2797,16 +2752,12 @@ def test_get_logging_payload_uses_recovered_combined_usage_on_failure():
         "model": "anthropic/claude-haiku-4-5",
         "call_type": "acompletion",
         "litellm_params": {"metadata": {"user_api_key": "sk-test"}},
-        "combined_usage_object": Usage(
-            prompt_tokens=30, completion_tokens=1, total_tokens=31
-        ),
+        "combined_usage_object": Usage(prompt_tokens=30, completion_tokens=1, total_tokens=31),
     }
     response_obj = Exception("MidStreamFallbackError: read timeout")
     now = datetime.datetime.now(timezone.utc)
 
-    payload = get_logging_payload(
-        kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now
-    )
+    payload = get_logging_payload(kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now)
 
     assert payload["prompt_tokens"] == 30
     assert payload["completion_tokens"] == 1
@@ -2825,9 +2776,7 @@ def test_get_logging_payload_failure_without_recovered_usage_is_zero():
     response_obj = Exception("BadRequestError")
     now = datetime.datetime.now(timezone.utc)
 
-    payload = get_logging_payload(
-        kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now
-    )
+    payload = get_logging_payload(kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now)
 
     assert payload["total_tokens"] == 0
 
@@ -2853,9 +2802,7 @@ def test_get_logging_payload_sets_litellm_call_id_for_correlation():
     }
     now = datetime.datetime.now(timezone.utc)
 
-    payload = get_logging_payload(
-        kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now
-    )
+    payload = get_logging_payload(kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now)
     metadata = json.loads(payload["metadata"])
 
     assert payload["request_id"] == provider_response_id
@@ -2882,9 +2829,7 @@ def test_get_logging_payload_litellm_call_id_falls_back_to_litellm_params():
     }
     now = datetime.datetime.now(timezone.utc)
 
-    payload = get_logging_payload(
-        kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now
-    )
+    payload = get_logging_payload(kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now)
 
     assert json.loads(payload["metadata"])["litellm_call_id"] == trace_call_id
 
@@ -2901,14 +2846,10 @@ def test_get_logging_payload_litellm_call_id_when_response_has_no_id():
         "litellm_call_id": trace_call_id,
         "litellm_params": {"metadata": {"user_api_key": "sk-test"}},
     }
-    response_obj = {
-        "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}
-    }
+    response_obj = {"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}}
     now = datetime.datetime.now(timezone.utc)
 
-    payload = get_logging_payload(
-        kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now
-    )
+    payload = get_logging_payload(kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now)
 
     assert json.loads(payload["metadata"])["litellm_call_id"] == trace_call_id
     assert payload["request_id"] == trace_call_id
@@ -2932,9 +2873,7 @@ def test_get_logging_payload_cache_hit_keeps_raw_litellm_call_id():
     }
     now = datetime.datetime.now(timezone.utc)
 
-    payload = get_logging_payload(
-        kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now
-    )
+    payload = get_logging_payload(kwargs=kwargs, response_obj=response_obj, start_time=now, end_time=now)
 
     assert json.loads(payload["metadata"])["litellm_call_id"] == trace_call_id
     assert "_cache_hit" in payload["request_id"]
@@ -3074,9 +3013,7 @@ def test_get_logging_payload_hashes_bearer_prefixed_api_key():
     assert not payload["api_key"].startswith("Bearer"), (
         f"api_key column contains plaintext Bearer key: {payload['api_key']}"
     )
-    assert not payload["api_key"].startswith("sk-"), (
-        f"api_key column contains unhashed key: {payload['api_key']}"
-    )
+    assert not payload["api_key"].startswith("sk-"), f"api_key column contains unhashed key: {payload['api_key']}"
 
     metadata_dict = json.loads(payload["metadata"])
     assert not metadata_dict["user_api_key"].startswith("Bearer"), (
@@ -3747,9 +3684,7 @@ def test_get_logging_payload_includes_fallback_info_in_spend_logs_metadata():
             startTime=1234567890.0,
             endTime=1234567891.0,
             completionStartTime=None,
-            model_map_information=StandardLoggingModelInformation(
-                model_map_key="gpt-3.5-turbo", model_map_value=None
-            ),
+            model_map_information=StandardLoggingModelInformation(model_map_key="gpt-3.5-turbo", model_map_value=None),
             model="gpt-3.5-turbo",
             model_id="model-123",
             model_group="openai",
@@ -3813,12 +3748,12 @@ def test_get_logging_payload_includes_fallback_info_in_spend_logs_metadata():
 
     metadata = json.loads(payload["metadata"])
 
-    assert (
-        metadata.get("attempted_fallbacks") == 2
-    ), f"Expected attempted_fallbacks=2, got {metadata.get('attempted_fallbacks')}"
-    assert (
-        metadata.get("original_model_group") == "azure-gpt-fallback"
-    ), f"Expected original_model_group=azure-gpt-fallback, got {metadata.get('original_model_group')}"
+    assert metadata.get("attempted_fallbacks") == 2, (
+        f"Expected attempted_fallbacks=2, got {metadata.get('attempted_fallbacks')}"
+    )
+    assert metadata.get("original_model_group") == "azure-gpt-fallback", (
+        f"Expected original_model_group=azure-gpt-fallback, got {metadata.get('original_model_group')}"
+    )
 
 
 def test_get_logging_payload_handles_missing_fallback_info_gracefully():
@@ -3844,9 +3779,7 @@ def test_get_logging_payload_handles_missing_fallback_info_gracefully():
             startTime=1234567890.0,
             endTime=1234567891.0,
             completionStartTime=None,
-            model_map_information=StandardLoggingModelInformation(
-                model_map_key="gpt-3.5-turbo", model_map_value=None
-            ),
+            model_map_information=StandardLoggingModelInformation(model_map_key="gpt-3.5-turbo", model_map_value=None),
             model="gpt-3.5-turbo",
             model_id="model-123",
             model_group="openai",
@@ -3910,12 +3843,10 @@ def test_get_logging_payload_handles_missing_fallback_info_gracefully():
 
     metadata = json.loads(payload["metadata"])
 
-    assert (
-        metadata.get("attempted_fallbacks") is None
-    ), "attempted_fallbacks should be None when not provided"
-    assert (
-        metadata.get("original_model_group") is None
-    ), "original_model_group should be None when not provided"
+    assert metadata.get("attempted_fallbacks") is None, "attempted_fallbacks should be None when not provided"
+    assert metadata.get("original_model_group") is None, "original_model_group should be None when not provided"
+
+
 @pytest.mark.parametrize("bucket", ["metadata", "litellm_metadata"])
 def test_injected_cache_breakpoints_survive_into_spend_log_metadata(bucket):
     """The injection marker only gates savings if it reaches the spend-log row.
diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py
index 8366e5546a9..72d37650963 100644
--- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py
+++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py
@@ -41,21 +41,18 @@ from litellm.litellm_core_utils.get_provider_specific_headers import (
 from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
     TRUSTED_CALLBACK_VARS_FIELD,
 )
-from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY
+from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY
 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
 from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id
 from litellm.types.utils import CredentialItem
 
 
-
 def test_check_if_token_is_service_account():
     """
     Test that only keys with `service_account_id` in metadata are considered service accounts
     """
     # Test case 1: Service account token
-    service_account_token = UserAPIKeyAuth(
-        api_key="test-key", metadata={"service_account_id": "test-service-account"}
-    )
+    service_account_token = UserAPIKeyAuth(api_key="test-key", metadata={"service_account_id": "test-service-account"})
     assert check_if_token_is_service_account(service_account_token) == True
 
     # Test case 2: Regular user token
@@ -63,9 +60,7 @@ def test_check_if_token_is_service_account():
     assert check_if_token_is_service_account(regular_token) == False
 
     # Test case 3: Token with other metadata
-    other_metadata_token = UserAPIKeyAuth(
-        api_key="test-key", metadata={"user_id": "test-user"}
-    )
+    other_metadata_token = UserAPIKeyAuth(api_key="test-key", metadata={"user_id": "test-user"})
     assert check_if_token_is_service_account(other_metadata_token) == False
 
 
@@ -112,15 +107,11 @@ class TestGetMetadataVariableName:
     def test_returns_litellm_metadata_for_bedrock_invoke(self):
         # GH#30629: bedrock passthrough must use litellm_metadata
         # to prevent key-level tags from leaking into provider body
-        request = self._make_request(
-            "/bedrock/model/us.anthropic.claude-sonnet-4-6/invoke"
-        )
+        request = self._make_request("/bedrock/model/us.anthropic.claude-sonnet-4-6/invoke")
         assert _get_metadata_variable_name(request) == "litellm_metadata"
 
     def test_returns_litellm_metadata_for_bedrock_converse(self):
-        request = self._make_request(
-            "/bedrock/model/us.anthropic.claude-sonnet-4-6/converse"
-        )
+        request = self._make_request("/bedrock/model/us.anthropic.claude-sonnet-4-6/converse")
         assert _get_metadata_variable_name(request) == "litellm_metadata"
 
 
@@ -128,9 +119,7 @@ def test_get_enforced_params_for_service_account_settings():
     """
     Test that service account enforced params are only added to service account keys
     """
-    service_account_token = UserAPIKeyAuth(
-        api_key="test-key", metadata={"service_account_id": "test-service-account"}
-    )
+    service_account_token = UserAPIKeyAuth(api_key="test-key", metadata={"service_account_id": "test-service-account"})
     general_settings_with_service_account_settings = {
         "service_account_settings": {"enforced_params": ["metadata.service"]},
     }
@@ -140,9 +129,7 @@ def test_get_enforced_params_for_service_account_settings():
     )
     assert result == ["metadata.service"]
 
-    regular_token = UserAPIKeyAuth(
-        api_key="test-key", metadata={"enforced_params": ["user"]}
-    )
+    regular_token = UserAPIKeyAuth(api_key="test-key", metadata={"enforced_params": ["user"]})
     result = _get_enforced_params(
         general_settings=general_settings_with_service_account_settings,
         user_api_key_dict=regular_token,
@@ -155,9 +142,7 @@ def test_get_enforced_params_for_service_account_settings():
     [
         (
             {"enforced_params": ["param1", "param2"]},
-            UserAPIKeyAuth(
-                api_key="test_api_key", user_id="test_user_id", org_id="test_org_id"
-            ),
+            UserAPIKeyAuth(api_key="test_api_key", user_id="test_user_id", org_id="test_org_id"),
             ["param1", "param2"],
         ),
         (
@@ -183,9 +168,7 @@ def test_get_enforced_params_for_service_account_settings():
         ),
     ],
 )
-def test_get_enforced_params(
-    general_settings, user_api_key_dict, expected_enforced_params
-):
+def test_get_enforced_params(general_settings, user_api_key_dict, expected_enforced_params):
     from litellm.proxy.litellm_pre_call_utils import _get_enforced_params
 
     enforced_params = _get_enforced_params(general_settings, user_api_key_dict)
@@ -441,9 +424,7 @@ async def test_add_litellm_data_to_request_strips_admin_injection_slots():
     populated = updated["metadata"]
     assert populated["user_api_key_metadata"] == real_admin_metadata
     assert populated["user_api_key_team_metadata"] == real_admin_metadata
-    assert "_pipeline_managed_guardrails" not in populated or populated[
-        "_pipeline_managed_guardrails"
-    ] != ["evaded"]
+    assert "_pipeline_managed_guardrails" not in populated or populated["_pipeline_managed_guardrails"] != ["evaded"]
 
     other = updated.get("litellm_metadata") or {}
     assert other.get("user_api_key_metadata") in (None, {}, real_admin_metadata)
@@ -697,9 +678,7 @@ async def test_add_litellm_data_to_request_proxy_server_request_body_is_post_str
     snapshot_body = updated["proxy_server_request"]["body"]
     assert snapshot_body is not None
     snapshot_metadata = snapshot_body.get("metadata") or {}
-    assert "user_api_key_user_id" not in snapshot_metadata or (
-        snapshot_metadata["user_api_key_user_id"] != "victim"
-    )
+    assert "user_api_key_user_id" not in snapshot_metadata or (snapshot_metadata["user_api_key_user_id"] != "victim")
 
 
 @pytest.mark.asyncio
@@ -754,9 +733,7 @@ async def test_add_litellm_data_to_request_body_snapshot_excludes_secret_fields(
     )
 
     # secret_fields must exist on the live data dict
-    assert (
-        "secret_fields" in updated
-    ), "secret_fields must still be present on the live data dict"
+    assert "secret_fields" in updated, "secret_fields must still be present on the live data dict"
     assert "raw_headers" in updated["secret_fields"]
 
     # But the body snapshot must NOT contain secret_fields
@@ -815,8 +792,7 @@ async def test_add_litellm_data_to_request_body_snapshot_excludes_proxy_server_r
 
     snapshot_body = updated["proxy_server_request"]["body"]
     assert "proxy_server_request" not in snapshot_body, (
-        "proxy_server_request must be excluded from its own body snapshot "
-        "to prevent the body from self-referencing"
+        "proxy_server_request must be excluded from its own body snapshot to prevent the body from self-referencing"
     )
 
 
@@ -1344,23 +1320,18 @@ async def test_add_litellm_data_to_request_strips_client_redaction_bypass_contro
     assert "turn_off_message_logging" not in (updated.get("litellm_params") or {}).get("metadata", {})
     assert "turn_off_message_logging" not in updated["metadata"]
     assert "turn_off_message_logging" not in (updated.get("litellm_metadata") or {})
+    assert "litellm-disable-message-redaction" not in {header.lower() for header in updated["metadata"]["headers"]}
     assert "litellm-disable-message-redaction" not in {
-        header.lower() for header in updated["metadata"]["headers"]
-    }
-    assert "litellm-disable-message-redaction" not in {
-        header.lower()
-        for header in updated["metadata"]["requester_metadata"].get("headers", {})
+        header.lower() for header in updated["metadata"]["requester_metadata"].get("headers", {})
     }
     assert "litellm-disable-message-redaction" not in {
         header.lower() for header in updated["proxy_server_request"]["headers"]
     }
     assert "litellm-disable-message-redaction" not in {
-        header.lower()
-        for header in updated["proxy_server_request"]["body"]["metadata"]["headers"]
+        header.lower() for header in updated["proxy_server_request"]["body"]["metadata"]["headers"]
     }
     assert "litellm-disable-message-redaction" not in {
-        header.lower()
-        for header in (updated.get("litellm_metadata") or {}).get("headers", {})
+        header.lower() for header in (updated.get("litellm_metadata") or {}).get("headers", {})
     }
 
 
@@ -1430,12 +1401,7 @@ async def test_add_litellm_data_to_request_admin_callback_vars_turn_off_message_
         dynamic_params = initialize_standard_callback_dynamic_params(updated)
         assert dynamic_params.get("turn_off_message_logging") == "False"
 
-        assert (
-            should_redact_message_logging(
-                {"standard_callback_dynamic_params": dynamic_params}
-            )
-            is False
-        )
+        assert should_redact_message_logging({"standard_callback_dynamic_params": dynamic_params}) is False
     finally:
         litellm.turn_off_message_logging = original_turn_off_message_logging
 
@@ -1506,12 +1472,7 @@ async def test_add_litellm_data_to_request_admin_callback_vars_turn_off_message_
         dynamic_params = initialize_standard_callback_dynamic_params(updated)
         assert dynamic_params.get("turn_off_message_logging") == "True"
 
-        assert (
-            should_redact_message_logging(
-                {"standard_callback_dynamic_params": dynamic_params}
-            )
-            is True
-        )
+        assert should_redact_message_logging({"standard_callback_dynamic_params": dynamic_params}) is True
     finally:
         litellm.turn_off_message_logging = original_turn_off_message_logging
 
@@ -1552,9 +1513,7 @@ async def test_add_litellm_data_to_request_allows_redaction_opt_out_with_admin_o
                     "headers": {"litellm-disable-message-redaction": "true"},
                     "turn_off_message_logging": False,
                 },
-                "litellm_metadata": json.dumps(
-                    {"headers": {"LiteLLM-Disable-Message-Redaction": "true"}}
-                ),
+                "litellm_metadata": json.dumps({"headers": {"LiteLLM-Disable-Message-Redaction": "true"}}),
             },
             request=request_mock,
             user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key", **auth_kwargs),
@@ -1567,19 +1526,15 @@ async def test_add_litellm_data_to_request_allows_redaction_opt_out_with_admin_o
 
     assert updated["turn_off_message_logging"] is False
     assert updated["metadata"]["turn_off_message_logging"] is False
+    assert "litellm-disable-message-redaction" in {header.lower() for header in updated["metadata"]["headers"]}
     assert "litellm-disable-message-redaction" in {
-        header.lower() for header in updated["metadata"]["headers"]
-    }
-    assert "litellm-disable-message-redaction" in {
-        header.lower()
-        for header in updated["metadata"]["requester_metadata"].get("headers", {})
+        header.lower() for header in updated["metadata"]["requester_metadata"].get("headers", {})
     }
     assert "litellm-disable-message-redaction" in {
         header.lower() for header in updated["proxy_server_request"]["headers"]
     }
     assert "litellm-disable-message-redaction" in {
-        header.lower()
-        for header in updated["proxy_server_request"]["body"]["metadata"]["headers"]
+        header.lower() for header in updated["proxy_server_request"]["body"]["metadata"]["headers"]
     }
     assert "litellm_metadata" not in updated
 
@@ -1870,9 +1825,7 @@ async def test_add_litellm_data_to_request_audio_transcription_multipart():
     request_mock.client.host = "127.0.0.1"
 
     # Simulate multipart data (metadata as string)
-    metadata_dict = {
-        "tags": ["jobID:214590dsff09fds", "taskName:run_page_classification"]
-    }
+    metadata_dict = {"tags": ["jobID:214590dsff09fds", "taskName:run_page_classification"]}
     stringified_metadata = json.dumps(metadata_dict)
 
     data = {
@@ -2200,23 +2153,15 @@ def test_key_dynamic_logging_settings():
     # Test with langfuse logging
     key_with_langfuse = UserAPIKeyAuth(
         api_key="test-key",
-        metadata={
-            "logging": [{"callback_name": "langfuse", "callback_type": "success"}]
-        },
+        metadata={"logging": [{"callback_name": "langfuse", "callback_type": "success"}]},
         team_metadata={},
     )
-    result = KeyAndTeamLoggingSettings.get_key_dynamic_logging_settings(
-        key_with_langfuse
-    )
+    result = KeyAndTeamLoggingSettings.get_key_dynamic_logging_settings(key_with_langfuse)
     assert result == [{"callback_name": "langfuse", "callback_type": "success"}]
 
     # Test with no logging metadata
-    key_without_logging = UserAPIKeyAuth(
-        api_key="test-key", metadata={}, team_metadata={}
-    )
-    result = KeyAndTeamLoggingSettings.get_key_dynamic_logging_settings(
-        key_without_logging
-    )
+    key_without_logging = UserAPIKeyAuth(api_key="test-key", metadata={}, team_metadata={})
+    result = KeyAndTeamLoggingSettings.get_key_dynamic_logging_settings(key_without_logging)
     assert result is None
 
 
@@ -2228,35 +2173,23 @@ def test_team_dynamic_logging_settings():
     key_with_team_arize = UserAPIKeyAuth(
         api_key="test-key",
         metadata={},
-        team_metadata={
-            "logging": [{"callback_name": "arize", "callback_type": "failure"}]
-        },
-    )
-    result = KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings(
-        key_with_team_arize
+        team_metadata={"logging": [{"callback_name": "arize", "callback_type": "failure"}]},
     )
+    result = KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings(key_with_team_arize)
     assert result == [{"callback_name": "arize", "callback_type": "failure"}]
 
     # Test with langfuse team logging
     key_with_team_langfuse = UserAPIKeyAuth(
         api_key="test-key",
         metadata={},
-        team_metadata={
-            "logging": [{"callback_name": "langfuse", "callback_type": "success"}]
-        },
-    )
-    result = KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings(
-        key_with_team_langfuse
+        team_metadata={"logging": [{"callback_name": "langfuse", "callback_type": "success"}]},
     )
+    result = KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings(key_with_team_langfuse)
     assert result == [{"callback_name": "langfuse", "callback_type": "success"}]
 
     # Test with no team logging metadata
-    key_without_team_logging = UserAPIKeyAuth(
-        api_key="test-key", metadata={}, team_metadata={}
-    )
-    result = KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings(
-        key_without_team_logging
-    )
+    key_without_team_logging = UserAPIKeyAuth(api_key="test-key", metadata={}, team_metadata={})
+    result = KeyAndTeamLoggingSettings.get_team_dynamic_logging_settings(key_without_team_logging)
     assert result is None
 
 
@@ -2337,9 +2270,7 @@ def test_get_dynamic_logging_metadata_with_arize_team_logging():
     mock_proxy_config = MagicMock()
 
     # Call the function
-    result = _get_dynamic_logging_metadata(
-        user_api_key_dict=user_api_key_dict, proxy_config=mock_proxy_config
-    )
+    result = _get_dynamic_logging_metadata(user_api_key_dict=user_api_key_dict, proxy_config=mock_proxy_config)
 
     # Verify the result
     assert result is not None
@@ -2355,9 +2286,7 @@ def test_add_team_callback_rejects_env_reference():
         AddTeamCallback(
             callback_name="langfuse",
             callback_type="success",
-            callback_vars={
-                "langfuse_secret_key": "os.environ/LANGFUSE_SECRET_KEY_TEMP"
-            },
+            callback_vars={"langfuse_secret_key": "os.environ/LANGFUSE_SECRET_KEY_TEMP"},
         )
 
     assert "os.environ/" in str(exc_info.value)
@@ -2388,9 +2317,7 @@ def test_get_dynamic_logging_metadata_ignores_env_reference_from_key_metadata(
         team_metadata={},
     )
 
-    result = _get_dynamic_logging_metadata(
-        user_api_key_dict=user_api_key_dict, proxy_config=MagicMock()
-    )
+    result = _get_dynamic_logging_metadata(user_api_key_dict=user_api_key_dict, proxy_config=MagicMock())
 
     assert result is None
 
@@ -2401,16 +2328,12 @@ def test_get_num_retries_from_request():
     """
     # Test case 1: Header is present with valid integer string
     headers_with_retries = {"x-litellm-num-retries": "3"}
-    result = LiteLLMProxyRequestSetup._get_num_retries_from_request(
-        headers_with_retries
-    )
+    result = LiteLLMProxyRequestSetup._get_num_retries_from_request(headers_with_retries)
     assert result == 3
 
     # Test case 2: Header is not present
     headers_without_retries = {"Content-Type": "application/json"}
-    result = LiteLLMProxyRequestSetup._get_num_retries_from_request(
-        headers_without_retries
-    )
+    result = LiteLLMProxyRequestSetup._get_num_retries_from_request(headers_without_retries)
     assert result is None
 
     # Test case 3: Empty headers dictionary
@@ -2425,9 +2348,7 @@ def test_get_num_retries_from_request():
 
     # Test case 5: Header present with large number
     headers_with_large_number = {"x-litellm-num-retries": "100"}
-    result = LiteLLMProxyRequestSetup._get_num_retries_from_request(
-        headers_with_large_number
-    )
+    result = LiteLLMProxyRequestSetup._get_num_retries_from_request(headers_with_large_number)
     assert result == 100
 
     # Test case 6: Multiple headers with num retries header
@@ -2441,19 +2362,17 @@ def test_get_num_retries_from_request():
 
     # Test case 7: Header present with invalid value (should raise ValueError when int() is called)
     headers_with_invalid = {"x-litellm-num-retries": "invalid"}
-    with pytest.raises(ValueError, match='invalid literal for int\\(\\) with base'):
+    with pytest.raises(ValueError, match="invalid literal for int\\(\\) with base"):
         LiteLLMProxyRequestSetup._get_num_retries_from_request(headers_with_invalid)
 
     # Test case 8: Header present with float string (should raise ValueError when int() is called)
     headers_with_float = {"x-litellm-num-retries": "3.5"}
-    with pytest.raises(ValueError, match='invalid literal for int\\(\\) with base'):
+    with pytest.raises(ValueError, match="invalid literal for int\\(\\) with base"):
         LiteLLMProxyRequestSetup._get_num_retries_from_request(headers_with_float)
 
     # Test case 9: Header present with negative number
     headers_with_negative = {"x-litellm-num-retries": "-1"}
-    result = LiteLLMProxyRequestSetup._get_num_retries_from_request(
-        headers_with_negative
-    )
+    result = LiteLLMProxyRequestSetup._get_num_retries_from_request(headers_with_negative)
     assert result == -1
 
 
@@ -2463,15 +2382,11 @@ def test_get_keepalive_seconds_from_request():
     """
     # Header present with valid float string
     headers_with_keepalive = {"x-litellm-keepalive-seconds": "15"}
-    result = LiteLLMProxyRequestSetup._get_keepalive_seconds_from_request(
-        headers_with_keepalive
-    )
+    result = LiteLLMProxyRequestSetup._get_keepalive_seconds_from_request(headers_with_keepalive)
     assert result == 15.0
 
     # Header not present
-    result = LiteLLMProxyRequestSetup._get_keepalive_seconds_from_request(
-        {"Content-Type": "application/json"}
-    )
+    result = LiteLLMProxyRequestSetup._get_keepalive_seconds_from_request({"Content-Type": "application/json"})
     assert result is None
 
     # Empty headers dictionary
@@ -2479,17 +2394,13 @@ def test_get_keepalive_seconds_from_request():
     assert result is None
 
     # Header present with a fractional value
-    result = LiteLLMProxyRequestSetup._get_keepalive_seconds_from_request(
-        {"x-litellm-keepalive-seconds": "1.5"}
-    )
+    result = LiteLLMProxyRequestSetup._get_keepalive_seconds_from_request({"x-litellm-keepalive-seconds": "1.5"})
     assert result == 1.5
 
     # Header present with invalid value raises ValueError, matching the other
     # x-litellm-* numeric header helpers (_get_timeout_from_request, etc.)
     with pytest.raises(ValueError, match="could not convert string to float: 'not-a-number"):
-        LiteLLMProxyRequestSetup._get_keepalive_seconds_from_request(
-            {"x-litellm-keepalive-seconds": "not-a-number"}
-        )
+        LiteLLMProxyRequestSetup._get_keepalive_seconds_from_request({"x-litellm-keepalive-seconds": "not-a-number"})
 
 
 def test_add_litellm_data_for_backend_llm_call_merges_keepalive_seconds_header():
@@ -2728,9 +2639,7 @@ def test_management_endpoint_metadata_drops_callback_credentials():
         ),
     ],
 )
-def test_add_headers_to_llm_call_by_model_group(
-    data, model_group_settings, expected_headers_added
-):
+def test_add_headers_to_llm_call_by_model_group(data, model_group_settings, expected_headers_added):
     """
     Test LiteLLMProxyRequestSetup.add_headers_to_llm_call_by_model_group method
 
@@ -2751,9 +2660,7 @@ def test_add_headers_to_llm_call_by_model_group(
         "X-Custom-Header": "custom-value",
     }
 
-    user_api_key_dict = UserAPIKeyAuth(
-        api_key="test-key", user_id="test-user", org_id="test-org"
-    )
+    user_api_key_dict = UserAPIKeyAuth(api_key="test-key", user_id="test-user", org_id="test-org")
 
     # Mock the model_group_settings
     original_model_group_settings = getattr(litellm, "model_group_settings", None)
@@ -2771,7 +2678,6 @@ def test_add_headers_to_llm_call_by_model_group(
             "add_headers_to_llm_call",
             return_value=expected_returned_headers if expected_headers_added else {},
         ) as mock_add_headers:
-
             # Make a copy of original data to verify it's not mutated unexpectedly
             original_data = copy.deepcopy(data)
 
@@ -2828,7 +2734,6 @@ def test_add_headers_to_llm_call_by_model_group_empty_headers_returned():
             "add_headers_to_llm_call",
             return_value={},  # Return empty dict
         ) as mock_add_headers:
-
             result = LiteLLMProxyRequestSetup.add_headers_to_llm_call_by_model_group(
                 data=data, headers=headers, user_api_key_dict=user_api_key_dict
             )
@@ -2876,7 +2781,6 @@ def test_add_headers_to_llm_call_by_model_group_existing_headers_in_data():
             "add_headers_to_llm_call",
             return_value=new_headers,
         ) as mock_add_headers:
-
             result = LiteLLMProxyRequestSetup.add_headers_to_llm_call_by_model_group(
                 data=data, headers=headers, user_api_key_dict=user_api_key_dict
             )
@@ -2990,13 +2894,9 @@ async def test_add_litellm_metadata_from_request_headers():
         general_settings = {}
 
         # Create mock select_data_generator with correct signature
-        def mock_select_data_generator(
-            response=None, user_api_key_dict=None, request_data=None
-        ):
+        def mock_select_data_generator(response=None, user_api_key_dict=None, request_data=None):
             async def mock_generator():
-                yield "data: " + json.dumps(
-                    {"choices": [{"delta": {"content": "Hello"}}]}
-                ) + "\n\n"
+                yield "data: " + json.dumps({"choices": [{"delta": {"content": "Hello"}}]}) + "\n\n"
                 yield "data: [DONE]\n\n"
 
             return mock_generator()
@@ -3023,21 +2923,19 @@ async def test_add_litellm_metadata_from_request_headers():
         await asyncio.sleep(3)
 
         # Check if standard_logging_object was set
-        assert (
-            test_logger.standard_logging_object is not None
-        ), "standard_logging_object should be populated after LLM request"
+        assert test_logger.standard_logging_object is not None, (
+            "standard_logging_object should be populated after LLM request"
+        )
 
         # Verify the logging object contains expected metadata
         standard_logging_obj = test_logger.standard_logging_object
 
-        print(
-            f"Standard logging object captured: {json.dumps(standard_logging_obj, indent=4, default=str)}"
-        )
+        print(f"Standard logging object captured: {json.dumps(standard_logging_obj, indent=4, default=str)}")
 
         SPEND_LOGS_METADATA = standard_logging_obj["metadata"]["spend_logs_metadata"]
-        assert SPEND_LOGS_METADATA == dict(
-            json.loads(headers["x-litellm-spend-logs-metadata"])
-        ), "spend_logs_metadata should be the same as the headers"
+        assert SPEND_LOGS_METADATA == dict(json.loads(headers["x-litellm-spend-logs-metadata"])), (
+            "spend_logs_metadata should be the same as the headers"
+        )
     finally:
         litellm.callbacks = original_callbacks
 
@@ -3188,11 +3086,7 @@ def test_add_litellm_metadata_from_request_headers_generic_session_id_header():
 
 
 def test_add_litellm_metadata_from_anthropic_user_id_sets_session_id():
-    data = {
-        "metadata": {
-            "user_id": "user_abc123_account__session_e96634a3-fa28-4083-b354-55542e2dca01"
-        }
-    }
+    data = {"metadata": {"user_id": "user_abc123_account__session_e96634a3-fa28-4083-b354-55542e2dca01"}}
     LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers(
         headers={}, data=data, _metadata_variable_name="metadata"
     )
@@ -3308,9 +3202,7 @@ def test_get_chain_id_from_headers_generic_vendor_session_id():
     from litellm.proxy.litellm_pre_call_utils import get_chain_id_from_headers
 
     assert (
-        get_chain_id_from_headers(
-            {"x-claude-code-session-id": "e96634a3-fa28-4083-b354-55542e2dca01"}
-        )
+        get_chain_id_from_headers({"x-claude-code-session-id": "e96634a3-fa28-4083-b354-55542e2dca01"})
         == "e96634a3-fa28-4083-b354-55542e2dca01"
     )
     # Short / non-alphanumeric values should be ignored
@@ -3600,19 +3492,13 @@ def test_get_internal_user_header_from_mapping_returns_expected_header():
         {"header_name": "X-OpenWebUI-User-Email", "litellm_user_role": "customer"},
     ]
 
-    header_name = LiteLLMProxyRequestSetup.get_internal_user_header_from_mapping(
-        mappings
-    )
+    header_name = LiteLLMProxyRequestSetup.get_internal_user_header_from_mapping(mappings)
     assert header_name == "X-OpenWebUI-User-Id"
 
 
 def test_get_internal_user_header_from_mapping_none_when_absent():
-    mappings = [
-        {"header_name": "X-OpenWebUI-User-Email", "litellm_user_role": "customer"}
-    ]
-    header_name = LiteLLMProxyRequestSetup.get_internal_user_header_from_mapping(
-        mappings
-    )
+    mappings = [{"header_name": "X-OpenWebUI-User-Email", "litellm_user_role": "customer"}]
+    header_name = LiteLLMProxyRequestSetup.get_internal_user_header_from_mapping(mappings)
     assert header_name is None
 
     single = {"header_name": "X-Only-Customer", "litellm_user_role": "customer"}
@@ -3633,9 +3519,7 @@ def test_add_internal_user_from_user_mapping_sets_user_id_when_header_present():
         ]
     }
 
-    result = LiteLLMProxyRequestSetup.add_internal_user_from_user_mapping(
-        general_settings, user_api_key_dict, headers
-    )
+    result = LiteLLMProxyRequestSetup.add_internal_user_from_user_mapping(general_settings, user_api_key_dict, headers)
 
     assert result is user_api_key_dict
     assert user_api_key_dict.user_id == "internal-user-123"
@@ -3651,9 +3535,7 @@ def test_add_internal_user_from_user_mapping_no_header_or_mapping_returns_unchan
     assert user_api_key_dict.user_id is None
 
     general_settings = {
-        "user_header_mappings": [
-            {"header_name": "X-OpenWebUI-User-Id", "litellm_user_role": "internal_user"}
-        ]
+        "user_header_mappings": [{"header_name": "X-OpenWebUI-User-Id", "litellm_user_role": "internal_user"}]
     }
     result = LiteLLMProxyRequestSetup.add_internal_user_from_user_mapping(
         general_settings, user_api_key_dict, {"Other": "value"}
@@ -3673,9 +3555,7 @@ def test_get_sanitized_user_information_from_key_includes_guardrails_metadata():
         metadata={"guardrails": ["presidio", "aporia"], "other_field": "value"},
     )
 
-    result = LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(
-        user_api_key_dict=user_api_key_dict
-    )
+    result = LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict=user_api_key_dict)
 
     assert result["user_api_key_auth_metadata"] is not None
     assert "guardrails" in result["user_api_key_auth_metadata"]
@@ -3704,9 +3584,7 @@ def test_user_and_team_spend_and_budget_flow_to_standard_logging_metadata():
         team_max_budget=1000.0,
     )
 
-    sanitized = LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(
-        user_api_key_dict=user_api_key_dict
-    )
+    sanitized = LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict=user_api_key_dict)
 
     assert sanitized["user_api_key_spend"] == 1.5
     assert sanitized["user_api_key_max_budget"] == 10.0
@@ -3715,9 +3593,7 @@ def test_user_and_team_spend_and_budget_flow_to_standard_logging_metadata():
     assert sanitized["user_api_key_team_spend"] == 250.75
     assert sanitized["user_api_key_team_max_budget"] == 1000.0
 
-    logging_metadata = StandardLoggingPayloadSetup.get_standard_logging_metadata(
-        dict(sanitized)
-    )
+    logging_metadata = StandardLoggingPayloadSetup.get_standard_logging_metadata(dict(sanitized))
 
     assert logging_metadata["user_api_key_user_spend"] == 25.5
     assert logging_metadata["user_api_key_user_max_budget"] == 100.0
@@ -3734,12 +3610,8 @@ def test_user_and_team_spend_and_budget_default_to_none_in_standard_logging_meta
 
     user_api_key_dict = UserAPIKeyAuth(api_key="test-key-hash")
 
-    sanitized = LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(
-        user_api_key_dict=user_api_key_dict
-    )
-    logging_metadata = StandardLoggingPayloadSetup.get_standard_logging_metadata(
-        dict(sanitized)
-    )
+    sanitized = LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict=user_api_key_dict)
+    logging_metadata = StandardLoggingPayloadSetup.get_standard_logging_metadata(dict(sanitized))
 
     assert logging_metadata["user_api_key_user_spend"] is None
     assert logging_metadata["user_api_key_user_max_budget"] is None
@@ -4071,22 +3943,16 @@ async def test_embedding_header_forwarding_with_model_group():
 
         # Verify that only x- prefixed headers (except x-stainless) were forwarded
         forwarded_headers = updated_data["headers"]
-        assert (
-            "X-Custom-Header" in forwarded_headers
-        ), "X-Custom-Header should be forwarded"
+        assert "X-Custom-Header" in forwarded_headers, "X-Custom-Header should be forwarded"
         assert forwarded_headers["X-Custom-Header"] == "custom-value"
         assert "X-Request-ID" in forwarded_headers, "X-Request-ID should be forwarded"
         assert forwarded_headers["X-Request-ID"] == "test-request-123"
 
         # Verify that authorization header was NOT forwarded (sensitive header)
-        assert (
-            "Authorization" not in forwarded_headers
-        ), "Authorization header should not be forwarded"
+        assert "Authorization" not in forwarded_headers, "Authorization header should not be forwarded"
 
         # Verify that Content-Type was NOT forwarded (doesn't start with x-)
-        assert (
-            "Content-Type" not in forwarded_headers
-        ), "Content-Type should not be forwarded"
+        assert "Content-Type" not in forwarded_headers, "Content-Type should not be forwarded"
 
         # Verify original data fields are preserved
         assert updated_data["model"] == "local-openai/text-embedding-3-small"
@@ -4142,9 +4008,9 @@ async def test_embedding_header_forwarding_without_model_group_config():
         )
 
         # Verify that headers were NOT added since model is not in forward list
-        assert (
-            "headers" not in updated_data or updated_data.get("headers") is None
-        ), "Headers should not be forwarded for models not in forward_client_headers_to_llm_api list"
+        assert "headers" not in updated_data or updated_data.get("headers") is None, (
+            "Headers should not be forwarded for models not in forward_client_headers_to_llm_api list"
+        )
 
         # Verify original data fields are preserved
         assert updated_data["model"] == "text-embedding-ada-002"
@@ -4198,9 +4064,7 @@ async def test_add_guardrails_from_policy_engine():
     attachment_registry = get_attachment_registry()
     attachment_registry._attachments = [
         PolicyAttachment(policy="global-baseline", scope="*"),  # applies to all
-        PolicyAttachment(
-            policy="healthcare", teams=["healthcare-team"]
-        ),  # applies to healthcare team
+        PolicyAttachment(policy="healthcare", teams=["healthcare-team"]),  # applies to healthcare team
     ]
     attachment_registry._initialized = True
 
@@ -4269,9 +4133,9 @@ async def test_add_guardrails_from_policy_engine_accepts_dynamic_policies_and_po
     )
 
     # Verify that 'policies' was removed from the request body
-    assert (
-        "policies" not in data
-    ), "'policies' should be removed from request body to prevent forwarding to LLM provider"
+    assert "policies" not in data, (
+        "'policies' should be removed from request body to prevent forwarding to LLM provider"
+    )
 
     # Verify that other fields are preserved
     assert "model" in data
@@ -4316,9 +4180,7 @@ async def test_api_created_global_policy_applies_to_new_key_without_restart():
             "runtime-global-policy",
             Policy(guardrails=PolicyGuardrails(add=["runtime-guardrail"])),
         )
-        attachment_registry.add_attachment(
-            PolicyAttachment(policy="runtime-global-policy", scope="*")
-        )
+        attachment_registry.add_attachment(PolicyAttachment(policy="runtime-global-policy", scope="*"))
 
         await add_guardrails_from_policy_engine(
             data=data,
@@ -4415,9 +4277,7 @@ async def test_bearer_token_not_in_debug_logs():
     from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request
     from litellm.proxy.proxy_server import ProxyConfig
 
-    secret_token = (
-        "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.fakesignature"
-    )
+    secret_token = "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.fakesignature"
 
     mock_request = MagicMock(spec=Request)
     mock_request.headers = {
@@ -4463,8 +4323,7 @@ async def test_bearer_token_not_in_debug_logs():
 
     log_output = log_capture.getvalue()
     assert secret_token not in log_output, (
-        f"Bearer token leaked in debug logs. "
-        f"Found token in log output:\n{log_output[:500]}"
+        f"Bearer token leaked in debug logs. Found token in log output:\n{log_output[:500]}"
     )
 
 
@@ -4629,9 +4488,7 @@ def test_apply_overrides_project_model_specific(setup_test_credentials):
         api_key="test-key",
         team_metadata={
             "model_config": {
-                "defaultconfig": {
-                    "azure": {"litellm_credentials": "hotel-azure-eastus"}
-                },
+                "defaultconfig": {"azure": {"litellm_credentials": "hotel-azure-eastus"}},
                 "gpt-4": {"azure": {"litellm_credentials": "hotel-azure-westus"}},
             }
         },
@@ -4642,9 +4499,7 @@ def test_apply_overrides_project_model_specific(setup_test_credentials):
             }
         },
     )
-    _apply_credential_overrides_from_model_config(
-        data=data, user_api_key_dict=user_api_key_dict
-    )
+    _apply_credential_overrides_from_model_config(data=data, user_api_key_dict=user_api_key_dict)
     assert data["api_base"] == "https://hotel-rec-vision.openai.azure.com/"
     assert data["api_key"] == "key-hotel-rec-vision"
     assert data["api_version"] == "2024-06-01"
@@ -4657,9 +4512,7 @@ def test_apply_overrides_project_default(setup_test_credentials):
         api_key="test-key",
         team_metadata={
             "model_config": {
-                "defaultconfig": {
-                    "azure": {"litellm_credentials": "hotel-azure-eastus"}
-                },
+                "defaultconfig": {"azure": {"litellm_credentials": "hotel-azure-eastus"}},
                 "gpt-4": {"azure": {"litellm_credentials": "hotel-azure-westus"}},
             }
         },
@@ -4670,9 +4523,7 @@ def test_apply_overrides_project_default(setup_test_credentials):
             }
         },
     )
-    _apply_credential_overrides_from_model_config(
-        data=data, user_api_key_dict=user_api_key_dict
-    )
+    _apply_credential_overrides_from_model_config(data=data, user_api_key_dict=user_api_key_dict)
     assert data["api_base"] == "https://hotel-rec-app.openai.azure.com/"
     assert data["api_key"] == "key-hotel-rec"
 
@@ -4684,17 +4535,13 @@ def test_apply_overrides_team_model_specific(setup_test_credentials):
         api_key="test-key",
         team_metadata={
             "model_config": {
-                "defaultconfig": {
-                    "azure": {"litellm_credentials": "hotel-azure-eastus"}
-                },
+                "defaultconfig": {"azure": {"litellm_credentials": "hotel-azure-eastus"}},
                 "gpt-4": {"azure": {"litellm_credentials": "hotel-azure-westus"}},
             }
         },
         project_metadata={},
     )
-    _apply_credential_overrides_from_model_config(
-        data=data, user_api_key_dict=user_api_key_dict
-    )
+    _apply_credential_overrides_from_model_config(data=data, user_api_key_dict=user_api_key_dict)
     assert data["api_base"] == "https://hotel-westus.openai.azure.com/"
     assert data["api_key"] == "key-hotel-westus"
 
@@ -4706,17 +4553,13 @@ def test_apply_overrides_team_default(setup_test_credentials):
         api_key="test-key",
         team_metadata={
             "model_config": {
-                "defaultconfig": {
-                    "azure": {"litellm_credentials": "hotel-azure-eastus"}
-                },
+                "defaultconfig": {"azure": {"litellm_credentials": "hotel-azure-eastus"}},
                 "gpt-4": {"azure": {"litellm_credentials": "hotel-azure-westus"}},
             }
         },
         project_metadata={},
     )
-    _apply_credential_overrides_from_model_config(
-        data=data, user_api_key_dict=user_api_key_dict
-    )
+    _apply_credential_overrides_from_model_config(data=data, user_api_key_dict=user_api_key_dict)
     assert data["api_base"] == "https://hotel-eastus.openai.azure.com/"
     assert data["api_key"] == "key-hotel-eastus"
 
@@ -4729,9 +4572,7 @@ def test_apply_overrides_no_config(setup_test_credentials):
         team_metadata={},
         project_metadata={},
     )
-    _apply_credential_overrides_from_model_config(
-        data=data, user_api_key_dict=user_api_key_dict
-    )
+    _apply_credential_overrides_from_model_config(data=data, user_api_key_dict=user_api_key_dict)
     assert "api_base" not in data
     assert "api_key" not in data
 
@@ -4747,17 +4588,9 @@ def test_apply_overrides_clientside_credentials_take_precedence(
     }
     user_api_key_dict = UserAPIKeyAuth(
         api_key="test-key",
-        team_metadata={
-            "model_config": {
-                "defaultconfig": {
-                    "azure": {"litellm_credentials": "hotel-azure-eastus"}
-                }
-            }
-        },
-    )
-    _apply_credential_overrides_from_model_config(
-        data=data, user_api_key_dict=user_api_key_dict
+        team_metadata={"model_config": {"defaultconfig": {"azure": {"litellm_credentials": "hotel-azure-eastus"}}}},
     )
+    _apply_credential_overrides_from_model_config(data=data, user_api_key_dict=user_api_key_dict)
     assert data["api_base"] == "https://my-custom-endpoint.openai.azure.com/"
     assert data["api_key"] == "my-custom-key"
 
@@ -4767,15 +4600,9 @@ def test_apply_overrides_missing_credential_name(setup_test_credentials):
     data = {"model": "gpt-4"}
     user_api_key_dict = UserAPIKeyAuth(
         api_key="test-key",
-        team_metadata={
-            "model_config": {
-                "gpt-4": {"azure": {"litellm_credentials": "nonexistent-credential"}}
-            }
-        },
-    )
-    _apply_credential_overrides_from_model_config(
-        data=data, user_api_key_dict=user_api_key_dict
+        team_metadata={"model_config": {"gpt-4": {"azure": {"litellm_credentials": "nonexistent-credential"}}}},
     )
+    _apply_credential_overrides_from_model_config(data=data, user_api_key_dict=user_api_key_dict)
     assert "api_base" not in data
     assert "api_key" not in data
 
@@ -4785,17 +4612,9 @@ def test_apply_overrides_api_version_only_if_present(setup_test_credentials):
     data = {"model": "gpt-3.5"}
     user_api_key_dict = UserAPIKeyAuth(
         api_key="test-key",
-        team_metadata={
-            "model_config": {
-                "defaultconfig": {
-                    "azure": {"litellm_credentials": "hotel-azure-eastus"}
-                }
-            }
-        },
-    )
-    _apply_credential_overrides_from_model_config(
-        data=data, user_api_key_dict=user_api_key_dict
+        team_metadata={"model_config": {"defaultconfig": {"azure": {"litellm_credentials": "hotel-azure-eastus"}}}},
     )
+    _apply_credential_overrides_from_model_config(data=data, user_api_key_dict=user_api_key_dict)
     assert data["api_base"] == "https://hotel-eastus.openai.azure.com/"
     assert data["api_key"] == "key-hotel-eastus"
     assert "api_version" not in data
@@ -4806,15 +4625,9 @@ def test_apply_overrides_no_model_in_data(setup_test_credentials):
     data = {"messages": [{"role": "user", "content": "hello"}]}
     user_api_key_dict = UserAPIKeyAuth(
         api_key="test-key",
-        team_metadata={
-            "model_config": {
-                "defaultconfig": {"azure": {"litellm_credentials": "some-cred"}}
-            }
-        },
-    )
-    _apply_credential_overrides_from_model_config(
-        data=data, user_api_key_dict=user_api_key_dict
+        team_metadata={"model_config": {"defaultconfig": {"azure": {"litellm_credentials": "some-cred"}}}},
     )
+    _apply_credential_overrides_from_model_config(data=data, user_api_key_dict=user_api_key_dict)
     assert "api_base" not in data
 
 
@@ -4826,9 +4639,7 @@ def test_apply_overrides_none_metadata(setup_test_credentials):
         team_metadata=None,
         project_metadata=None,
     )
-    _apply_credential_overrides_from_model_config(
-        data=data, user_api_key_dict=user_api_key_dict
-    )
+    _apply_credential_overrides_from_model_config(data=data, user_api_key_dict=user_api_key_dict)
     assert "api_base" not in data
 
 
@@ -4837,15 +4648,9 @@ def test_apply_overrides_clientside_api_version_preserved(setup_test_credentials
     data = {"model": "gpt-4-vision", "api_version": "2025-01-01"}
     user_api_key_dict = UserAPIKeyAuth(
         api_key="test-key",
-        team_metadata={
-            "model_config": {
-                "gpt-4-vision": {"azure": {"litellm_credentials": "hotel-rec-vision"}}
-            }
-        },
-    )
-    _apply_credential_overrides_from_model_config(
-        data=data, user_api_key_dict=user_api_key_dict
+        team_metadata={"model_config": {"gpt-4-vision": {"azure": {"litellm_credentials": "hotel-rec-vision"}}}},
     )
+    _apply_credential_overrides_from_model_config(data=data, user_api_key_dict=user_api_key_dict)
     # api_base and api_key should be set from credential
     assert data["api_base"] == "https://hotel-rec-vision.openai.azure.com/"
     assert data["api_key"] == "key-hotel-rec-vision"
@@ -4858,9 +4663,7 @@ def test_resolve_non_dict_model_config_ignored():
     result = _resolve_credential_from_model_config("gpt-4", "not-a-dict", None)
     assert result is None
 
-    result = _resolve_credential_from_model_config(
-        "gpt-4", None, ["also", "not", "a", "dict"]
-    )
+    result = _resolve_credential_from_model_config("gpt-4", None, ["also", "not", "a", "dict"])
     assert result is None
 
     # Valid config still works alongside invalid one
@@ -4878,9 +4681,7 @@ def test_resolve_pre_alias_model_name_fallback():
         "gpt-4": {"azure": {"litellm_credentials": "team-gpt4"}},
     }
     # Post-alias name doesn't match, but pre-alias does (team scope)
-    result = _resolve_credential_from_model_config(
-        "azure/gpt-4-0613", None, team_config, pre_alias_model_name="gpt-4"
-    )
+    result = _resolve_credential_from_model_config("azure/gpt-4-0613", None, team_config, pre_alias_model_name="gpt-4")
     assert result == "team-gpt4"
 
     # Same test for project scope
@@ -4900,15 +4701,11 @@ def test_resolve_post_alias_name_takes_priority():
         "gpt-4o-team-1": {"azure": {"litellm_credentials": "post-alias-cred"}},
     }
     # Team scope
-    result = _resolve_credential_from_model_config(
-        "gpt-4o-team-1", None, team_config, pre_alias_model_name="gpt-4"
-    )
+    result = _resolve_credential_from_model_config("gpt-4o-team-1", None, team_config, pre_alias_model_name="gpt-4")
     assert result == "post-alias-cred"
 
     # Project scope
-    result = _resolve_credential_from_model_config(
-        "gpt-4o-team-1", team_config, None, pre_alias_model_name="gpt-4"
-    )
+    result = _resolve_credential_from_model_config("gpt-4o-team-1", team_config, None, pre_alias_model_name="gpt-4")
     assert result == "post-alias-cred"
 
 
@@ -4940,15 +4737,9 @@ def test_apply_overrides_feature_flag_disabled_by_default():
     data = {"model": "gpt-4"}
     user_api_key_dict = UserAPIKeyAuth(
         api_key="test-key",
-        team_metadata={
-            "model_config": {
-                "gpt-4": {"azure": {"litellm_credentials": "hotel-azure-eastus"}}
-            }
-        },
-    )
-    _apply_credential_overrides_from_model_config(
-        data=data, user_api_key_dict=user_api_key_dict
+        team_metadata={"model_config": {"gpt-4": {"azure": {"litellm_credentials": "hotel-azure-eastus"}}}},
     )
+    _apply_credential_overrides_from_model_config(data=data, user_api_key_dict=user_api_key_dict)
     assert "api_base" not in data
     assert "api_key" not in data
 
@@ -5108,9 +4899,7 @@ async def test_team_guardrail_merges_with_global_policy():
     policy_registry = get_policy_registry()
     policy_registry._policies = {
         "global-policy": Policy(
-            guardrails=PolicyGuardrails(
-                add=["policy-guardrail-1", "policy-guardrail-2"]
-            ),
+            guardrails=PolicyGuardrails(add=["policy-guardrail-1", "policy-guardrail-2"]),
         ),
     }
     policy_registry._initialized = True
@@ -5131,18 +4920,10 @@ async def test_team_guardrail_merges_with_global_policy():
 
         guardrails = data["metadata"].get("guardrails", [])
 
-        assert (
-            "team-direct-guardrail" in guardrails
-        ), f"Team guardrail missing from merged list: {guardrails}"
-        assert (
-            "policy-guardrail-1" in guardrails
-        ), f"policy-guardrail-1 missing: {guardrails}"
-        assert (
-            "policy-guardrail-2" in guardrails
-        ), f"policy-guardrail-2 missing: {guardrails}"
-        assert len(guardrails) == len(
-            set(guardrails)
-        ), f"Duplicates in guardrails list: {guardrails}"
+        assert "team-direct-guardrail" in guardrails, f"Team guardrail missing from merged list: {guardrails}"
+        assert "policy-guardrail-1" in guardrails, f"policy-guardrail-1 missing: {guardrails}"
+        assert "policy-guardrail-2" in guardrails, f"policy-guardrail-2 missing: {guardrails}"
+        assert len(guardrails) == len(set(guardrails)), f"Duplicates in guardrails list: {guardrails}"
 
         # Verify get_guardrail_from_metadata returns the merged list even
         # when litellm_metadata is present (the bug: it returned [] before fix)
@@ -5153,9 +4934,9 @@ async def test_team_guardrail_merges_with_global_policy():
 
         dummy = _DummyGuardrail(guardrail_name="team-direct-guardrail")
         returned = dummy.get_guardrail_from_metadata(data)
-        assert (
-            "team-direct-guardrail" in returned
-        ), f"get_guardrail_from_metadata shadowed by litellm_metadata; got: {returned}"
+        assert "team-direct-guardrail" in returned, (
+            f"get_guardrail_from_metadata shadowed by litellm_metadata; got: {returned}"
+        )
 
     finally:
         policy_registry._policies = {}
@@ -5208,9 +4989,7 @@ def test_get_guardrail_from_metadata_reads_litellm_metadata_when_no_metadata():
     }
 
     result = dummy.get_guardrail_from_metadata(data)
-    assert result == [
-        "my-guardrail"
-    ], f"Expected guardrails from litellm_metadata fallback, got: {result}"
+    assert result == ["my-guardrail"], f"Expected guardrails from litellm_metadata fallback, got: {result}"
 
 
 def _build_request_mock_with_headers(headers: dict) -> Request:
@@ -5237,9 +5016,7 @@ class TestApplyClientTagPolicyPreAuth:
     """
 
     def test_merges_header_tags_into_metadata(self):
-        request_mock = _build_request_mock_with_headers(
-            {"x-litellm-tags": "tenant:acme,env:prod"}
-        )
+        request_mock = _build_request_mock_with_headers({"x-litellm-tags": "tenant:acme,env:prod"})
         data = {"model": "gpt-3.5-turbo"}
         user_api_key_dict = UserAPIKeyAuth(
             api_key="hashed-key",
@@ -5256,9 +5033,7 @@ class TestApplyClientTagPolicyPreAuth:
         assert data["metadata"]["tags"] == ["tenant:acme", "env:prod"]
 
     def test_unions_header_tags_with_existing_metadata_tags(self):
-        request_mock = _build_request_mock_with_headers(
-            {"x-litellm-tags": "tenant:acme,env:prod"}
-        )
+        request_mock = _build_request_mock_with_headers({"x-litellm-tags": "tenant:acme,env:prod"})
         data = {
             "model": "gpt-3.5-turbo",
             "metadata": {"tags": ["env:prod", "team:platform"]},
@@ -5283,9 +5058,7 @@ class TestApplyClientTagPolicyPreAuth:
         # (inside common_checks) enforces per-tag budgets on whatever tags
         # it sees in request_data, including body tags. The helper only
         # adds header tags to metadata.tags.
-        request_mock = _build_request_mock_with_headers(
-            {"x-litellm-tags": "tenant:acme"}
-        )
+        request_mock = _build_request_mock_with_headers({"x-litellm-tags": "tenant:acme"})
         data = {
             "model": "gpt-3.5-turbo",
             "tags": ["root-tag"],
@@ -5312,9 +5085,7 @@ class TestApplyClientTagPolicyPreAuth:
         ]
 
     def test_uses_litellm_metadata_when_present(self):
-        request_mock = _build_request_mock_with_headers(
-            {"x-litellm-tags": "tenant:acme"}
-        )
+        request_mock = _build_request_mock_with_headers({"x-litellm-tags": "tenant:acme"})
         data = {
             "model": "gpt-3.5-turbo",
             "litellm_metadata": {"foo": "bar"},
@@ -5409,9 +5180,7 @@ class TestApplyClientTagPolicyPreAuth:
             litellm_budget_table=LiteLLM_BudgetTable(max_budget=0.10),
         )
 
-        async def mock_get_current_spend(
-            counter_key, fallback_spend, max_budget=None, **kwargs
-        ):
+        async def mock_get_current_spend(counter_key, fallback_spend, max_budget=None, **kwargs):
             if counter_key == "spend:tag:paid":
                 return 0.50
             return fallback_spend
@@ -5446,9 +5215,7 @@ class TestApplyClientTagPolicyPreAuth:
         from litellm.proxy.auth.auth_checks import _tag_max_budget_check
         from litellm.proxy.utils import ProxyLogging
 
-        request_mock = _build_request_mock_with_headers(
-            {"x-litellm-tags": "tenant:acme"}
-        )
+        request_mock = _build_request_mock_with_headers({"x-litellm-tags": "tenant:acme"})
         data = {"model": "gpt-3.5-turbo"}
         user_api_key_dict = UserAPIKeyAuth(
             api_key="hashed-key",
@@ -5468,9 +5235,7 @@ class TestApplyClientTagPolicyPreAuth:
             litellm_budget_table=LiteLLM_BudgetTable(max_budget=0.10),
         )
 
-        async def mock_get_current_spend(
-            counter_key, fallback_spend, max_budget=None, **kwargs
-        ):
+        async def mock_get_current_spend(counter_key, fallback_spend, max_budget=None, **kwargs):
             if counter_key == "spend:tag:tenant:acme":
                 return 0.50
             return fallback_spend
@@ -5505,9 +5270,7 @@ class TestApplyClientTagPolicyPreAuth:
             "/v1/messages",
         ],
     )
-    async def test_header_tags_visible_to_tag_max_budget_check_on_metadata_route(
-        self, route
-    ):
+    async def test_header_tags_visible_to_tag_max_budget_check_on_metadata_route(self, route):
         """Regression: on LITELLM_METADATA_ROUTES (bedrock, /v1/messages, ...),
         common_checks pre-seeds ``litellm_metadata`` and writes key tags there
         before ``_tag_max_budget_check`` reads from the same key. The auth wrapper
@@ -5521,9 +5284,7 @@ class TestApplyClientTagPolicyPreAuth:
         from litellm.proxy.auth.auth_checks import common_checks
         from litellm.proxy.utils import ProxyLogging
 
-        request_mock = _build_request_mock_with_headers(
-            {"x-litellm-tags": "tenant:acme"}
-        )
+        request_mock = _build_request_mock_with_headers({"x-litellm-tags": "tenant:acme"})
         data = {"model": "us.anthropic.claude-sonnet-4-6"}
         valid_token = UserAPIKeyAuth(
             token="test-token",
@@ -5548,9 +5309,7 @@ class TestApplyClientTagPolicyPreAuth:
             litellm_budget_table=LiteLLM_BudgetTable(max_budget=0.10),
         )
 
-        async def mock_get_current_spend(
-            counter_key, fallback_spend, max_budget=None, **kwargs
-        ):
+        async def mock_get_current_spend(counter_key, fallback_spend, max_budget=None, **kwargs):
             if counter_key == "spend:tag:tenant:acme":
                 return 0.50
             return fallback_spend
@@ -5718,9 +5477,7 @@ class TestApplyKeyTagsPreAuth:
             litellm_budget_table=LiteLLM_BudgetTable(max_budget=0.10),
         )
 
-        async def mock_get_current_spend(
-            counter_key, fallback_spend, max_budget=None, **kwargs
-        ):
+        async def mock_get_current_spend(counter_key, fallback_spend, max_budget=None, **kwargs):
             if counter_key == "spend:tag:engineering":
                 return 0.50
             return fallback_spend
@@ -5771,9 +5528,7 @@ class TestApplyKeyTagsPreAuth:
             litellm_budget_table=LiteLLM_BudgetTable(max_budget=0.10),
         )
 
-        async def mock_get_current_spend(
-            counter_key, fallback_spend, max_budget=None, **kwargs
-        ):
+        async def mock_get_current_spend(counter_key, fallback_spend, max_budget=None, **kwargs):
             if counter_key == "spend:tag:engineering":
                 return 0.05
             return fallback_spend
@@ -5854,9 +5609,7 @@ def test_resolve_provider_from_deployment_falls_back_to_pre_alias():
 
     router.get_deployment_by_model_group_name.side_effect = lookup
 
-    result = _resolve_provider_from_deployment(
-        router, "post-alias-name", pre_alias_model_name="pre-alias-name"
-    )
+    result = _resolve_provider_from_deployment(router, "post-alias-name", pre_alias_model_name="pre-alias-name")
     assert result == "bedrock"
 
 
@@ -5921,17 +5674,9 @@ def test_apply_overrides_no_router_keeps_legacy_behaviour(setup_test_credentials
     data = {"model": "gpt-4"}
     user_api_key_dict = UserAPIKeyAuth(
         api_key="test-key",
-        team_metadata={
-            "model_config": {
-                "defaultconfig": {
-                    "azure": {"litellm_credentials": "hotel-azure-eastus"}
-                }
-            }
-        },
-    )
-    _apply_credential_overrides_from_model_config(
-        data=data, user_api_key_dict=user_api_key_dict, llm_router=None
+        team_metadata={"model_config": {"defaultconfig": {"azure": {"litellm_credentials": "hotel-azure-eastus"}}}},
     )
+    _apply_credential_overrides_from_model_config(data=data, user_api_key_dict=user_api_key_dict, llm_router=None)
     assert data["api_base"] == "https://hotel-eastus.openai.azure.com/"
     assert data["api_key"] == "key-hotel-eastus"
 
@@ -5957,9 +5702,7 @@ def test_apply_overrides_provider_prefix_in_model_skips_router_lookup(
     )
 
     router = MagicMock()
-    _apply_credential_overrides_from_model_config(
-        data=data, user_api_key_dict=user_api_key_dict, llm_router=router
-    )
+    _apply_credential_overrides_from_model_config(data=data, user_api_key_dict=user_api_key_dict, llm_router=router)
     assert data["api_base"] == "https://hotel-eastus.openai.azure.com/"
     assert data["api_key"] == "key-hotel-eastus"
     router.get_deployment_by_model_group_name.assert_not_called()
@@ -6338,9 +6081,7 @@ def test_get_sanitized_user_information_from_key_drops_callback_config():
         },
     )
 
-    result = LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(
-        user_api_key_dict=user_api_key_dict
-    )
+    result = LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict=user_api_key_dict)
 
     auth_metadata = result["user_api_key_auth_metadata"]
     assert "logging" not in auth_metadata
@@ -6380,9 +6121,7 @@ def test_team_alias_targeting_deleted_team_deployment_keeps_requested_model(monk
     )
 
     with patch("litellm.proxy.proxy_server.llm_router", _MockRouter()):
-        _update_model_if_team_alias_exists(
-            data=test_data, user_api_key_dict=user_api_key_dict
-        )
+        _update_model_if_team_alias_exists(data=test_data, user_api_key_dict=user_api_key_dict)
 
     assert test_data.get("model") == "gpt-4"
 
@@ -6406,9 +6145,7 @@ def test_team_alias_targeting_live_team_deployment_still_rewrites(monkeypatch):
     )
 
     with patch("litellm.proxy.proxy_server.llm_router", _MockRouter()):
-        _update_model_if_team_alias_exists(
-            data=test_data, user_api_key_dict=user_api_key_dict
-        )
+        _update_model_if_team_alias_exists(data=test_data, user_api_key_dict=user_api_key_dict)
 
     assert test_data.get("model") == "model_name_team-1_live-uuid"
 
@@ -6545,7 +6282,6 @@ async def test_add_litellm_data_to_request_keeps_every_forwarded_credential_out_
         assert value not in logged
 
 
-
 @pytest.mark.parametrize(
     "header, expected_redacted",
     [
@@ -6571,7 +6307,6 @@ def test_redact_credential_headers_classifies_each_header(header, expected_redac
     assert headers[header] == "secret-value"
 
 
-
 @pytest.mark.asyncio
 async def test_add_litellm_data_to_request_debug_log_does_not_print_credentials():
     """The request-header debug line carries values the stdout secret filter does not match."""
@@ -7191,8 +6926,7 @@ AUTHORIZATION_HEADER_CASINGS = ["authorization", "Authorization", "AUTHORIZATION
 LEAK_TARGET_PROVIDERS = ["bedrock", "bedrock_converse", "vertex_ai"]
 
 BEDROCK_ENDPOINT = (
-    "https://bedrock-runtime.us-west-2.amazonaws.com"
-    "/model/us.anthropic.claude-sonnet-4-5-20250929-v1:0/invoke"
+    "https://bedrock-runtime.us-west-2.amazonaws.com/model/us.anthropic.claude-sonnet-4-5-20250929-v1:0/invoke"
 )
 BEDROCK_REGION = "us-west-2"
 BEDROCK_REQUEST_DATA = {"messages": [{"role": "user", "content": "Say OK"}], "max_tokens": 32}
@@ -7250,9 +6984,7 @@ def _signed_headers_component(signature: str, component: str) -> str:
 
 @pytest.mark.parametrize("authorization_header_name", AUTHORIZATION_HEADER_CASINGS)
 @pytest.mark.parametrize("custom_llm_provider", LEAK_TARGET_PROVIDERS)
-def test_oauth_credential_is_never_forwarded_to_bedrock_or_vertex(
-    authorization_header_name, custom_llm_provider
-):
+def test_oauth_credential_is_never_forwarded_to_bedrock_or_vertex(authorization_header_name, custom_llm_provider):
     """
     A client's Anthropic OAuth credential is meaningless to AWS and Google, and sending it
     there both breaks the request and hands a third-party cloud a credential it should
@@ -7280,9 +7012,7 @@ def test_oauth_credential_entry_is_scoped_to_anthropic_alone():
     if not isinstance(scoped_headers, list):
         scoped_headers = [scoped_headers]
 
-    credential_entries = [
-        entry for entry in scoped_headers if OAUTH_TOKEN in entry["extra_headers"].values()
-    ]
+    credential_entries = [entry for entry in scoped_headers if OAUTH_TOKEN in entry["extra_headers"].values()]
     assert [entry["custom_llm_provider"] for entry in credential_entries] == ["anthropic"]
 
 
@@ -7344,9 +7074,7 @@ def test_bedrock_get_request_headers_keeps_the_sigv4_signature():
 def test_bedrock_api_key_deployment_keeps_its_own_bearer_token():
     forwarded = _headers_forwarded_to(_client_headers(), "bedrock")
 
-    signed = _signed_headers_for_bedrock(
-        {"Content-Type": "application/json", **forwarded}, api_key=BEDROCK_API_KEY
-    )
+    signed = _signed_headers_for_bedrock({"Content-Type": "application/json", **forwarded}, api_key=BEDROCK_API_KEY)
 
     assert _authorization_values(signed) == [f"Bearer {BEDROCK_API_KEY}"]
 
@@ -7369,6 +7097,8 @@ def test_vertex_sends_exactly_one_authorization_header():
     vertex_request_headers.update(forwarded)
 
     assert _authorization_values(vertex_request_headers) == [GOOGLE_ACCESS_TOKEN]
+
+
 @pytest.mark.asyncio
 async def test_newrelic_team_callback_vars_reach_trusted_field():
     """A key with a newrelic team callback stamps its vars into the proxy-owned
@@ -7737,13 +7467,13 @@ def _request_for(path: str) -> MagicMock:
     return request
 
 
-def _spend_log_session_id(data: dict[str, object]) -> str:
-    """Resolve session_id the way LiteLLM_SpendLogs does: standard_logging_payload.trace_id."""
+def _spend_log_session_id(data: dict[str, object], metadata_key: str = "metadata") -> str | None:
+    """Resolve session_id the way LiteLLM_SpendLogs does, reading the omit decision stamped on the request."""
     from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
     from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
     from litellm.proxy.spend_tracking.spend_tracking_utils import _get_session_id_for_spend_log
 
-    metadata = data["metadata"]
+    metadata = data[metadata_key]
     assert isinstance(metadata, dict)
     litellm_params = get_litellm_params(
         litellm_session_id=str(data["litellm_session_id"]) if "litellm_session_id" in data else None,
@@ -7754,7 +7484,12 @@ def _spend_log_session_id(data: dict[str, object]) -> str:
         logging_obj=SimpleNamespace(litellm_trace_id="per-call-random-trace-id"),
         litellm_params=litellm_params,
     )
-    return _get_session_id_for_spend_log(kwargs={}, standard_logging_payload={"trace_id": trace_id})
+    return _get_session_id_for_spend_log(
+        kwargs={},
+        metadata=metadata,
+        standard_logging_payload={"trace_id": trace_id},
+        omit_when_missing=bool(metadata.get(SESSION_ID_OMITTED_METADATA_KEY)),
+    )
 
 
 @pytest.mark.asyncio
@@ -7780,9 +7515,10 @@ async def test_missing_session_id_generate_makes_spend_log_and_callback_session_
     assert isinstance(callback_session_id, str) and len(callback_session_id) == 36
     assert _spend_log_session_id(updated) == callback_session_id
     assert updated["metadata"][SESSION_ID_GENERATED_METADATA_KEY] is True
-    assert get_fireworks_session_id(
-        {"litellm_session_id": updated["litellm_session_id"], "metadata": updated["metadata"]}
-    ) is None
+    assert (
+        get_fireworks_session_id({"litellm_session_id": updated["litellm_session_id"], "metadata": updated["metadata"]})
+        is None
+    )
 
 
 @pytest.mark.asyncio
@@ -7800,6 +7536,44 @@ async def test_missing_session_id_unset_keeps_legacy_divergence():
     assert _spend_log_session_id(updated) == "per-call-random-trace-id"
 
 
+@pytest.mark.asyncio
+async def test_missing_session_id_omit_leaves_spend_log_session_id_null():
+    """Under `omit` a traceparent still becomes the trace id but never a session id, so SpendLogs and
+    Langfuse agree on having no session."""
+    request = _request_for("/v1/chat/completions")
+    request.headers = {"traceparent": "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"}
+
+    updated = await add_litellm_data_to_request(
+        data={"model": "gpt-4o", "messages": []},
+        request=request,
+        user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"),
+        proxy_config=MagicMock(),
+        general_settings={"missing_session_id": "omit"},
+    )
+
+    assert "session_id" not in updated["metadata"]
+    assert updated["litellm_trace_id"] == "4bf92f3577b34da6a3ce929d0e0e4736"
+    assert updated["metadata"][SESSION_ID_OMITTED_METADATA_KEY] is True
+    assert _spend_log_session_id(updated) is None
+
+
+@pytest.mark.asyncio
+async def test_missing_session_id_omit_keeps_client_supplied_session_id():
+    request = _request_for("/v1/chat/completions")
+    request.headers = {"x-litellm-session-id": "client-session-1"}
+
+    updated = await add_litellm_data_to_request(
+        data={"model": "gpt-4o", "messages": []},
+        request=request,
+        user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"),
+        proxy_config=MagicMock(),
+        general_settings={"missing_session_id": "omit"},
+    )
+
+    assert updated["metadata"]["session_id"] == "client-session-1"
+    assert _spend_log_session_id(updated) == "client-session-1"
+
+
 @pytest.mark.asyncio
 async def test_missing_session_id_generate_reuses_traceparent_trace_id():
     """A W3C traceparent already decides SpendLogs.session_id, so the callback session id must reuse it."""
@@ -7895,3 +7669,38 @@ async def test_missing_session_id_unknown_value_is_ignored():
     )
 
     assert "session_id" not in updated["metadata"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "path, sent_in, metadata_key, general_settings",
+    [
+        ("/v1/chat/completions", "metadata", "metadata", {}),
+        ("/v1/chat/completions", "litellm_metadata", "metadata", {}),
+        ("/v1/chat/completions", "litellm_metadata", "metadata", {"missing_session_id": "generate"}),
+        ("/v1/messages", "litellm_metadata", "litellm_metadata", {}),
+        ("/v1/messages", "metadata", "litellm_metadata", {}),
+        ("/mcp/tools", "metadata", "metadata", {"missing_session_id": "omit"}),
+        ("/mcp/tools", "litellm_metadata", "metadata", {"missing_session_id": "omit"}),
+    ],
+)
+async def test_client_supplied_omit_marker_never_reaches_the_spend_log(
+    path: str, sent_in: str, metadata_key: str, general_settings: dict[str, str]
+):
+    """The omit marker is proxy-owned: only the pre-call policy may set it. A caller that sends it in either
+    metadata bucket, including the one later merged into the route's bucket, must not be able to null out
+    SpendLogs.session_id on a request the proxy did not omit."""
+    updated = await add_litellm_data_to_request(
+        data={"model": "gpt-4o", "messages": [], sent_in: {SESSION_ID_OMITTED_METADATA_KEY: True}},
+        request=_request_for(path),
+        user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"),
+        proxy_config=MagicMock(),
+        general_settings=general_settings,
+    )
+
+    assert SESSION_ID_OMITTED_METADATA_KEY not in updated[metadata_key]
+    assert _spend_log_session_id(updated, metadata_key) == (
+        updated[metadata_key]["session_id"]
+        if general_settings.get("missing_session_id") == "generate"
+        else "per-call-random-trace-id"
+    )
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index 098b43f6433..5246aaf6904 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -25779,9 +25779,9 @@ export interface components {
             mcp_xff_num_trusted_hops?: number | null;
             /**
              * Missing Session Id
-             * @description What to do with LLM API requests that carry no session id (x-litellm-session-id header, metadata.session_id, etc.). 'generate' stamps one id into litellm_session_id, litellm_trace_id and metadata.session_id so SpendLogs and logging callbacks agree; 'reject' returns 400. Unset keeps the legacy behavior where SpendLogs falls back to the trace id while callbacks get no session id.
+             * @description What to do with LLM API requests that carry no session id (x-litellm-session-id header, metadata.session_id, etc.). 'generate' stamps one id into litellm_session_id, litellm_trace_id and metadata.session_id so SpendLogs and logging callbacks agree; 'reject' returns 400; 'omit' leaves SpendLogs.session_id null, matching callbacks such as Langfuse that only record a client-established metadata.session_id. Unset keeps the legacy behavior where SpendLogs falls back to the trace id while callbacks get no session id.
              */
-            missing_session_id?: ("generate" | "reject") | null;
+            missing_session_id?: ("generate" | "reject" | "omit") | null;
             /**
              * Model List Healthy Only
              * @description When true, `/models`, `/v1/models/{id}` and `/model/info` hide models whose backing deployments are all unhealthy, for every caller, without needing `healthy_only=true` per request. Requires `background_health_checks: true`, and keeps deployment health state cached without turning on `enable_health_check_routing`, so routing is unaffected. With no health state nothing is hidden. Hiding is presentation-only, a hidden model can still be called.

From 9c7c7a05ac92aebf1694ba087501f4b7c0c1416d Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Thu, 3 Sep 2026 11:58:25 -0700
Subject: [PATCH 33/42] test(router): type the deployment affinity JWT test
 helpers

---
 .../test_deployment_affinity_check.py         | 24 ++++++++++---------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/tests/test_litellm/router_utils/pre_call_checks/test_deployment_affinity_check.py b/tests/test_litellm/router_utils/pre_call_checks/test_deployment_affinity_check.py
index 1852d641f0a..b5651062098 100644
--- a/tests/test_litellm/router_utils/pre_call_checks/test_deployment_affinity_check.py
+++ b/tests/test_litellm/router_utils/pre_call_checks/test_deployment_affinity_check.py
@@ -1,11 +1,12 @@
 import asyncio
+import itertools
+import json
+from collections.abc import Sequence
+from typing import Final
 from unittest.mock import AsyncMock, patch
 
 import pytest
 
-
-import json
-
 import litellm
 from litellm.caching.dual_cache import DualCache
 from litellm.router_utils.pre_call_checks.deployment_affinity_check import (
@@ -102,11 +103,10 @@ async def test_async_user_key_affinity_routes_to_same_deployment():
 
     # Deterministic routing: first selection uses seq[0], second selection attempts seq[1]
     # unless the list has been filtered to length=1 by deployment affinity.
-    choice_calls = {"count": 0}
+    choice_calls: Final = itertools.count(1)
 
-    def deterministic_choice(seq):
-        choice_calls["count"] += 1
-        if choice_calls["count"] == 1:
+    def deterministic_choice(seq: Sequence[dict[str, object]]) -> dict[str, object]:
+        if next(choice_calls) == 1:
             return seq[0]
         return seq[1] if len(seq) > 1 else seq[0]
 
@@ -1000,7 +1000,7 @@ async def test_model_group_affinity_config_overrides_global():
     assert len(filtered) == 2
 
 
-def _jwt_metadata(user_id: str) -> dict:
+def _jwt_metadata(user_id: str) -> dict[str, str | None]:
     return {"user_api_key_hash": None, "user_api_key_user_id": user_id}
 
 
@@ -1090,7 +1090,7 @@ async def test_proxy_jwt_auth_metadata_pins_per_user():
         enable_responses_api_affinity=False,
     )
 
-    def proxy_request(user_id: str) -> dict:
+    def proxy_request(user_id: str) -> dict[str, object]:
         return LiteLLMProxyRequestSetup.add_user_api_key_auth_to_request_metadata(
             data={"model": model_group, "messages": [{"role": "user", "content": "hi"}], "metadata": {}},
             user_api_key_dict=UserAPIKeyAuth(api_key=None, user_id=user_id),
@@ -1098,12 +1098,14 @@ async def test_proxy_jwt_auth_metadata_pins_per_user():
         )
 
     alice_request = proxy_request("jwt-user-alice")
-    assert alice_request["metadata"]["user_api_key_hash"] is None
+    alice_metadata = alice_request["metadata"]
+    assert isinstance(alice_metadata, dict)
+    assert alice_metadata["user_api_key_hash"] is None
 
     await callback.async_pre_call_deployment_hook(
         kwargs={
             **alice_request,
-            "metadata": {**alice_request["metadata"], "deployment_model_name": model_group},
+            "metadata": {**alice_metadata, "deployment_model_name": model_group},
             "model_info": {"id": "openai-deployment-b"},
         },
         call_type=None,

From 92122086ecc9271640d284fe4e1db32bdfc205ff Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Thu, 3 Sep 2026 19:12:11 +0000
Subject: [PATCH 34/42] fix: stop a cleared Organization field from failing key
 creation (#39316)

* fix: stop a cleared Organization field from failing key creation

Clearing the Organization combobox in the Create Key modal left organization_id set to an empty string, so /key/generate looked up an organization named "" and failed with "Organization doesn't exist in db. Organization=".

OrganizationDropdown now emits null on clear, and GenerateKeyRequest normalizes an empty organization_id or project_id to None the same way it already does for team_id.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test: drop customer-specific docstring from key request normalization test

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: yassin <yassin@berri.ai>
---
 litellm/proxy/_types.py                              |  2 +-
 .../test_key_management_endpoints.py                 | 12 ++++++++++++
 .../common_components/OrganizationDropdown.test.tsx  | 11 +++++++++++
 .../common_components/OrganizationDropdown.tsx       |  4 ++--
 .../src/components/organisms/create_key_button.tsx   |  6 +++---
 .../src/components/templates/key_edit_view.tsx       |  6 +++---
 6 files changed, 32 insertions(+), 9 deletions(-)

diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index 0aea72be1e2..98464d3a127 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -1216,7 +1216,7 @@ class GenerateKeyRequest(KeyRequestBase):
     organization_id: str | None = None
     project_id: str | None = None
 
-    @field_validator("team_id", "organization_id", mode="before")
+    @field_validator("team_id", "organization_id", "project_id", mode="before")
     @classmethod
     def treat_cleared_id_as_unset(cls, v: object) -> object:
         if v == "":
diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py
index 68704f476b5..7954a4693cc 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py
@@ -17505,6 +17505,18 @@ def test_generate_key_request_blank_team_id_is_personal():
     assert GenerateKeyRequest(team_id="team-1").team_id == "team-1"
 
 
+def test_generate_key_request_blank_organization_and_project_id_are_unset():
+    from litellm.proxy._types import RegenerateKeyRequest
+
+    cleared = GenerateKeyRequest(organization_id="", project_id="")
+    assert cleared.organization_id is None
+    assert cleared.project_id is None
+    assert "organization_id" not in cleared.model_dump(exclude_none=True)
+    assert RegenerateKeyRequest(organization_id="").organization_id is None
+    assert GenerateKeyRequest(organization_id="org-1", project_id="proj-1").organization_id == "org-1"
+    assert GenerateKeyRequest(organization_id="org-1", project_id="proj-1").project_id == "proj-1"
+
+
 def test_key_request_blank_organization_id_is_unset():
     from litellm.proxy._types import RegenerateKeyRequest, UpdateKeyRequest
 
diff --git a/ui/litellm-dashboard/src/components/common_components/OrganizationDropdown.test.tsx b/ui/litellm-dashboard/src/components/common_components/OrganizationDropdown.test.tsx
index e0b2b897b36..524ec6a715e 100644
--- a/ui/litellm-dashboard/src/components/common_components/OrganizationDropdown.test.tsx
+++ b/ui/litellm-dashboard/src/components/common_components/OrganizationDropdown.test.tsx
@@ -58,6 +58,17 @@ describe("OrganizationDropdown", () => {
     expect(onChange.mock.calls[0][0]).toBe("org-1");
   });
 
+  it("emits null, never the empty string, when the selection is cleared", async () => {
+    const onChange = vi.fn();
+    const user = userEvent.setup();
+    render(<OrganizationDropdown organizations={MOCK_ORGS} value="org-1" onChange={onChange} />);
+
+    await user.click(screen.getByRole("button", { name: "Clear" }));
+
+    expect(onChange).toHaveBeenCalledTimes(1);
+    expect(onChange).toHaveBeenCalledWith(null);
+  });
+
   it("should filter options by organization id", async () => {
     const user = userEvent.setup();
     render(<OrganizationDropdown organizations={MOCK_ORGS} />);
diff --git a/ui/litellm-dashboard/src/components/common_components/OrganizationDropdown.tsx b/ui/litellm-dashboard/src/components/common_components/OrganizationDropdown.tsx
index 8da35ecd02e..663028b2d92 100644
--- a/ui/litellm-dashboard/src/components/common_components/OrganizationDropdown.tsx
+++ b/ui/litellm-dashboard/src/components/common_components/OrganizationDropdown.tsx
@@ -5,7 +5,7 @@ import { Organization } from "../networking";
 interface OrganizationDropdownProps {
   organizations?: Organization[] | null;
   value?: string;
-  onChange?: (value: string) => void;
+  onChange?: (value: string | null) => void;
   disabled?: boolean;
   loading?: boolean;
   style?: React.CSSProperties;
@@ -32,7 +32,7 @@ const OrganizationDropdown: React.FC<OrganizationDropdownProps> = ({
           sublabel: org.organization_id,
         }))}
         value={value}
-        onValueChange={(organizationId) => onChange?.(organizationId)}
+        onValueChange={(organizationId) => onChange?.(organizationId || null)}
         placeholder={placeholder}
         emptyText={loading ? "Loading organizations…" : "No organizations found"}
         disabled={disabled}
diff --git a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx
index 5749541dcea..ee3a88acba0 100644
--- a/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx
+++ b/ui/litellm-dashboard/src/components/organisms/create_key_button.tsx
@@ -587,9 +587,9 @@ const CreateKey: React.FC<CreateKeyProps> = ({ team, teams, data, addKey, autoOp
     }
   };
 
-  const changeOrganization = (write: FieldWrite) => (orgId: string) => {
-    write(orgId || undefined);
-    setSelectedOrganizationId(orgId || null);
+  const changeOrganization = (write: FieldWrite) => (orgId: string | null) => {
+    write(orgId ?? undefined);
+    setSelectedOrganizationId(orgId);
     // Clear team and project when org changes
     setSelectedCreateKeyTeam(null);
     setSelectedProjectId(null);
diff --git a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx
index 3e772fd0e9b..1330be2788b 100644
--- a/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx
+++ b/ui/litellm-dashboard/src/components/templates/key_edit_view.tsx
@@ -303,9 +303,9 @@ export function KeyEditView({
     }
   };
 
-  const handleOrganizationChange = (setField: (value: string | null) => void, orgId: string | undefined) => {
-    setField(orgId || null);
-    setSelectedOrganizationId(orgId || null);
+  const handleOrganizationChange = (setField: (value: string | null) => void, orgId: string | null) => {
+    setField(orgId);
+    setSelectedOrganizationId(orgId);
     form.setValue("team_id", undefined);
   };
 

From 51d821ae45aef7fa95e222ebcb55486210d08543 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Thu, 3 Sep 2026 12:21:54 -0700
Subject: [PATCH 35/42] fix(cost): bill bedrock_mantle web search at $12 per 1k
 queries using Bedrock's reported count

---
 .../llm_cost_calc/tool_call_cost_tracking.py  |  34 +++--
 ...odel_prices_and_context_window_backup.json |  25 ++++
 litellm/types/llms/openai.py                  |  13 ++
 model_prices_and_context_window.json          |  25 ++++
 .../test_tool_call_cost_tracking.py           | 118 ++++++++++++++++++
 5 files changed, 207 insertions(+), 8 deletions(-)

diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py
index 5504756ceb8..bf99035a6b1 100644
--- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py
+++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py
@@ -5,6 +5,8 @@ Helper utilities for tracking the cost of built-in tools.
 from collections.abc import Mapping
 from typing import Final, Literal
 
+from pydantic import ValidationError
+
 import litellm
 from litellm.constants import OPENAI_FILE_SEARCH_COST_PER_1K_CALLS
 from litellm.litellm_core_utils.llm_cost_calc.utils import (
@@ -13,6 +15,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
 from litellm.types.llms.openai import (
     FileSearchTool,
     ResponsesAPIResponse,
+    ResponsesToolUsage,
     WebSearchOptions,
 )
 from litellm.types.utils import (
@@ -32,6 +35,17 @@ def _output_item_type(output_item: object) -> str | None:
     return item_type if isinstance(item_type, str) else None
 
 
+def _reported_web_search_requests(response_object: ResponsesAPIResponse) -> int | None:
+    tool_usage: Final = getattr(response_object, "tool_usage", None)
+    if tool_usage is None:
+        return None
+    try:
+        web_search: Final = ResponsesToolUsage.model_validate(tool_usage).web_search
+    except ValidationError:
+        return None
+    return None if web_search is None else web_search.num_requests
+
+
 def _usage_reports_server_side_web_search_calls(usage: Usage) -> bool:
     details: Final = getattr(usage, "server_side_tool_usage_details", None)
     if not isinstance(details, Mapping):
@@ -182,15 +196,19 @@ class StandardBuiltInToolCostTracking:
 
         Providers that report a request count in usage (gemini, anthropic, xai, vertex) are handled by
         get_cost_for_web_search_request and never reach here. This path prices per call, so it must count
-        the web_search_call items. Chat-completions responses only expose url_citation annotations with no
-        count, so they floor to a single billable search.
+        the web_search_call items, unless the response reports the billable count itself
+        (Bedrock's tool_usage.web_search.num_requests, which excludes open_page fetches). Chat-completions
+        responses only expose url_citation annotations with no count, so they floor to a single billable search.
         """
-        if isinstance(response_object, ResponsesAPIResponse):
-            count = sum(
-                1 for output_item in response_object.output if _output_item_type(output_item) == "web_search_call"
-            )
-            return max(count, 1)
-        return 1
+        if not isinstance(response_object, ResponsesAPIResponse):
+            return 1
+        reported: Final = _reported_web_search_requests(response_object)
+        if reported is not None:
+            return reported
+        count: Final = sum(
+            1 for output_item in response_object.output if _output_item_type(output_item) == "web_search_call"
+        )
+        return max(count, 1)
 
     @staticmethod
     def _handle_file_search_cost(
diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 9b358a1cedd..ffac34ea37e 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -52911,6 +52911,11 @@
         "cache_read_input_token_cost_above_272k_tokens": 1.1e-06,
         "output_cost_per_token": 3.3e-05,
         "output_cost_per_token_above_272k_tokens": 4.95e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.012,
+            "search_context_size_low": 0.012,
+            "search_context_size_medium": 0.012
+        },
         "litellm_provider": "bedrock_mantle",
         "max_input_tokens": 1050000,
         "max_output_tokens": 128000,
@@ -52945,6 +52950,11 @@
         "cache_read_input_token_cost_above_272k_tokens": 4.4e-07,
         "output_cost_per_token": 1.32e-05,
         "output_cost_per_token_above_272k_tokens": 1.98e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.012,
+            "search_context_size_low": 0.012,
+            "search_context_size_medium": 0.012
+        },
         "litellm_provider": "bedrock_mantle",
         "max_input_tokens": 1050000,
         "max_output_tokens": 128000,
@@ -53007,6 +53017,11 @@
         "cache_read_input_token_cost_above_272k_tokens": 4.4e-08,
         "output_cost_per_token": 1.32e-06,
         "output_cost_per_token_above_272k_tokens": 1.98e-06,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.012,
+            "search_context_size_low": 0.012,
+            "search_context_size_medium": 0.012
+        },
         "litellm_provider": "bedrock_mantle",
         "max_input_tokens": 1050000,
         "max_output_tokens": 128000,
@@ -53195,6 +53210,11 @@
         "cache_read_input_token_cost_above_272k_tokens": 1.1e-06,
         "output_cost_per_token": 3.3e-05,
         "output_cost_per_token_above_272k_tokens": 4.95e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.012,
+            "search_context_size_low": 0.012,
+            "search_context_size_medium": 0.012
+        },
         "litellm_provider": "bedrock_mantle",
         "max_input_tokens": 1050000,
         "max_output_tokens": 128000,
@@ -53226,6 +53246,11 @@
         "cache_read_input_token_cost_above_272k_tokens": 5.5e-07,
         "output_cost_per_token": 1.65e-05,
         "output_cost_per_token_above_272k_tokens": 2.475e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.012,
+            "search_context_size_low": 0.012,
+            "search_context_size_medium": 0.012
+        },
         "litellm_provider": "bedrock_mantle",
         "max_input_tokens": 1050000,
         "max_output_tokens": 128000,
diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py
index 32d88da0085..b33ff954c35 100644
--- a/litellm/types/llms/openai.py
+++ b/litellm/types/llms/openai.py
@@ -66,6 +66,7 @@ from pydantic import (
     ConfigDict,
     Discriminator,
     Field,
+    NonNegativeInt,
     PrivateAttr,
     SerializerFunctionWrapHandler,
     field_serializer,
@@ -1321,6 +1322,18 @@ class ResponseAPIUsage(BaseLiteLLMOpenAIResponseObject):
     model_config = {"extra": "allow"}
 
 
+class WebSearchToolUsage(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    num_requests: NonNegativeInt
+
+
+class ResponsesToolUsage(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    web_search: WebSearchToolUsage | None = None
+
+
 ResponsesAPIStatus = Literal["completed", "failed", "in_progress", "cancelled", "queued", "incomplete"]
 """
 The status of the response generation.
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index 9b358a1cedd..ffac34ea37e 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -52911,6 +52911,11 @@
         "cache_read_input_token_cost_above_272k_tokens": 1.1e-06,
         "output_cost_per_token": 3.3e-05,
         "output_cost_per_token_above_272k_tokens": 4.95e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.012,
+            "search_context_size_low": 0.012,
+            "search_context_size_medium": 0.012
+        },
         "litellm_provider": "bedrock_mantle",
         "max_input_tokens": 1050000,
         "max_output_tokens": 128000,
@@ -52945,6 +52950,11 @@
         "cache_read_input_token_cost_above_272k_tokens": 4.4e-07,
         "output_cost_per_token": 1.32e-05,
         "output_cost_per_token_above_272k_tokens": 1.98e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.012,
+            "search_context_size_low": 0.012,
+            "search_context_size_medium": 0.012
+        },
         "litellm_provider": "bedrock_mantle",
         "max_input_tokens": 1050000,
         "max_output_tokens": 128000,
@@ -53007,6 +53017,11 @@
         "cache_read_input_token_cost_above_272k_tokens": 4.4e-08,
         "output_cost_per_token": 1.32e-06,
         "output_cost_per_token_above_272k_tokens": 1.98e-06,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.012,
+            "search_context_size_low": 0.012,
+            "search_context_size_medium": 0.012
+        },
         "litellm_provider": "bedrock_mantle",
         "max_input_tokens": 1050000,
         "max_output_tokens": 128000,
@@ -53195,6 +53210,11 @@
         "cache_read_input_token_cost_above_272k_tokens": 1.1e-06,
         "output_cost_per_token": 3.3e-05,
         "output_cost_per_token_above_272k_tokens": 4.95e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.012,
+            "search_context_size_low": 0.012,
+            "search_context_size_medium": 0.012
+        },
         "litellm_provider": "bedrock_mantle",
         "max_input_tokens": 1050000,
         "max_output_tokens": 128000,
@@ -53226,6 +53246,11 @@
         "cache_read_input_token_cost_above_272k_tokens": 5.5e-07,
         "output_cost_per_token": 1.65e-05,
         "output_cost_per_token_above_272k_tokens": 2.475e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.012,
+            "search_context_size_low": 0.012,
+            "search_context_size_medium": 0.012
+        },
         "litellm_provider": "bedrock_mantle",
         "max_input_tokens": 1050000,
         "max_output_tokens": 128000,
diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py
index fd795ffcc96..0d75301a57a 100644
--- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py
+++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py
@@ -928,3 +928,121 @@ def test_web_search_gate_reads_server_side_tool_usage_details_without_citations(
         standard_built_in_tools_params=None,
     )
     assert cost == 3 * _DEFAULT_WEB_SEARCH_COST_PER_CALL
+
+
+_BEDROCK_MANTLE_WEB_SEARCH_MODELS = (
+    "bedrock_mantle/openai.gpt-5.6-sol",
+    "bedrock_mantle/openai.gpt-5.6-terra",
+    "bedrock_mantle/openai.gpt-5.6-luna",
+    "bedrock_mantle/openai.gpt-5.5",
+    "bedrock_mantle/openai.gpt-5.4",
+)
+
+_BEDROCK_MANTLE_WEB_SEARCH_RATE = 0.012
+
+
+def _bedrock_mantle_responses_with_web_search(model, actions, tool_usage=None):
+    from litellm.types.llms.openai import ResponsesAPIResponse
+
+    payload = {
+        "id": "resp_1",
+        "created_at": 1756900000,
+        "model": model.split("/", 1)[-1],
+        "object": "response",
+        "status": "completed",
+        "output": [
+            {"type": "web_search_call", "id": f"ws_{i}", "status": "completed", "action": action}
+            for i, action in enumerate(actions)
+        ],
+    }
+    return ResponsesAPIResponse.model_validate(
+        payload if tool_usage is None else {**payload, "tool_usage": tool_usage}
+    )
+
+
+def _bedrock_mantle_web_search_cost(model, response):
+    from litellm.types.utils import Usage
+
+    return StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
+        model=model,
+        response_object=response,
+        usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15),
+        custom_llm_provider="bedrock_mantle",
+        standard_built_in_tools_params=None,
+    )
+
+
+@pytest.mark.parametrize("model", _BEDROCK_MANTLE_WEB_SEARCH_MODELS)
+def test_bedrock_mantle_web_search_billed_per_query(local_model_cost_map, model):
+    """
+    Regression for LIT-6870: the bedrock_mantle GPT ids forward the web_search tool but carried no
+    search_context_cost_per_query, so every Bedrock web search (billed at $12 per 1,000 queries)
+    was costed at $0. Two reported queries must bill 2 x $0.012, whichever model prefix shape the
+    cost path resolves the deployment under.
+    """
+    pricing = litellm.get_model_info(model)["search_context_cost_per_query"]
+    assert pricing == {
+        "search_context_size_low": _BEDROCK_MANTLE_WEB_SEARCH_RATE,
+        "search_context_size_medium": _BEDROCK_MANTLE_WEB_SEARCH_RATE,
+        "search_context_size_high": _BEDROCK_MANTLE_WEB_SEARCH_RATE,
+    }
+
+    response = _bedrock_mantle_responses_with_web_search(
+        model,
+        actions=[{"type": "search", "query": "litellm"}, {"type": "search", "query": "bedrock web search"}],
+        tool_usage={"web_search": {"num_requests": 2}},
+    )
+    for cost_model in (model, model.split("/", 1)[1]):
+        cost = _bedrock_mantle_web_search_cost(cost_model, response)
+        assert cost == pytest.approx(2 * _BEDROCK_MANTLE_WEB_SEARCH_RATE), (
+            f"{cost_model} must bill 2 x ${_BEDROCK_MANTLE_WEB_SEARCH_RATE} for 2 web searches, got ${cost}"
+        )
+
+
+@pytest.mark.parametrize("num_requests", [1, 0])
+def test_web_search_call_count_prefers_provider_reported_num_requests(local_model_cost_map, num_requests):
+    """
+    Regression for LIT-6870: Bedrock bills one query per search and reports the billable count as
+    tool_usage.web_search.num_requests, while its open_page fetches share the web_search_call item
+    type. A search plus an open_page must bill the reported count, never the two items.
+    """
+    model = "bedrock_mantle/openai.gpt-5.6-sol"
+    response = _bedrock_mantle_responses_with_web_search(
+        model,
+        actions=[
+            {"type": "search", "query": "litellm"},
+            {"type": "open_page", "url": "https://docs.litellm.ai/"},
+        ],
+        tool_usage={"web_search": {"num_requests": num_requests}},
+    )
+
+    cost = _bedrock_mantle_web_search_cost(model, response)
+
+    assert cost == pytest.approx(num_requests * _BEDROCK_MANTLE_WEB_SEARCH_RATE), (
+        f"{num_requests} reported web search requests must bill {num_requests} x "
+        f"${_BEDROCK_MANTLE_WEB_SEARCH_RATE}, got ${cost}"
+    )
+
+
+@pytest.mark.parametrize(
+    "tool_usage",
+    [None, {}, {"web_search": None}, {"web_search": {"num_requests": "many"}}, {"web_search": {"num_requests": -1}}],
+)
+def test_web_search_call_count_falls_back_to_items_without_reported_count(local_model_cost_map, tool_usage):
+    """
+    Without a usable reported count (no tool_usage, no web_search block, or a malformed one) the
+    per-call path must keep counting web_search_call items instead of raising or billing zero.
+    """
+    model = "bedrock_mantle/openai.gpt-5.6-sol"
+    response = _bedrock_mantle_responses_with_web_search(
+        model,
+        actions=[{"type": "search", "query": "litellm"}, {"type": "search", "query": "bedrock web search"}],
+        tool_usage=tool_usage,
+    )
+
+    cost = _bedrock_mantle_web_search_cost(model, response)
+
+    assert cost == pytest.approx(2 * _BEDROCK_MANTLE_WEB_SEARCH_RATE), (
+        f"2 web_search_call items with tool_usage={tool_usage!r} must bill 2 x "
+        f"${_BEDROCK_MANTLE_WEB_SEARCH_RATE}, got ${cost}"
+    )

From 339da4183d998b1ff2db54bc0468fa994a620095 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Thu, 3 Sep 2026 12:34:54 -0700
Subject: [PATCH 36/42] test(cost): type the web search cost helpers and cover
 OpenAI-shaped tool_usage

---
 .../test_tool_call_cost_tracking.py           | 59 ++++++++++---------
 1 file changed, 32 insertions(+), 27 deletions(-)

diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py
index 0d75301a57a..6cc3dcceebc 100644
--- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py
+++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py
@@ -1,4 +1,5 @@
 import os
+from collections.abc import Mapping, Sequence
 
 import pytest
 
@@ -6,7 +7,7 @@ import litellm
 from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
     StandardBuiltInToolCostTracking,
 )
-from litellm.types.llms.openai import FileSearchTool, WebSearchOptions
+from litellm.types.llms.openai import FileSearchTool, ResponsesAPIResponse, WebSearchOptions
 from litellm.types.utils import ModelResponse, StandardBuiltInToolsParams
 
 
@@ -941,9 +942,9 @@ _BEDROCK_MANTLE_WEB_SEARCH_MODELS = (
 _BEDROCK_MANTLE_WEB_SEARCH_RATE = 0.012
 
 
-def _bedrock_mantle_responses_with_web_search(model, actions, tool_usage=None):
-    from litellm.types.llms.openai import ResponsesAPIResponse
-
+def _responses_with_web_search(
+    model: str, actions: Sequence[Mapping[str, str]], tool_usage: Mapping[str, object] | None = None
+) -> ResponsesAPIResponse:
     payload = {
         "id": "resp_1",
         "created_at": 1756900000,
@@ -960,26 +961,21 @@ def _bedrock_mantle_responses_with_web_search(model, actions, tool_usage=None):
     )
 
 
-def _bedrock_mantle_web_search_cost(model, response):
+def _web_search_cost(model: str, response: ResponsesAPIResponse, custom_llm_provider: str) -> float:
     from litellm.types.utils import Usage
 
     return StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
         model=model,
         response_object=response,
         usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15),
-        custom_llm_provider="bedrock_mantle",
+        custom_llm_provider=custom_llm_provider,
         standard_built_in_tools_params=None,
     )
 
 
 @pytest.mark.parametrize("model", _BEDROCK_MANTLE_WEB_SEARCH_MODELS)
 def test_bedrock_mantle_web_search_billed_per_query(local_model_cost_map, model):
-    """
-    Regression for LIT-6870: the bedrock_mantle GPT ids forward the web_search tool but carried no
-    search_context_cost_per_query, so every Bedrock web search (billed at $12 per 1,000 queries)
-    was costed at $0. Two reported queries must bill 2 x $0.012, whichever model prefix shape the
-    cost path resolves the deployment under.
-    """
+    """Two Bedrock-reported web searches bill 2 x $0.012 under the prefixed and the bare model id alike."""
     pricing = litellm.get_model_info(model)["search_context_cost_per_query"]
     assert pricing == {
         "search_context_size_low": _BEDROCK_MANTLE_WEB_SEARCH_RATE,
@@ -987,13 +983,13 @@ def test_bedrock_mantle_web_search_billed_per_query(local_model_cost_map, model)
         "search_context_size_high": _BEDROCK_MANTLE_WEB_SEARCH_RATE,
     }
 
-    response = _bedrock_mantle_responses_with_web_search(
+    response = _responses_with_web_search(
         model,
         actions=[{"type": "search", "query": "litellm"}, {"type": "search", "query": "bedrock web search"}],
         tool_usage={"web_search": {"num_requests": 2}},
     )
     for cost_model in (model, model.split("/", 1)[1]):
-        cost = _bedrock_mantle_web_search_cost(cost_model, response)
+        cost = _web_search_cost(cost_model, response, "bedrock_mantle")
         assert cost == pytest.approx(2 * _BEDROCK_MANTLE_WEB_SEARCH_RATE), (
             f"{cost_model} must bill 2 x ${_BEDROCK_MANTLE_WEB_SEARCH_RATE} for 2 web searches, got ${cost}"
         )
@@ -1001,13 +997,9 @@ def test_bedrock_mantle_web_search_billed_per_query(local_model_cost_map, model)
 
 @pytest.mark.parametrize("num_requests", [1, 0])
 def test_web_search_call_count_prefers_provider_reported_num_requests(local_model_cost_map, num_requests):
-    """
-    Regression for LIT-6870: Bedrock bills one query per search and reports the billable count as
-    tool_usage.web_search.num_requests, while its open_page fetches share the web_search_call item
-    type. A search plus an open_page must bill the reported count, never the two items.
-    """
+    """A search plus an open_page fetch bills tool_usage.web_search.num_requests, never the two items."""
     model = "bedrock_mantle/openai.gpt-5.6-sol"
-    response = _bedrock_mantle_responses_with_web_search(
+    response = _responses_with_web_search(
         model,
         actions=[
             {"type": "search", "query": "litellm"},
@@ -1016,7 +1008,7 @@ def test_web_search_call_count_prefers_provider_reported_num_requests(local_mode
         tool_usage={"web_search": {"num_requests": num_requests}},
     )
 
-    cost = _bedrock_mantle_web_search_cost(model, response)
+    cost = _web_search_cost(model, response, "bedrock_mantle")
 
     assert cost == pytest.approx(num_requests * _BEDROCK_MANTLE_WEB_SEARCH_RATE), (
         f"{num_requests} reported web search requests must bill {num_requests} x "
@@ -1029,20 +1021,33 @@ def test_web_search_call_count_prefers_provider_reported_num_requests(local_mode
     [None, {}, {"web_search": None}, {"web_search": {"num_requests": "many"}}, {"web_search": {"num_requests": -1}}],
 )
 def test_web_search_call_count_falls_back_to_items_without_reported_count(local_model_cost_map, tool_usage):
-    """
-    Without a usable reported count (no tool_usage, no web_search block, or a malformed one) the
-    per-call path must keep counting web_search_call items instead of raising or billing zero.
-    """
+    """Without a usable reported count the per-call path keeps counting web_search_call items."""
     model = "bedrock_mantle/openai.gpt-5.6-sol"
-    response = _bedrock_mantle_responses_with_web_search(
+    response = _responses_with_web_search(
         model,
         actions=[{"type": "search", "query": "litellm"}, {"type": "search", "query": "bedrock web search"}],
         tool_usage=tool_usage,
     )
 
-    cost = _bedrock_mantle_web_search_cost(model, response)
+    cost = _web_search_cost(model, response, "bedrock_mantle")
 
     assert cost == pytest.approx(2 * _BEDROCK_MANTLE_WEB_SEARCH_RATE), (
         f"2 web_search_call items with tool_usage={tool_usage!r} must bill 2 x "
         f"${_BEDROCK_MANTLE_WEB_SEARCH_RATE}, got ${cost}"
     )
+
+
+def test_web_search_call_count_reads_reported_count_beside_other_tool_usage_entries(local_model_cost_map):
+    """OpenAI reports web_search.num_requests next to other tool entries, which must not disable the reported count."""
+    response = _responses_with_web_search(
+        "gpt-5.6",
+        actions=[{"type": "search", "query": "S&P 500 close"}, {"type": "open_page", "url": "https://example.com/"}],
+        tool_usage={
+            "image_gen": {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0},
+            "web_search": {"num_requests": 1},
+        },
+    )
+
+    cost = _web_search_cost("gpt-5.6", response, "openai")
+
+    assert cost == pytest.approx(0.01), f"1 reported OpenAI web search must bill 1 x $0.01, not the 2 items, got ${cost}"

From 1e75668a259ee9b6c7d77abd17ef51df70250b1c Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Thu, 3 Sep 2026 12:37:17 -0700
Subject: [PATCH 37/42] fix(openai): default stream usage on PrivateLink and
 regional api.openai.com hosts

---
 litellm/llms/openai/common_utils.py           |  9 ++++
 litellm/llms/openai/openai.py                 |  8 ++-
 tests/test_litellm/llms/openai/test_openai.py | 52 +++++++++++++++++++
 .../llms/openai/test_openai_common_utils.py   | 21 +++++++-
 4 files changed, 84 insertions(+), 6 deletions(-)
 create mode 100644 tests/test_litellm/llms/openai/test_openai.py

diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py
index bcd4ea43243..2db6d78a218 100644
--- a/litellm/llms/openai/common_utils.py
+++ b/litellm/llms/openai/common_utils.py
@@ -11,6 +11,7 @@ import time
 import uuid
 from collections.abc import AsyncIterator, Iterator, Mapping
 from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, Optional
+from urllib.parse import urlsplit
 
 import httpx
 import openai
@@ -43,6 +44,14 @@ _OPENAI_INIT_PARAMS: Final[tuple[str, ...]] = _get_client_init_params(OpenAI)
 _AZURE_OPENAI_INIT_PARAMS: Final[tuple[str, ...]] = _get_client_init_params(AzureOpenAI)
 
 
+_OPENAI_API_HOST: Final[str] = "api.openai.com"
+
+
+def is_openai_backed_api_base(api_base: str) -> bool:
+    hostname: Final = urlsplit(api_base).hostname
+    return hostname is not None and (hostname == _OPENAI_API_HOST or hostname.endswith(f".{_OPENAI_API_HOST}"))
+
+
 class OpenAIError(BaseLLMException):
     def __init__(
         self,
diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py
index 1cfc6e06ee9..edc8d64d9c2 100644
--- a/litellm/llms/openai/openai.py
+++ b/litellm/llms/openai/openai.py
@@ -2,7 +2,6 @@ import time
 import types
 from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Iterator, Mapping
 from typing import TYPE_CHECKING, Any, Final, Literal, Optional, cast
-from urllib.parse import urlparse
 
 import httpx
 
@@ -55,6 +54,7 @@ from .common_utils import (
     OpenAIError,
     build_output_token_limit_response,
     drop_params_from_unprocessable_entity_error,
+    is_openai_backed_api_base,
     is_output_token_limit_error,
 )
 from .workload_identity import resolve_openai_workload_identity_config
@@ -1190,10 +1190,8 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         """
         if stream_options is not None:
             return {"stream_options": stream_options}
-        else:
-            # by default litellm will include usage for openai endpoints
-            if api_base is None or urlparse(api_base).hostname == "api.openai.com":
-                return {"stream_options": {"include_usage": True}}
+        if api_base is None or is_openai_backed_api_base(api_base):
+            return {"stream_options": {"include_usage": True}}
         return {}
 
     # Embedding
diff --git a/tests/test_litellm/llms/openai/test_openai.py b/tests/test_litellm/llms/openai/test_openai.py
new file mode 100644
index 00000000000..136b837f191
--- /dev/null
+++ b/tests/test_litellm/llms/openai/test_openai.py
@@ -0,0 +1,52 @@
+import pytest
+
+from litellm.llms.openai.openai import OpenAIChatCompletion
+
+
+@pytest.mark.parametrize(
+    "api_base",
+    [
+        None,
+        "https://api.openai.com/v1",
+        "https://api.openai.com:443/v1",
+        "https://southcentralus.privatelink.api.openai.com/v1",
+        "https://eu.api.openai.com/v1",
+        "https://us.api.openai.com/v1",
+        "HTTPS://API.OPENAI.COM/v1/",
+    ],
+)
+def test_get_stream_options_defaults_include_usage_on_every_openai_backed_host(api_base):
+    """
+    PrivateLink and regional hostnames reach the real OpenAI backend, so a stream with no caller
+    stream_options must ask for the usage chunk exactly as the default base does. Regression guard
+    for LIT-6875: spend for those deployments fell back to local token counting.
+    """
+    assert OpenAIChatCompletion().get_stream_options(stream_options=None, api_base=api_base) == {
+        "stream_options": {"include_usage": True}
+    }
+
+
+@pytest.mark.parametrize(
+    "api_base",
+    [
+        "https://my-gateway.example/v1",
+        "https://api.openai.com.evil.example/v1",
+        "https://notapi.openai.com/v1",
+        "https://gateway.example/v1?upstream=api.openai.com",
+        "https://openai.internal.example/api.openai.com/v1",
+    ],
+)
+def test_get_stream_options_leaves_foreign_hosts_without_a_usage_default(api_base):
+    """Only the host decides: an OpenAI-compatible backend elsewhere may not support stream_options at all."""
+    assert OpenAIChatCompletion().get_stream_options(stream_options=None, api_base=api_base) == {}
+
+
+@pytest.mark.parametrize(
+    "api_base",
+    ["https://southcentralus.privatelink.api.openai.com/v1", "https://my-gateway.example/v1"],
+)
+def test_get_stream_options_passes_caller_stream_options_through_on_any_host(api_base):
+    caller_options = {"include_usage": False}
+    assert OpenAIChatCompletion().get_stream_options(stream_options=caller_options, api_base=api_base) == {
+        "stream_options": caller_options
+    }
diff --git a/tests/test_litellm/llms/openai/test_openai_common_utils.py b/tests/test_litellm/llms/openai/test_openai_common_utils.py
index 3ae29e411e8..d3c21c5bd5a 100644
--- a/tests/test_litellm/llms/openai/test_openai_common_utils.py
+++ b/tests/test_litellm/llms/openai/test_openai_common_utils.py
@@ -7,7 +7,7 @@ import pytest
 
 import litellm
 from litellm.litellm_core_utils.token_counter import token_counter
-from litellm.llms.openai.common_utils import BaseOpenAILLM
+from litellm.llms.openai.common_utils import BaseOpenAILLM, is_openai_backed_api_base
 
 # Test parameters for different API functions
 API_FUNCTION_PARAMS = [
@@ -392,3 +392,22 @@ async def test_async_genuine_bad_request_still_raises(provider, stream):
 
     with pytest.raises(litellm.BadRequestError):
         await _call_and_drain()
+
+
+@pytest.mark.parametrize(
+    ("api_base", "expected"),
+    [
+        ("https://api.openai.com/v1", True),
+        ("https://api.openai.com:443/v1/", True),
+        ("https://southcentralus.privatelink.api.openai.com/v1", True),
+        ("https://eu.api.openai.com/v1", True),
+        ("HTTPS://API.OPENAI.COM/v1", True),
+        ("https://my-gateway.example/v1", False),
+        ("https://api.openai.com.evil.example/v1", False),
+        ("https://notapi.openai.com/v1", False),
+        ("https://gateway.example/v1?upstream=api.openai.com", False),
+        ("not a url", False),
+    ],
+)
+def test_is_openai_backed_api_base_decides_by_hostname_only(api_base, expected):
+    assert is_openai_backed_api_base(api_base) is expected

From 425e3069b93ec005e3186f5d32fa21ef70effe17 Mon Sep 17 00:00:00 2001
From: moe-berri <moe@berri.ai>
Date: Thu, 3 Sep 2026 12:40:22 -0700
Subject: [PATCH 38/42] fix(proxy): expose configured model mode

---
 litellm/proxy/utils.py                       |  3 ++
 litellm/router.py                            | 11 ++++++
 litellm/types/proxy/model_listing.py         |  4 +-
 tests/test_litellm/proxy/test_proxy_utils.py | 41 ++++++++++++++++++++
 4 files changed, 57 insertions(+), 2 deletions(-)

diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index cab2bd6d9db..24a65457928 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -7531,6 +7531,9 @@ def create_model_info_response(
             max_input_tokens = configured_input
         if configured_output is not None:
             max_output_tokens = configured_output
+        configured_mode: Final = llm_router.get_configured_mode(model_id)
+        if isinstance(configured_mode, str):
+            base["mode"] = configured_mode
 
     if max_input_tokens is not None:
         base["max_input_tokens"] = max_input_tokens
diff --git a/litellm/router.py b/litellm/router.py
index 2b8b342d253..cfb080a24a0 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -9981,6 +9981,17 @@ class Router:
             coerce_token_limit(model_info.get("max_output_tokens")),
         )
 
+    def get_configured_mode(self, model_name: str) -> "str | None":
+        """Return the mode explicitly configured for a concrete deployment."""
+        deployment: Final = self.get_deployment_by_model_group_name(model_group_name=model_name)
+        if deployment is None:
+            return None
+
+        mode: Final = deployment.model_info.get("mode")
+        if isinstance(mode, str) and mode.strip():
+            return mode
+        return None
+
     def get_configured_display_name(self, model_name: str) -> "str | None":
         """
         Return the display_name explicitly configured in a concrete deployment's
diff --git a/litellm/types/proxy/model_listing.py b/litellm/types/proxy/model_listing.py
index b59c0f2cf19..24cfa85eee4 100644
--- a/litellm/types/proxy/model_listing.py
+++ b/litellm/types/proxy/model_listing.py
@@ -11,8 +11,8 @@ class ModelInfoMetadata(TypedDict):
 
 class ModelInfoResponse(TypedDict):
     """OpenAI-compatible model object. `mode`, `max_input_tokens`, and
-    `max_output_tokens` are attached when the cost map knows them; `metadata`
-    is present only when the endpoint is called with include_metadata=true.
+    `max_output_tokens` are attached when the cost map or deployment config
+    knows them; `metadata` is present only with include_metadata=true.
     """
 
     id: str
diff --git a/tests/test_litellm/proxy/test_proxy_utils.py b/tests/test_litellm/proxy/test_proxy_utils.py
index dcaad968663..f16c6c937d0 100644
--- a/tests/test_litellm/proxy/test_proxy_utils.py
+++ b/tests/test_litellm/proxy/test_proxy_utils.py
@@ -942,6 +942,47 @@ def test_create_model_info_response_uses_deployment_limits_when_not_in_cost_map(
     assert response["max_output_tokens"] == 8000
 
 
+def test_create_model_info_response_uses_deployment_mode_for_auto_router():
+    router = litellm.Router(
+        model_list=[
+            {
+                "model_name": "claude-sonnet",
+                "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "test-key"},
+            },
+            {
+                "model_name": "claude-auto",
+                "litellm_params": {
+                    "model": "auto_router/complexity_router",
+                    "complexity_router_config": {
+                        "tiers": {
+                            "SIMPLE": "claude-sonnet",
+                            "MEDIUM": "claude-sonnet",
+                            "COMPLEX": "claude-sonnet",
+                        }
+                    },
+                    "complexity_router_default_model": "claude-sonnet",
+                },
+                "model_info": {
+                    "mode": "chat",
+                    "max_input_tokens": 1_000_000,
+                    "max_output_tokens": 128_000,
+                },
+            },
+        ]
+    )
+
+    response = create_model_info_response(
+        model_id="claude-auto",
+        provider="openai",
+        llm_router=router,
+        get_model_info=_raise_unmapped,
+    )
+
+    assert response["mode"] == "chat"
+    assert response["max_input_tokens"] == 1_000_000
+    assert response["max_output_tokens"] == 128_000
+
+
 def test_create_model_info_response_deployment_limits_override_cost_map():
     router = MagicMock()
     router.get_configured_token_limits.return_value = (200000, None)

From 897fba08c8ddb6dba99288b040ae5c7cad8a5757 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Thu, 3 Sep 2026 11:08:03 -0700
Subject: [PATCH 39/42] feat(models): add gpt-6-astra pricing and metadata

Adds the OpenAI gpt-6-astra entry to both price files with standard, flex, priority (fast mode), batch, and above-272K long-context rates, and regression tests covering each tier and the batch rates.
---
 ...odel_prices_and_context_window_backup.json | 68 +++++++++++++++++++
 model_prices_and_context_window.json          | 68 +++++++++++++++++++
 .../llm_cost_calc/test_llm_cost_calc_utils.py | 48 +++++++++++++
 tests/test_litellm/test_cost_calculator.py    | 14 ++++
 ...penai_service_tier_long_context_pricing.py |  7 ++
 5 files changed, 205 insertions(+)

diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 9b358a1cedd..04a67200ced 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -29277,6 +29277,74 @@
         "supports_xhigh_reasoning_effort": true,
         "supports_minimal_reasoning_effort": true
     },
+    "gpt-6-astra": {
+        "cache_creation_input_token_cost": 1.25e-05,
+        "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05,
+        "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05,
+        "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05,
+        "cache_creation_input_token_cost_flex": 6.25e-06,
+        "cache_creation_input_token_cost_priority": 2.5e-05,
+        "cache_read_input_token_cost": 1e-06,
+        "cache_read_input_token_cost_above_272k_tokens": 2e-06,
+        "cache_read_input_token_cost_above_272k_tokens_flex": 1e-06,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 4e-06,
+        "cache_read_input_token_cost_flex": 5e-07,
+        "cache_read_input_token_cost_priority": 2e-06,
+        "input_cost_per_token": 1e-05,
+        "input_cost_per_token_above_272k_tokens": 2e-05,
+        "input_cost_per_token_above_272k_tokens_flex": 1e-05,
+        "input_cost_per_token_above_272k_tokens_priority": 4e-05,
+        "input_cost_per_token_batches": 5e-06,
+        "input_cost_per_token_flex": 5e-06,
+        "input_cost_per_token_priority": 2e-05,
+        "litellm_provider": "openai",
+        "max_input_tokens": 922000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-05,
+        "output_cost_per_token_above_272k_tokens": 7.5e-05,
+        "output_cost_per_token_above_272k_tokens_flex": 3.75e-05,
+        "output_cost_per_token_above_272k_tokens_priority": 0.00015,
+        "output_cost_per_token_batches": 2.5e-05,
+        "output_cost_per_token_flex": 2.5e-05,
+        "output_cost_per_token_priority": 0.0001,
+        "regional_processing_uplift_multiplier_eu": 1.1,
+        "regional_processing_uplift_multiplier_us": 1.1,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/batch",
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_minimal_reasoning_effort": false,
+        "supports_native_streaming": true,
+        "supports_none_reasoning_effort": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_cache_breakpoint": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "supports_xhigh_reasoning_effort": true
+    },
     "gpt-5.6": {
         "cache_creation_input_token_cost": 5e-06,
         "cache_creation_input_token_cost_above_272k_tokens": 1e-05,
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index 9b358a1cedd..04a67200ced 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -29277,6 +29277,74 @@
         "supports_xhigh_reasoning_effort": true,
         "supports_minimal_reasoning_effort": true
     },
+    "gpt-6-astra": {
+        "cache_creation_input_token_cost": 1.25e-05,
+        "cache_creation_input_token_cost_above_272k_tokens": 2.5e-05,
+        "cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-05,
+        "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05,
+        "cache_creation_input_token_cost_flex": 6.25e-06,
+        "cache_creation_input_token_cost_priority": 2.5e-05,
+        "cache_read_input_token_cost": 1e-06,
+        "cache_read_input_token_cost_above_272k_tokens": 2e-06,
+        "cache_read_input_token_cost_above_272k_tokens_flex": 1e-06,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 4e-06,
+        "cache_read_input_token_cost_flex": 5e-07,
+        "cache_read_input_token_cost_priority": 2e-06,
+        "input_cost_per_token": 1e-05,
+        "input_cost_per_token_above_272k_tokens": 2e-05,
+        "input_cost_per_token_above_272k_tokens_flex": 1e-05,
+        "input_cost_per_token_above_272k_tokens_priority": 4e-05,
+        "input_cost_per_token_batches": 5e-06,
+        "input_cost_per_token_flex": 5e-06,
+        "input_cost_per_token_priority": 2e-05,
+        "litellm_provider": "openai",
+        "max_input_tokens": 922000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-05,
+        "output_cost_per_token_above_272k_tokens": 7.5e-05,
+        "output_cost_per_token_above_272k_tokens_flex": 3.75e-05,
+        "output_cost_per_token_above_272k_tokens_priority": 0.00015,
+        "output_cost_per_token_batches": 2.5e-05,
+        "output_cost_per_token_flex": 2.5e-05,
+        "output_cost_per_token_priority": 0.0001,
+        "regional_processing_uplift_multiplier_eu": 1.1,
+        "regional_processing_uplift_multiplier_us": 1.1,
+        "search_context_cost_per_query": {
+            "search_context_size_high": 0.01,
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01
+        },
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/batch",
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_minimal_reasoning_effort": false,
+        "supports_native_streaming": true,
+        "supports_none_reasoning_effort": true,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_cache_breakpoint": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "supports_xhigh_reasoning_effort": true
+    },
     "gpt-5.6": {
         "cache_creation_input_token_cost": 5e-06,
         "cache_creation_input_token_cost_above_272k_tokens": 1e-05,
diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
index b7f0ca1efe1..0b6832d4bef 100644
--- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
+++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py
@@ -1662,6 +1662,54 @@ def test_generic_cost_per_token_gpt56_cyber(
     assert completion_cost == pytest.approx(completion_tokens * output_rate)
 
 
+@pytest.mark.parametrize(
+    "service_tier,tier_multiplier",
+    [(None, 1.0), ("flex", 0.5), ("priority", 2.0), ("fast", 2.0)],
+)
+@pytest.mark.parametrize(
+    "prompt_tokens,input_side_multiplier,output_multiplier",
+    [(100000, 1.0, 1.0), (300000, 2.0, 1.5)],
+)
+def test_generic_cost_per_token_gpt_6_astra_price_sheet(
+    _local_model_cost_map,
+    service_tier,
+    tier_multiplier,
+    prompt_tokens,
+    input_side_multiplier,
+    output_multiplier,
+):
+    """gpt-6-astra launch price sheet: $10 input, $1 cache read, $12.50 cache write, $50 output per 1M tokens.
+
+    Above 272K prompt tokens the input-side rates double and the output rate is 1.5x on the whole
+    request. Flex is half the applicable rate and fast mode, billed as priority, is double it.
+    """
+    cached_tokens = 50000
+    cache_write_tokens = 40000
+    text_tokens = prompt_tokens - cached_tokens - cache_write_tokens
+    completion_tokens = 1000
+    usage = Usage(
+        prompt_tokens=prompt_tokens,
+        completion_tokens=completion_tokens,
+        total_tokens=prompt_tokens + completion_tokens,
+        prompt_tokens_details=PromptTokensDetailsWrapper(
+            cached_tokens=cached_tokens, cache_write_tokens=cache_write_tokens
+        ),
+    )
+
+    prompt_cost, completion_cost = generic_cost_per_token(
+        model="gpt-6-astra",
+        usage=usage,
+        custom_llm_provider="openai",
+        service_tier=service_tier,
+    )
+
+    input_side = tier_multiplier * input_side_multiplier
+    assert prompt_cost == pytest.approx(
+        input_side * (text_tokens * 1e-5 + cached_tokens * 1e-6 + cache_write_tokens * 1.25e-5)
+    )
+    assert completion_cost == pytest.approx(tier_multiplier * output_multiplier * completion_tokens * 5e-5)
+
+
 @pytest.mark.parametrize(
     "model,input_cost,output_cost,cache_read_cost",
     [
diff --git a/tests/test_litellm/test_cost_calculator.py b/tests/test_litellm/test_cost_calculator.py
index 7c2174018e8..6dc3b2790c9 100644
--- a/tests/test_litellm/test_cost_calculator.py
+++ b/tests/test_litellm/test_cost_calculator.py
@@ -4473,3 +4473,17 @@ def test_explicit_pricing_precedes_private_provider_response_model(
     )
 
     assert selected == expected
+
+
+def test_batch_cost_calculator_gpt_6_astra_bills_half_the_standard_rate(_local_model_cost_map):
+    """gpt-6-astra batch pricing is 50% off the standard $10 input and $50 output rates per 1M tokens."""
+    from litellm.cost_calculator import batch_cost_calculator
+
+    usage = Usage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500)
+
+    prompt_cost, completion_cost = batch_cost_calculator(
+        usage=usage, model="gpt-6-astra", custom_llm_provider="openai"
+    )
+
+    assert prompt_cost == pytest.approx(1000 * 5e-6)
+    assert completion_cost == pytest.approx(500 * 2.5e-5)
diff --git a/tests/test_litellm/test_openai_service_tier_long_context_pricing.py b/tests/test_litellm/test_openai_service_tier_long_context_pricing.py
index c0860a5b55f..70e9c2720b8 100644
--- a/tests/test_litellm/test_openai_service_tier_long_context_pricing.py
+++ b/tests/test_litellm/test_openai_service_tier_long_context_pricing.py
@@ -52,6 +52,12 @@ PRIORITY_LONG_CONTEXT = {
         "cache_read_input_token_cost_above_272k_tokens_priority": 8e-08,
         "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-06,
     },
+    "gpt-6-astra": {
+        "input_cost_per_token_above_272k_tokens_priority": 4e-05,
+        "output_cost_per_token_above_272k_tokens_priority": 0.00015,
+        "cache_read_input_token_cost_above_272k_tokens_priority": 4e-06,
+        "cache_creation_input_token_cost_above_272k_tokens_priority": 5e-05,
+    },
 }
 
 EXPECTED = {**FLEX_LONG_CONTEXT, **PRIORITY_LONG_CONTEXT}
@@ -114,6 +120,7 @@ TIERED_COST_CASES = [
     ("gpt-5.6-sol", "priority", 1.6e-05, 6e-05),
     ("gpt-5.6-terra", "priority", 8e-06, 3.6e-05),
     ("gpt-5.6-luna", "priority", 8e-07, 3.6e-06),
+    ("gpt-6-astra", "priority", 4e-05, 0.00015),
 ]
 
 

From 4991d0bf3e58fc1022d97ce8baf1a517c656d5a2 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Thu, 3 Sep 2026 12:47:13 -0700
Subject: [PATCH 40/42] fix(models): match gpt-6-astra reasoning effort levels
 to OpenAI docs

OpenAI documents low, medium, high, xhigh, and max for gpt-6-astra, with no none level, so the entry stops advertising none and starts advertising max.
---
 .../model_prices_and_context_window_backup.json |  3 ++-
 model_prices_and_context_window.json            |  3 ++-
 .../test_reasoning_effort_capability.py         | 17 +++++++++++++++++
 3 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 04a67200ced..385cf22d06c 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -29330,9 +29330,10 @@
         ],
         "supports_computer_use": true,
         "supports_function_calling": true,
+        "supports_max_reasoning_effort": true,
         "supports_minimal_reasoning_effort": false,
         "supports_native_streaming": true,
-        "supports_none_reasoning_effort": true,
+        "supports_none_reasoning_effort": false,
         "supports_parallel_function_calling": true,
         "supports_pdf_input": true,
         "supports_prompt_cache_breakpoint": true,
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index 04a67200ced..385cf22d06c 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -29330,9 +29330,10 @@
         ],
         "supports_computer_use": true,
         "supports_function_calling": true,
+        "supports_max_reasoning_effort": true,
         "supports_minimal_reasoning_effort": false,
         "supports_native_streaming": true,
-        "supports_none_reasoning_effort": true,
+        "supports_none_reasoning_effort": false,
         "supports_parallel_function_calling": true,
         "supports_pdf_input": true,
         "supports_prompt_cache_breakpoint": true,
diff --git a/tests/test_litellm/router_utils/test_reasoning_effort_capability.py b/tests/test_litellm/router_utils/test_reasoning_effort_capability.py
index a0dbf3b6637..7b2e45ab3ed 100644
--- a/tests/test_litellm/router_utils/test_reasoning_effort_capability.py
+++ b/tests/test_litellm/router_utils/test_reasoning_effort_capability.py
@@ -371,3 +371,20 @@ class TestKimiK3AdvertisesItsDocumentedLevels:
             "low",
             "high",
         )
+
+
+class TestGpt6AstraAdvertisesItsDocumentedLevels:
+    def test_the_entry_advertises_low_through_max_without_none(self, local_model_cost_map):
+        """OpenAI documents low, medium, high, xhigh and max for gpt-6-astra. Unlike gpt-5.6-sol it
+        does not take none, so a group must not offer none and must offer max."""
+        from litellm.utils import _get_model_info_helper
+
+        model_info = dict(_get_model_info_helper(model="gpt-6-astra", custom_llm_provider="openai"))
+
+        assert resolve_supported_reasoning_efforts(model_info, deployment_is_mapped=True) == (
+            "low",
+            "medium",
+            "high",
+            "xhigh",
+            "max",
+        )

From f40f14ae39b4452cb55f117ab3b06949bb3135a7 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Thu, 3 Sep 2026 13:09:46 -0700
Subject: [PATCH 41/42] fix(tests): fold the local price map into the provider
 model sets

CI unit shards load the price map from main at import, so a model that only
exists on the branch never reaches open_ai_chat_completion_models and
cost_per_token cannot infer its provider. Refresh the sets after swapping in
the local map so the tier pricing cases resolve gpt-6-astra before merge
---
 .../test_openai_service_tier_long_context_pricing.py             | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_litellm/test_openai_service_tier_long_context_pricing.py b/tests/test_litellm/test_openai_service_tier_long_context_pricing.py
index 70e9c2720b8..bdb2dc26813 100644
--- a/tests/test_litellm/test_openai_service_tier_long_context_pricing.py
+++ b/tests/test_litellm/test_openai_service_tier_long_context_pricing.py
@@ -69,6 +69,7 @@ NO_PUBLISHED_PRIORITY_LONG_CONTEXT = ("gpt-5.4", "gpt-5.5")
 def _local_model_cost_map(monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
     monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
+    litellm.add_known_models()
 
 
 @lru_cache(maxsize=2)

From 1f20b381151d1c01d38edb56743dee4b771fa25a Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Thu, 3 Sep 2026 13:17:48 -0700
Subject: [PATCH 42/42] fix(vector_stores): only list vector stores the caller
 was granted (#39612)

* fix(vector_stores): only list vector stores the caller was granted

/vector_store/list returned every managed vector store with no team_id to any
key, and let a dashboard session see stores created from the dashboard because
every session shares the litellm-dashboard team id. Non-admin listings now show
a store only when the key or one of the caller's real teams is allowlisted for
it via object_permission.vector_stores, or the team owns it

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(vector_stores): keep a dashboard session key's own grants when the user has no teams

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../management_endpoints.py                   |  14 ++-
 litellm/proxy/vector_store_endpoints/utils.py |  78 +++++++++++-
 .../test_vector_store_access_control.py       | 116 +++++++++++++++++-
 3 files changed, 192 insertions(+), 16 deletions(-)

diff --git a/litellm/proxy/vector_store_endpoints/management_endpoints.py b/litellm/proxy/vector_store_endpoints/management_endpoints.py
index c928398a87f..fe4732c6492 100644
--- a/litellm/proxy/vector_store_endpoints/management_endpoints.py
+++ b/litellm/proxy/vector_store_endpoints/management_endpoints.py
@@ -30,7 +30,10 @@ from litellm.proxy._types import (
 )
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.common_utils.rbac_utils import check_feature_access_for_user
-from litellm.proxy.vector_store_endpoints.utils import can_user_access_vector_store
+from litellm.proxy.vector_store_endpoints.utils import (
+    can_user_access_vector_store,
+    filter_listable_vector_stores,
+)
 from litellm.repositories.prisma_protocols import TableActions
 from litellm.repositories.table_repositories import ManagedVectorStoresRepository
 from litellm.types.vector_stores import (
@@ -390,11 +393,10 @@ async def list_vector_stores(
 
         # Filter vector stores based on access control
         accessible_vector_stores: Final = []
-        for vs in vector_store_map.values():
-            if await _check_vector_store_access(vs, user_api_key_dict):
-                redacted = LiteLLM_ManagedVectorStore(**vs)
-                redacted["litellm_params"] = _redact_sensitive_litellm_params(vs.get("litellm_params"))
-                accessible_vector_stores.append(redacted)
+        for vs in await filter_listable_vector_stores(vector_store_map.values(), user_api_key_dict):
+            redacted = LiteLLM_ManagedVectorStore(**vs)
+            redacted["litellm_params"] = _redact_sensitive_litellm_params(vs.get("litellm_params"))
+            accessible_vector_stores.append(redacted)
 
         total_count: Final = len(accessible_vector_stores)
         total_pages: Final = (total_count + page_size - 1) // page_size
diff --git a/litellm/proxy/vector_store_endpoints/utils.py b/litellm/proxy/vector_store_endpoints/utils.py
index 93f1510bf22..6e94a5a88ac 100644
--- a/litellm/proxy/vector_store_endpoints/utils.py
+++ b/litellm/proxy/vector_store_endpoints/utils.py
@@ -1,11 +1,17 @@
 import json
 import re
+from collections.abc import Iterable
+from types import MappingProxyType
 from typing import Any, Final, Literal
 
 from fastapi import HTTPException, Request
 
 import litellm
 from litellm._logging import verbose_proxy_logger
+from litellm.proxy._experimental.mcp_server.ui_session_utils import (
+    is_ui_session_credential,
+    resolve_ui_session_team_ids,
+)
 from litellm.proxy._types import (
     LiteLLM_ObjectPermissionTable,
     LitellmUserRoles,
@@ -160,10 +166,16 @@ async def can_user_access_vector_store(
     if _is_proxy_admin(user_api_key_dict):
         return True
 
-    vector_store_team_id: Final = vector_store.get("team_id")
-    if vector_store_team_id is None:
+    if vector_store.get("team_id") is None:
         return True
 
+    return await _is_vector_store_granted(vector_store, user_api_key_dict)
+
+
+async def _is_vector_store_granted(
+    vector_store: LiteLLM_ManagedVectorStore,
+    user_api_key_dict: UserAPIKeyAuth,
+) -> bool:
     vector_store_id: Final = vector_store.get("vector_store_id") or ""
 
     key_object_permission = user_api_key_dict.object_permission
@@ -178,12 +190,70 @@ async def can_user_access_vector_store(
     if _object_permission_allows_vector_store(team_object_permission, vector_store_id):
         return True
 
-    if user_api_key_dict.team_id is not None and user_api_key_dict.team_id == vector_store_team_id:
-        return True
+    return user_api_key_dict.team_id is not None and user_api_key_dict.team_id == vector_store.get("team_id")
 
+
+async def _team_auth_context(team_id: str, user_api_key_dict: UserAPIKeyAuth) -> UserAPIKeyAuth:
+    from litellm.proxy.auth.auth_checks import get_team_object
+    from litellm.proxy.proxy_server import (
+        prisma_client,
+        proxy_logging_obj,
+        user_api_key_cache,
+    )
+
+    team: Final = await get_team_object(
+        team_id=team_id,
+        prisma_client=prisma_client,
+        user_api_key_cache=user_api_key_cache,
+        parent_otel_span=user_api_key_dict.parent_otel_span,
+        proxy_logging_obj=proxy_logging_obj,
+    )
+    return user_api_key_dict.model_copy(
+        update=MappingProxyType(
+            {
+                "team_id": team_id,
+                "team_object_permission": team.object_permission,
+                "team_object_permission_id": team.object_permission_id,
+            }
+        )
+    )
+
+
+async def _vector_store_listing_auth_contexts(
+    user_api_key_dict: UserAPIKeyAuth,
+) -> tuple[UserAPIKeyAuth, ...]:
+    if not is_ui_session_credential(user_api_key_dict):
+        return (user_api_key_dict,)
+    session_key_context: Final = user_api_key_dict.model_copy(
+        update=MappingProxyType({"team_id": None, "team_object_permission": None, "team_object_permission_id": None})
+    )
+    team_ids: Final = await resolve_ui_session_team_ids(user_api_key_dict)
+    team_contexts: Final = tuple([await _team_auth_context(team_id, user_api_key_dict) for team_id in team_ids])
+    return (session_key_context, *team_contexts)
+
+
+async def _is_vector_store_granted_to_any(
+    vector_store: LiteLLM_ManagedVectorStore,
+    auth_contexts: tuple[UserAPIKeyAuth, ...],
+) -> bool:
+    for auth_context in auth_contexts:
+        if await _is_vector_store_granted(vector_store, auth_context):
+            return True
     return False
 
 
+async def filter_listable_vector_stores(
+    vector_stores: Iterable[LiteLLM_ManagedVectorStore],
+    user_api_key_dict: UserAPIKeyAuth,
+) -> tuple[LiteLLM_ManagedVectorStore, ...]:
+    """Non-admins only see stores their key, one of their teams' object_permission, or team ownership grants."""
+    if _is_proxy_admin(user_api_key_dict):
+        return tuple(vector_stores)
+
+    auth_contexts: Final = await _vector_store_listing_auth_contexts(user_api_key_dict)
+    return tuple([vs for vs in vector_stores if await _is_vector_store_granted_to_any(vs, auth_contexts)])
+
+
 async def get_litellm_managed_vector_store(
     vector_store_id: str,
 ) -> LiteLLM_ManagedVectorStore | None:
diff --git a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_access_control.py b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_access_control.py
index 7d72121456a..93049b21460 100644
--- a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_access_control.py
+++ b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_access_control.py
@@ -120,9 +120,7 @@ async def test_delete_vector_store_checks_access():
             "team_id": "team_456",
         }
     )
-    mock_prisma.db.litellm_managedvectorstorestable.find_unique = AsyncMock(
-        return_value=mock_vector_store
-    )
+    mock_prisma.db.litellm_managedvectorstorestable.find_unique = AsyncMock(return_value=mock_vector_store)
 
     # User from different team should get 403
     user_api_key_dict = UserAPIKeyAuth(team_id="team_789")
@@ -134,9 +132,115 @@ async def test_delete_vector_store_checks_access():
     ):
         with patch("litellm.vector_store_registry", None):
             with pytest.raises(HTTPException) as exc_info:
-                await delete_vector_store(
-                    data=request, user_api_key_dict=user_api_key_dict
-                )
+                await delete_vector_store(data=request, user_api_key_dict=user_api_key_dict)
 
             assert exc_info.value.status_code == 403
             assert "Access denied" in exc_info.value.detail
+
+
+_UNSCOPED: LiteLLM_ManagedVectorStore = {
+    "vector_store_id": "vs_unscoped",
+    "custom_llm_provider": "openai",
+    "team_id": None,
+}
+_TEAM_A_OWNED: LiteLLM_ManagedVectorStore = {
+    "vector_store_id": "vs_team_a",
+    "custom_llm_provider": "openai",
+    "team_id": "team_a",
+}
+_UI_CREATED: LiteLLM_ManagedVectorStore = {
+    "vector_store_id": "vs_ui_created",
+    "custom_llm_provider": "openai",
+    "team_id": "litellm-dashboard",
+}
+
+
+async def _listed_ids(user_api_key_dict: UserAPIKeyAuth) -> list[str]:
+    from litellm.proxy.vector_store_endpoints.management_endpoints import (
+        list_vector_stores,
+    )
+
+    with patch(  # test-quality-ok: the list route reads rows through this module-level DB helper, no injection seam
+        "litellm.proxy.vector_store_endpoints.management_endpoints.VectorStoreRegistry._get_vector_stores_from_db",
+        new=AsyncMock(return_value=[_UNSCOPED, _TEAM_A_OWNED, _UI_CREATED]),
+    ):
+        response = await list_vector_stores(user_api_key_dict=user_api_key_dict)
+    return sorted(vs["vector_store_id"] for vs in response["data"])
+
+
+@pytest.mark.asyncio
+async def test_list_vector_stores_hides_ungranted_stores_from_non_admin_keys():
+    """A store with no team_id and no allowlist entry is not listed for a key it was never granted to;
+    only team ownership or an explicit object_permission grant makes a store visible."""
+    assert await _listed_ids(UserAPIKeyAuth()) == []
+    assert await _listed_ids(UserAPIKeyAuth(team_id="team_a")) == ["vs_team_a"]
+    assert await _listed_ids(
+        UserAPIKeyAuth(
+            team_id="team_b",
+            object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="op-1", vector_stores=["vs_unscoped"]),
+        )
+    ) == ["vs_unscoped"]
+    assert await _listed_ids(
+        UserAPIKeyAuth(
+            team_id="team_b",
+            team_object_permission=LiteLLM_ObjectPermissionTable(
+                object_permission_id="op-2", vector_stores=["vs_unscoped"]
+            ),
+        )
+    ) == ["vs_unscoped"]
+    assert await _listed_ids(UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN)) == [
+        "vs_team_a",
+        "vs_ui_created",
+        "vs_unscoped",
+    ]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("user_team_ids", "session_key_grants", "expected"),
+    [
+        ([], None, []),
+        ([], ["vs_unscoped"], ["vs_unscoped"]),
+        (["team_a"], None, ["vs_team_a"]),
+        (["team_a", "team_granted"], None, ["vs_team_a", "vs_unscoped"]),
+    ],
+)
+async def test_list_vector_stores_dashboard_session_resolves_real_teams(
+    user_team_ids: list[str], session_key_grants: list[str] | None, expected: list[str]
+):
+    """A dashboard session lists through the user's real teams plus the session key's own grants: stores created
+    from the dashboard (team_id litellm-dashboard) are not visible just because every session shares that team id,
+    while stores owned by or granted to one of the user's teams, or granted to the session key itself, are."""
+    from litellm.models.team import LiteLLM_TeamTableCachedObj
+
+    alice = UserAPIKeyAuth(
+        team_id="litellm-dashboard",
+        user_id="alice",
+        user_role=LitellmUserRoles.INTERNAL_USER,
+        object_permission=(
+            LiteLLM_ObjectPermissionTable(object_permission_id="op-4", vector_stores=session_key_grants)
+            if session_key_grants is not None
+            else None
+        ),
+    )
+    teams = {
+        "team_a": LiteLLM_TeamTableCachedObj(team_id="team_a"),
+        "team_granted": LiteLLM_TeamTableCachedObj(
+            team_id="team_granted",
+            object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="op-3", vector_stores=["vs_unscoped"]),
+        ),
+    }
+
+    async def fake_get_team_object(team_id: str, **_kwargs: object) -> LiteLLM_TeamTableCachedObj:
+        return teams[team_id]
+
+    with (
+        patch(  # test-quality-ok: team rows come from the module-level prisma client, no injection seam
+            "litellm.proxy.auth.auth_checks.get_team_object", new=fake_get_team_object
+        ),
+        patch(  # test-quality-ok: the user row comes from the module-level prisma client, no injection seam
+            "litellm.proxy.vector_store_endpoints.utils.resolve_ui_session_team_ids",
+            new=AsyncMock(return_value=user_team_ids),
+        ),
+    ):
+        assert await _listed_ids(alice) == expected