From 1e59cd9e3561be8df396619fcf3e6f87a89a5e57 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Fri, 17 Jul 2026 14:14:54 -0400
Subject: [PATCH 01/85] fix(utils): honor string drop_params values from config
 and DB deployments

---
 litellm/litellm_core_utils/core_helpers.py    | 12 +++++++
 litellm/types/router.py                       |  7 ++++
 litellm/utils.py                              |  9 +++--
 .../litellm_core_utils/test_core_helpers.py   | 22 ++++++++++++
 tests/test_litellm/test_router.py             | 34 +++++++++++++++++++
 tests/test_litellm/test_utils.py              | 28 +++++++++++++++
 ui/litellm-dashboard/src/lib/http/schema.d.ts |  4 +++
 7 files changed, 113 insertions(+), 3 deletions(-)

diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py
index 002a46771e3..838019264c9 100644
--- a/litellm/litellm_core_utils/core_helpers.py
+++ b/litellm/litellm_core_utils/core_helpers.py
@@ -36,6 +36,18 @@ def safe_divide_seconds(seconds: float, denominator: float, default: Optional[fl
     return float(seconds / denominator)
 
 
+def normalize_drop_params(value: object) -> bool | None:
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        lowered = value.strip().lower()
+        if lowered == "true":
+            return True
+        if lowered == "false":
+            return False
+    return None
+
+
 def safe_divide(
     numerator: Union[int, float],
     denominator: Union[int, float],
diff --git a/litellm/types/router.py b/litellm/types/router.py
index 28e4a8272e8..77d1815d28d 100644
--- a/litellm/types/router.py
+++ b/litellm/types/router.py
@@ -23,6 +23,7 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_valida
 from typing_extensions import Protocol, Required, TypedDict, runtime_checkable
 
 from litellm._uuid import uuid
+from litellm.litellm_core_utils.core_helpers import normalize_drop_params
 
 from .completion import CompletionRequest
 from .embedding import EmbeddingRequest
@@ -233,6 +234,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
         None  # timeout when making stream=True calls, if str, pass in as os.environ/
     )
     max_retries: Optional[int] = None
+    drop_params: Optional[bool] = None
     organization: Optional[str] = None  # for openai orgs
     configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None
     litellm_credential_name: Optional[str] = None
@@ -311,6 +313,11 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
             return filtered
         return data
 
+    @field_validator("drop_params", mode="before")
+    @classmethod
+    def coerce_drop_params(cls, value: object) -> Optional[bool]:
+        return normalize_drop_params(value)
+
     def __contains__(self, key):
         # Define custom behavior for the 'in' operator
         return hasattr(self, key)
diff --git a/litellm/utils.py b/litellm/utils.py
index e19d2b36a52..e10b63ceb37 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -60,6 +60,7 @@ from litellm._lazy_imports import (
     _get_token_counter_new,
 )
 from litellm._uuid import uuid
+from litellm.litellm_core_utils.core_helpers import normalize_drop_params
 from litellm.litellm_core_utils.fallback_generalizations import (
     match_capability_generalizations,
 )
@@ -2852,7 +2853,7 @@ def get_optional_params_transcription(
 
     passed_params.pop("OPENAI_TRANSCRIPTION_PARAMS")
     custom_llm_provider = passed_params.pop("custom_llm_provider")
-    drop_params = passed_params.pop("drop_params")
+    drop_params = normalize_drop_params(passed_params.pop("drop_params"))
     special_params = passed_params.pop("kwargs")
     for k, v in special_params.items():
         passed_params[k] = v
@@ -2960,7 +2961,7 @@ def get_optional_params_image_gen(
     model = passed_params.pop("model", None)
     custom_llm_provider = passed_params.pop("custom_llm_provider")
     provider_config = passed_params.pop("provider_config", None)
-    drop_params = passed_params.pop("drop_params", None)
+    drop_params = normalize_drop_params(passed_params.pop("drop_params", None))
     additional_drop_params = passed_params.pop("additional_drop_params", None)
     special_params = passed_params.pop("kwargs")
     for k, v in special_params.items():
@@ -3084,7 +3085,7 @@ def get_optional_params_embeddings(
     custom_llm_provider = passed_params.pop("custom_llm_provider", None)
     special_params = passed_params.pop("kwargs")
 
-    drop_params = passed_params.pop("drop_params", None)
+    drop_params = normalize_drop_params(passed_params.pop("drop_params", None))
     additional_drop_params = passed_params.pop("additional_drop_params", None)
     allowed_openai_params = passed_params.pop("allowed_openai_params", None) or []
     # Remove function objects from passed_params to avoid JSON serialization errors
@@ -3797,6 +3798,8 @@ def get_optional_params(
 ):
     passed_params = locals().copy()
     special_params = passed_params.pop("kwargs")
+    drop_params = normalize_drop_params(drop_params)
+    passed_params["drop_params"] = drop_params
     # Remove base_model from passed_params so it doesn't interfere with
     # non_default_params / _check_valid_arg — it's a routing hint, not an
     # OpenAI param.
diff --git a/tests/test_litellm/litellm_core_utils/test_core_helpers.py b/tests/test_litellm/litellm_core_utils/test_core_helpers.py
index b67ea91bb0b..a7f93e3c997 100644
--- a/tests/test_litellm/litellm_core_utils/test_core_helpers.py
+++ b/tests/test_litellm/litellm_core_utils/test_core_helpers.py
@@ -5,6 +5,7 @@ import pytest
 from litellm.litellm_core_utils.core_helpers import (
     _FINISH_REASON_MAP,
     map_finish_reason,
+    normalize_drop_params,
     reconstruct_model_name,
     redact_nested_match_and_regex_keys,
 )
@@ -201,3 +202,24 @@ class TestRedactNestedMatchAndRegexKeys:
     def test_passes_through_none_and_str(self):
         assert redact_nested_match_and_regex_keys(None) is None
         assert redact_nested_match_and_regex_keys("plain") == "plain"
+
+
+@pytest.mark.parametrize(
+    "value, expected",
+    [
+        (True, True),
+        (False, False),
+        ("true", True),
+        ("True", True),
+        (" TRUE ", True),
+        ("false", False),
+        ("False", False),
+        (None, None),
+        ("yes", None),
+        ("", None),
+        (1, None),
+        (0, None),
+    ],
+)
+def test_normalize_drop_params(value, expected):
+    assert normalize_drop_params(value) is expected
diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py
index c2c98c8869c..81927d7e959 100644
--- a/tests/test_litellm/test_router.py
+++ b/tests/test_litellm/test_router.py
@@ -5430,3 +5430,37 @@ class TestRouterRequestTimeoutPropagation:
             )
             == 60
         )
+
+
+@pytest.mark.asyncio
+async def test_router_deployment_drop_params_string_true_is_honored(monkeypatch):
+    from litellm import Router
+
+    monkeypatch.setattr(litellm, "drop_params", False)
+    router = Router(
+        model_list=[
+            {
+                "model_name": "gpt-5-nano",
+                "litellm_params": {
+                    "model": "openai/gpt-5-nano",
+                    "api_key": "sk-fake",
+                    "temperature": 1,
+                    "reasoning_effort": "minimal",
+                    "drop_params": "true",
+                    "mock_response": "Hello, world!",
+                },
+            }
+        ],
+        num_retries=0,
+    )
+
+    deployment = router.get_deployment_by_model_group_name(model_group_name="gpt-5-nano")
+    assert deployment is not None
+    assert deployment.litellm_params.drop_params is True
+
+    response = await router.acompletion(
+        model="gpt-5-nano",
+        messages=[{"role": "user", "content": "hi"}],
+        temperature=0.1,
+    )
+    assert response.choices[0].message.content == "Hello, world!"
diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py
index 073ff17991e..0fbf9c0db20 100644
--- a/tests/test_litellm/test_utils.py
+++ b/tests/test_litellm/test_utils.py
@@ -4814,3 +4814,31 @@ def test_is_prompt_caching_valid_prompt_explicit_min_token_count_overrides_model
         is_prompt_caching_valid_prompt(model="claude-opus-4-8", messages=PROMPT_CACHE_MESSAGES, min_token_count=8192)
         is False
     )
+
+
+class TestDropParamsStringCoercion:
+    @pytest.mark.parametrize("drop_params", ["true", "True", True])
+    def test_truthy_drop_params_drops_unsupported_temperature(self, drop_params, monkeypatch):
+        from litellm.utils import get_optional_params
+
+        monkeypatch.setattr(litellm, "drop_params", False)
+        result = get_optional_params(
+            model="gpt-5-nano",
+            custom_llm_provider="openai",
+            temperature=0.1,
+            drop_params=drop_params,
+        )
+        assert "temperature" not in result
+
+    @pytest.mark.parametrize("drop_params", ["false", False, None])
+    def test_falsy_drop_params_still_raises(self, drop_params, monkeypatch):
+        from litellm.utils import get_optional_params
+
+        monkeypatch.setattr(litellm, "drop_params", False)
+        with pytest.raises(litellm.UnsupportedParamsError):
+            get_optional_params(
+                model="gpt-5-nano",
+                custom_llm_provider="openai",
+                temperature=0.1,
+                drop_params=drop_params,
+            )
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index 0d8f55164f9..62103e4f742 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -25667,6 +25667,8 @@ export interface components {
             default_api_key_rpm_limit?: number | null;
             /** Default Api Key Tpm Limit */
             default_api_key_tpm_limit?: number | null;
+            /** Drop Params */
+            drop_params?: boolean | null;
             /** Gcs Bucket Name */
             gcs_bucket_name?: string | null;
             /** Input Cost Per Audio Per Second */
@@ -33503,6 +33505,8 @@ export interface components {
             default_api_key_rpm_limit?: number | null;
             /** Default Api Key Tpm Limit */
             default_api_key_tpm_limit?: number | null;
+            /** Drop Params */
+            drop_params?: boolean | null;
             /** Gcs Bucket Name */
             gcs_bucket_name?: string | null;
             /** Input Cost Per Audio Per Second */

From dfcea2c1866630313ec3083794a922ff6971583a Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Fri, 28 Aug 2026 17:16:00 -0700
Subject: [PATCH 02/85] fix(policy_engine): execute post_call guardrail
 pipelines on responses

---
 litellm/proxy/utils.py                        |  47 +++++-
 .../proxy_logging/test_guardrail_pipeline.py  | 152 +++++++++++++++++-
 2 files changed, 196 insertions(+), 3 deletions(-)

diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index d880b529727..bd8ba6ae9f0 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -455,6 +455,30 @@ def _pipeline_managed_guardrail_names(data: Mapping[str, object]) -> frozenset[s
     )
 
 
+def _raise_for_streaming_post_call_pipelines(data: Mapping[str, object]) -> None:
+    if data.get("stream") is not True:
+        return
+    post_call_policies: Final = tuple(
+        policy_name for policy_name, pipeline in _policy_pipelines(data) if pipeline.mode == "post_call"
+    )
+    if not post_call_policies:
+        return
+    raise HTTPException(
+        status_code=400,
+        detail={
+            "error": {
+                "message": (
+                    "Policies with post_call guardrail pipelines cannot govern streaming responses yet: "
+                    f"{', '.join(post_call_policies)}. Retry with stream=false, or move these policies' output "
+                    "guardrails from pipeline steps to guardrails.add, which scans streamed output."
+                ),
+                "type": "guardrail_pipeline_error",
+                "policies": list(post_call_policies),
+            }
+        },
+    )
+
+
 def _prompt_block_text(block: object) -> str:
     if isinstance(block, str):
         return block
@@ -1578,6 +1602,7 @@ class ProxyLogging:
         call_type: str,
         event_hook: str,
         raw_request_snapshot: dict | None = None,  # mutable-ok: same request-payload shape as data
+        response: LLMResponseTypes | None = None,
     ) -> dict:
         """
         Execute guardrail pipelines if any are configured for this request.
@@ -1596,6 +1621,8 @@ class ProxyLogging:
         if not pipelines:
             return data
 
+        step_input: Final = {**data, "response": response} if response is not None else data
+
         for policy_name, pipeline in pipelines:
             if pipeline.mode != event_hook:
                 continue
@@ -1603,7 +1630,7 @@ class ProxyLogging:
             result: PipelineExecutionResult = await PipelineExecutor.execute_steps(
                 steps=pipeline.steps,
                 mode=pipeline.mode,
-                data=data,
+                data=step_input,
                 user_api_key_dict=user_api_key_dict,
                 call_type=call_type,
                 policy_name=policy_name,
@@ -1614,6 +1641,7 @@ class ProxyLogging:
                 result=result,
                 data=data,
                 policy_name=policy_name,
+                original_response=response,
             )
 
         return data
@@ -1623,14 +1651,18 @@ class ProxyLogging:
         result: PipelineExecutionResult,
         data: dict,
         policy_name: str,
+        original_response: LLMResponseTypes | None = None,
     ) -> dict:
         """
         Handle a PipelineExecutionResult — allow, block, or modify_response.
 
         Returns data dict if allowed, raises on block/modify_response.
+        ``original_response`` is set on the post_call path, where allowed
+        modifications land on the response object in place, so the request
+        payload (already sent upstream) is left untouched.
         """
         if result.terminal_action == "allow":
-            if result.modified_data is not None:
+            if result.modified_data is not None and original_response is None:
                 data.update(result.modified_data)
             return data
 
@@ -1671,6 +1703,7 @@ class ProxyLogging:
                 request_data=data,
                 guardrail_name=f"pipeline:{policy_name}",
                 detection_info=None,
+                original_response=original_response,
             )
 
         return data
@@ -1786,6 +1819,8 @@ class ProxyLogging:
         )
 
         try:
+            _raise_for_streaming_post_call_pipelines(data)
+
             # Execute guardrail pipelines before the normal callback loop
             data = await self._maybe_execute_pipelines(
                 data=data,
@@ -2774,6 +2809,14 @@ class ProxyLogging:
         from litellm.proxy.proxy_server import llm_router
         from litellm.types.guardrails import GuardrailEventHooks
 
+        await self._maybe_execute_pipelines(
+            data=data,
+            user_api_key_dict=user_api_key_dict,
+            call_type=getattr(data.get("litellm_logging_obj"), "call_type", None) or "acompletion",
+            event_hook="post_call",
+            response=response,
+        )
+
         guardrail_callbacks: Final[list[CustomGuardrail]] = []
         other_callbacks: Final[list[CustomLogger]] = []
         try:
diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
index e99e34d65d4..b7e86b68fe4 100644
--- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
+++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
@@ -23,7 +23,7 @@ from litellm.integrations.custom_guardrail import (
     ModifyResponseException,
 )
 from litellm.integrations.prometheus import PrometheusLogger
-from litellm.proxy.utils import ProxyLogging
+from litellm.proxy.utils import ProxyLogging, _raise_for_streaming_post_call_pipelines
 from litellm.types.guardrails import GuardrailEventHooks
 from litellm.types.proxy.policy_engine.pipeline_types import (
     GuardrailPipeline,
@@ -865,3 +865,153 @@ async def test_process_prompt_template_aresponses_swaps_model_and_merges_input(p
     hook_kwargs = logging_obj.async_get_chat_completion_prompt.await_args.kwargs
     assert hook_kwargs["messages"] == [{"role": "user", "content": "Who are you?"}]
     assert hook_kwargs["prompt_spec"] is prompt_spec
+
+
+# ---------------------------------------------------------------------------
+# post_call pipeline execution (LIT-6410)
+# ---------------------------------------------------------------------------
+
+
+def _post_call_pipeline_data(guardrail: str = "gr-post", **extra: Any) -> Dict[str, Any]:
+    pipeline = GuardrailPipeline(
+        mode="post_call",
+        steps=[PipelineStep(guardrail=guardrail, on_pass="allow", on_fail="block")],
+    )
+    return {
+        "model": "m",
+        "messages": [{"role": "user", "content": "hi"}],
+        "metadata": {
+            "_guardrail_pipelines": [("response-governance", pipeline)],
+            "_pipeline_managed_guardrails": {guardrail},
+        },
+        **extra,
+    }
+
+
+@pytest.mark.asyncio
+async def test_post_call_success_hook_runs_post_call_pipeline_and_reraises_block(
+    proxy_logging, make_user_api_key_auth, monkeypatch
+):
+    seen: Dict[str, Any] = {}
+
+    class OutputBlockingGuardrail(CustomGuardrail):
+        async def async_post_call_success_hook(self, data, user_api_key_dict, response):
+            seen["response"] = response
+            raise HTTPException(status_code=400, detail={"error": "output blocked"})
+
+    monkeypatch.setattr(
+        litellm,
+        "callbacks",
+        [OutputBlockingGuardrail(guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=False)],
+    )
+    monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
+    data = _post_call_pipeline_data()
+    response = litellm.ModelResponse()
+
+    with pytest.raises(HTTPException) as info:
+        await proxy_logging.post_call_success_hook(
+            data=data, response=response, user_api_key_dict=make_user_api_key_auth()
+        )
+
+    assert info.value.detail["error"] == "output blocked"
+    assert seen["response"] is response
+
+
+@pytest.mark.asyncio
+async def test_post_call_pipeline_pass_runs_once_and_leaves_request_data_untouched(
+    proxy_logging, make_user_api_key_auth, monkeypatch
+):
+    seen: Dict[str, Any] = {"count": 0}
+
+    class RecordingGuardrail(CustomGuardrail):
+        async def async_post_call_success_hook(self, data, user_api_key_dict, response):
+            seen["count"] += 1
+            seen["response"] = response
+            return None
+
+    monkeypatch.setattr(
+        litellm,
+        "callbacks",
+        [RecordingGuardrail(guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=False)],
+    )
+    monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
+    data = _post_call_pipeline_data()
+    response = litellm.ModelResponse()
+
+    out = await proxy_logging.post_call_success_hook(
+        data=data, response=response, user_api_key_dict=make_user_api_key_auth()
+    )
+
+    assert out is response
+    assert seen["response"] is response
+    assert seen["count"] == 1
+    assert "response" not in data
+    assert "guardrails" not in data["metadata"]
+
+
+def test_handle_pipeline_result_modify_response_carries_original_response():
+    result = MagicMock()
+    result.terminal_action = "modify_response"
+    result.modify_response_message = "filtered"
+    response = litellm.ModelResponse()
+
+    with pytest.raises(ModifyResponseException) as info:
+        ProxyLogging._handle_pipeline_result(
+            result=result, data={"model": "m"}, policy_name="p", original_response=response
+        )
+
+    assert info.value.original_response is response
+
+
+def test_handle_pipeline_result_allow_discards_modifications_on_post_call():
+    data = {"a": 1, "metadata": {"guardrails": ["other"]}}
+    result = MagicMock()
+    result.terminal_action = "allow"
+    result.modified_data = {"metadata": {"guardrails": ["gr-post"]}, "response": object()}
+
+    out = ProxyLogging._handle_pipeline_result(
+        result=result, data=data, policy_name="p", original_response=litellm.ModelResponse()
+    )
+
+    assert out is data
+    assert data == {"a": 1, "metadata": {"guardrails": ["other"]}}
+
+
+@pytest.mark.asyncio
+async def test_pre_call_hook_rejects_streaming_request_with_post_call_pipeline(
+    proxy_logging, make_user_api_key_auth, monkeypatch
+):
+    monkeypatch.setattr(litellm, "callbacks", [])
+    data = _post_call_pipeline_data(stream=True)
+
+    with pytest.raises(HTTPException) as info:
+        await proxy_logging.pre_call_hook(
+            user_api_key_dict=make_user_api_key_auth(),
+            data=data,
+            call_type="completion",
+            guardrails_only=True,
+        )
+
+    assert info.value.status_code == 400
+    assert info.value.detail["error"]["policies"] == ["response-governance"]
+    assert "stream=false" in info.value.detail["error"]["message"]
+
+
+def test_raise_for_streaming_post_call_pipelines_ignores_non_streaming_and_pre_call():
+    post_call = GuardrailPipeline(mode="post_call", steps=[PipelineStep(guardrail="g", on_fail="block")])
+    pre_call = GuardrailPipeline(mode="pre_call", steps=[PipelineStep(guardrail="g", on_fail="block")])
+
+    assert (
+        _raise_for_streaming_post_call_pipelines(
+            {"stream": False, "metadata": {"_guardrail_pipelines": [("p", post_call)]}}
+        )
+        is None
+    )
+    assert _raise_for_streaming_post_call_pipelines({"metadata": {"_guardrail_pipelines": [("p", post_call)]}}) is None
+    assert (
+        _raise_for_streaming_post_call_pipelines(
+            {"stream": True, "metadata": {"_guardrail_pipelines": [("p", pre_call)]}}
+        )
+        is None
+    )
+    assert _raise_for_streaming_post_call_pipelines({"stream": True}) is None

From e6edd62f5d0f010d34c203d9df8192462a1622c0 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Fri, 28 Aug 2026 17:40:48 -0700
Subject: [PATCH 03/85] fix(policy_engine): propagate post_call pipeline
 replacement responses to the client

---
 .../proxy/policy_engine/pipeline_executor.py  | 19 +++--
 litellm/proxy/utils.py                        | 32 +++++---
 .../proxy_logging/test_guardrail_pipeline.py  | 81 ++++++++++++++++++-
 .../utils/proxy_logging/test_pre_call_hook.py |  2 +-
 4 files changed, 112 insertions(+), 22 deletions(-)

diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py
index a5619821197..190784a2a60 100644
--- a/litellm/proxy/policy_engine/pipeline_executor.py
+++ b/litellm/proxy/policy_engine/pipeline_executor.py
@@ -108,8 +108,10 @@ class PipelineExecutor:
                 action,
             )
 
-            # Forward modified data to next step if pass_data is True
-            if step.pass_data and modified_data is not None:
+            # Forward modified data to the next step if pass_data is True;
+            # post_call response replacements always chain, matching the flat
+            # callback loop where each hook sees the previous hook's response
+            if modified_data is not None and (step.pass_data or mode == "post_call"):
                 working_data = {**working_data, **modified_data}
 
             # Handle terminal actions
@@ -227,11 +229,14 @@ class PipelineExecutor:
             # same contract as run_in_parallel/scan_raw_request elsewhere: any
             # data it returned is discarded, since applying it on top of the
             # raw snapshot would silently undo whatever an earlier step in
-            # this pipeline already did.
-            modified_data = None
-            if response is not None and isinstance(response, dict) and not scans_raw_request:
-                modified_data = response
-            return ("pass", modified_data, None, None)
+            # this pipeline already did. A post_call hook's non-None return is
+            # a replacement response (the flat callback-loop contract), carried
+            # under the same "response" key the step input uses.
+            if response is None or scans_raw_request:
+                return ("pass", None, None, None)
+            if mode == "post_call":
+                return ("pass", {"response": response}, None, None)
+            return ("pass", response if isinstance(response, dict) else None, None, None)
 
         except Exception as e:
             if CustomGuardrail._is_guardrail_intervention(e):
diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index bd8ba6ae9f0..25c1068cc6d 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -1603,7 +1603,7 @@ class ProxyLogging:
         event_hook: str,
         raw_request_snapshot: dict | None = None,  # mutable-ok: same request-payload shape as data
         response: LLMResponseTypes | None = None,
-    ) -> dict:
+    ) -> tuple[dict, LLMResponseTypes | None]:
         """
         Execute guardrail pipelines if any are configured for this request.
 
@@ -1615,18 +1615,21 @@ class ProxyLogging:
         ``scan_raw_request`` evaluates the pristine request, not whatever an
         earlier ``pass_data`` step in the same pipeline already rewrote.
 
-        Returns the (possibly modified) data dict.
+        Returns the (possibly modified) data dict, plus the replacement
+        response when a post_call pipeline step returned one (None when the
+        response is unchanged), matching the flat callback-loop contract.
         """
         pipelines: Final = _policy_pipelines(data)
         if not pipelines:
-            return data
-
-        step_input: Final = {**data, "response": response} if response is not None else data
+            return data, None
 
+        current_response = response  # rebind-ok: chains each pipeline's replacement response into the next
         for policy_name, pipeline in pipelines:
             if pipeline.mode != event_hook:
                 continue
 
+            step_input: dict = {**data, "response": current_response} if current_response is not None else data
+
             result: PipelineExecutionResult = await PipelineExecutor.execute_steps(
                 steps=pipeline.steps,
                 mode=pipeline.mode,
@@ -1641,10 +1644,13 @@ class ProxyLogging:
                 result=result,
                 data=data,
                 policy_name=policy_name,
-                original_response=response,
+                original_response=current_response,
             )
 
-        return data
+            if current_response is not None and result.modified_data is not None:
+                current_response = result.modified_data.get("response", current_response)
+
+        return data, current_response if current_response is not response else None
 
     @staticmethod
     def _handle_pipeline_result(
@@ -1657,9 +1663,9 @@ class ProxyLogging:
         Handle a PipelineExecutionResult — allow, block, or modify_response.
 
         Returns data dict if allowed, raises on block/modify_response.
-        ``original_response`` is set on the post_call path, where allowed
-        modifications land on the response object in place, so the request
-        payload (already sent upstream) is left untouched.
+        ``original_response`` is set on the post_call path, where the request
+        payload (already sent upstream) must stay untouched; a replacement
+        response carried in ``modified_data`` is adopted by the caller.
         """
         if result.terminal_action == "allow":
             if result.modified_data is not None and original_response is None:
@@ -1822,7 +1828,7 @@ class ProxyLogging:
             _raise_for_streaming_post_call_pipelines(data)
 
             # Execute guardrail pipelines before the normal callback loop
-            data = await self._maybe_execute_pipelines(
+            data, _ = await self._maybe_execute_pipelines(
                 data=data,
                 user_api_key_dict=user_api_key_dict,
                 call_type=call_type,
@@ -2809,13 +2815,15 @@ class ProxyLogging:
         from litellm.proxy.proxy_server import llm_router
         from litellm.types.guardrails import GuardrailEventHooks
 
-        await self._maybe_execute_pipelines(
+        _, pipeline_response = await self._maybe_execute_pipelines(
             data=data,
             user_api_key_dict=user_api_key_dict,
             call_type=getattr(data.get("litellm_logging_obj"), "call_type", None) or "acompletion",
             event_hook="post_call",
             response=response,
         )
+        if pipeline_response is not None:
+            response = pipeline_response  # rebind-ok: adopt the pipeline's replacement response, same contract as the callback loops below
 
         guardrail_callbacks: Final[list[CustomGuardrail]] = []
         other_callbacks: Final[list[CustomLogger]] = []
diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
index b7e86b68fe4..8bc71f6e178 100644
--- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
+++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
@@ -326,13 +326,14 @@ def test_process_guardrail_metadata_invalid_data_raises(proxy_logging):
 @pytest.mark.asyncio
 async def test_maybe_execute_pipelines_no_pipelines_returns_data(proxy_logging, make_user_api_key_auth):
     data = {"messages": [{"role": "user"}], "model": "m", "temperature": 0.1}
-    out = await proxy_logging._maybe_execute_pipelines(
+    out, replacement = await proxy_logging._maybe_execute_pipelines(
         data=data,
         user_api_key_dict=make_user_api_key_auth(),
         call_type="completion",
         event_hook="pre_call",
     )
     assert out == {"messages": [{"role": "user"}], "model": "m", "temperature": 0.1}
+    assert replacement is None
 
 
 @pytest.mark.asyncio
@@ -344,7 +345,7 @@ async def test_maybe_execute_pipelines_skips_pipelines_with_other_mode(proxy_log
     monkeypatch.setattr(
         "litellm.proxy.policy_engine.pipeline_executor.PipelineExecutor.execute_steps", executed
     )
-    out = await proxy_logging._maybe_execute_pipelines(
+    out, replacement = await proxy_logging._maybe_execute_pipelines(
         data=data,
         user_api_key_dict=make_user_api_key_auth(),
         call_type="completion",
@@ -352,6 +353,7 @@ async def test_maybe_execute_pipelines_skips_pipelines_with_other_mode(proxy_log
     )
     executed.assert_not_called()
     assert out is data
+    assert replacement is None
 
 
 @pytest.mark.parametrize(
@@ -949,6 +951,81 @@ async def test_post_call_pipeline_pass_runs_once_and_leaves_request_data_untouch
     assert "guardrails" not in data["metadata"]
 
 
+@pytest.mark.asyncio
+async def test_post_call_pipeline_replacement_response_reaches_caller(
+    proxy_logging, make_user_api_key_auth, monkeypatch
+):
+    masked = litellm.ModelResponse()
+
+    class MaskingGuardrail(CustomGuardrail):
+        async def async_post_call_success_hook(self, data, user_api_key_dict, response):
+            return masked
+
+    monkeypatch.setattr(
+        litellm,
+        "callbacks",
+        [MaskingGuardrail(guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=False)],
+    )
+    monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
+    data = _post_call_pipeline_data()
+
+    out = await proxy_logging.post_call_success_hook(
+        data=data, response=litellm.ModelResponse(), user_api_key_dict=make_user_api_key_auth()
+    )
+
+    assert out is masked
+    assert "response" not in data
+
+
+@pytest.mark.asyncio
+async def test_post_call_pipeline_replacement_chains_to_next_step_without_pass_data(
+    proxy_logging, make_user_api_key_auth, monkeypatch
+):
+    masked = litellm.ModelResponse()
+    seen: Dict[str, Any] = {}
+
+    class MaskingGuardrail(CustomGuardrail):
+        async def async_post_call_success_hook(self, data, user_api_key_dict, response):
+            return masked
+
+    class RecordingGuardrail(CustomGuardrail):
+        async def async_post_call_success_hook(self, data, user_api_key_dict, response):
+            seen["response"] = response
+            return None
+
+    pipeline = GuardrailPipeline(
+        mode="post_call",
+        steps=[
+            PipelineStep(guardrail="gr-mask", on_pass="next", on_fail="block"),
+            PipelineStep(guardrail="gr-audit", on_pass="allow", on_fail="block"),
+        ],
+    )
+    monkeypatch.setattr(
+        litellm,
+        "callbacks",
+        [
+            MaskingGuardrail(guardrail_name="gr-mask", event_hook=GuardrailEventHooks.post_call, default_on=False),
+            RecordingGuardrail(guardrail_name="gr-audit", event_hook=GuardrailEventHooks.post_call, default_on=False),
+        ],
+    )
+    monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
+    data = {
+        "model": "m",
+        "messages": [{"role": "user", "content": "hi"}],
+        "metadata": {
+            "_guardrail_pipelines": [("response-governance", pipeline)],
+            "_pipeline_managed_guardrails": {"gr-mask", "gr-audit"},
+        },
+    }
+
+    out = await proxy_logging.post_call_success_hook(
+        data=data, response=litellm.ModelResponse(), user_api_key_dict=make_user_api_key_auth()
+    )
+
+    assert out is masked
+    assert seen["response"] is masked
+
+
 def test_handle_pipeline_result_modify_response_carries_original_response():
     result = MagicMock()
     result.terminal_action = "modify_response"
diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_pre_call_hook.py b/tests/test_litellm/proxy/utils/proxy_logging/test_pre_call_hook.py
index 0971ce09d79..06cf328a20c 100644
--- a/tests/test_litellm/proxy/utils/proxy_logging/test_pre_call_hook.py
+++ b/tests/test_litellm/proxy/utils/proxy_logging/test_pre_call_hook.py
@@ -660,7 +660,7 @@ async def test_scan_raw_request_snapshot_taken_before_pipelines(
         for msg in data.get("messages", []):
             if "SECRET" in msg.get("content", ""):
                 msg["content"] = msg["content"].replace("SECRET", "[REDACTED]")
-        return data
+        return data, None
 
     monkeypatch.setattr(ProxyLogging, "_maybe_execute_pipelines", fake_pipelines)
     monkeypatch.setattr(litellm, "callbacks", [_BlockOnSecretGuardrail(scan_raw_request=True)])

From aeac6a412c98c07cce64c5ddbd74d005f2e60e7e Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Fri, 28 Aug 2026 17:59:15 -0700
Subject: [PATCH 04/85] fix(policy_engine): skip pipeline-managed guardrails in
 the response-path guardrail loop

---
 litellm/proxy/utils.py                        |  9 ++++++-
 .../proxy_logging/test_guardrail_pipeline.py  | 26 +++++++++++++++++++
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index 25c1068cc6d..75cc5c259a7 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -2825,6 +2825,7 @@ class ProxyLogging:
         if pipeline_response is not None:
             response = pipeline_response  # rebind-ok: adopt the pipeline's replacement response, same contract as the callback loops below
 
+        pipeline_managed: Final = _pipeline_managed_guardrail_names(data)
         guardrail_callbacks: Final[list[CustomGuardrail]] = []
         other_callbacks: Final[list[CustomLogger]] = []
         try:
@@ -2849,12 +2850,18 @@ class ProxyLogging:
             guardrail_data: Final = _check_and_merge_model_level_guardrails(data=data, llm_router=llm_router)
 
             parallel_guardrails: Final[tuple[CustomGuardrail, ...]] = tuple(
-                callback for callback in guardrail_callbacks if getattr(callback, "run_in_parallel", False)
+                callback
+                for callback in guardrail_callbacks
+                if getattr(callback, "run_in_parallel", False)
+                and not (callback.guardrail_name and callback.guardrail_name in pipeline_managed)
             )
 
             for callback in guardrail_callbacks:
                 # Main - V2 Guardrails implementation
 
+                if callback.guardrail_name and callback.guardrail_name in pipeline_managed:
+                    continue
+
                 if getattr(callback, "run_in_parallel", False):
                     continue
 
diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
index 8bc71f6e178..c92f5fa6c55 100644
--- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
+++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
@@ -951,6 +951,32 @@ async def test_post_call_pipeline_pass_runs_once_and_leaves_request_data_untouch
     assert "guardrails" not in data["metadata"]
 
 
+@pytest.mark.asyncio
+async def test_post_call_pipeline_managed_default_on_guardrail_runs_exactly_once(
+    proxy_logging, make_user_api_key_auth, monkeypatch
+):
+    seen: Dict[str, Any] = {"count": 0}
+
+    class CountingGuardrail(CustomGuardrail):
+        async def async_post_call_success_hook(self, data, user_api_key_dict, response):
+            seen["count"] += 1
+            return None
+
+    monkeypatch.setattr(
+        litellm,
+        "callbacks",
+        [CountingGuardrail(guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=True)],
+    )
+    monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
+    data = _post_call_pipeline_data()
+
+    await proxy_logging.post_call_success_hook(
+        data=data, response=litellm.ModelResponse(), user_api_key_dict=make_user_api_key_auth()
+    )
+
+    assert seen["count"] == 1
+
+
 @pytest.mark.asyncio
 async def test_post_call_pipeline_replacement_response_reaches_caller(
     proxy_logging, make_user_api_key_auth, monkeypatch

From 55569729b05d601c139e43b8faba447983e89f74 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Fri, 28 Aug 2026 18:16:35 -0700
Subject: [PATCH 05/85] fix(policy_engine): scope pipeline-managed guardrail
 skips to the pipeline's mode

---
 litellm/proxy/utils.py                        | 20 ++---
 .../proxy_logging/test_guardrail_pipeline.py  | 73 +++++++++++++++++++
 2 files changed, 84 insertions(+), 9 deletions(-)

diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index 75cc5c259a7..d3f2e1d7301 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -11,7 +11,7 @@ import sys
 import threading
 import time
 import traceback
-from collections.abc import AsyncGenerator, Awaitable, Callable, Collection, Coroutine, Mapping, Sequence
+from collections.abc import AsyncGenerator, Awaitable, Callable, Coroutine, Mapping, Sequence
 from dataclasses import dataclass, field
 from datetime import date, datetime, timedelta, timezone
 from email.mime.multipart import MIMEMultipart
@@ -446,12 +446,14 @@ def _policy_pipelines(data: Mapping[str, object]) -> tuple[tuple[str, "Guardrail
     )
 
 
-def _pipeline_managed_guardrail_names(data: Mapping[str, object]) -> frozenset[str]:
-    managed: Final = _policy_state_metadata(data).get("_pipeline_managed_guardrails")
-    return (
-        frozenset(cast("Collection[str]", managed))  # cast-ok: the policy engine wrote these guardrail names
-        if managed
-        else frozenset()
+def _pipeline_managed_guardrail_names(
+    data: Mapping[str, object], mode: Literal["pre_call", "post_call"]
+) -> frozenset[str]:
+    return frozenset(
+        step.guardrail
+        for _policy_name, pipeline in _policy_pipelines(data)
+        if pipeline.mode == mode
+        for step in pipeline.steps
     )
 
 
@@ -1837,7 +1839,7 @@ class ProxyLogging:
             )
 
             # Get pipeline-managed guardrails to skip in normal loop
-            pipeline_managed: Final = _pipeline_managed_guardrail_names(data)
+            pipeline_managed: Final = _pipeline_managed_guardrail_names(data, "pre_call")
 
             caps: Final = ProxyLogging._callback_capabilities()
             # Skip the per-request callback walk entirely when nothing in
@@ -2825,7 +2827,7 @@ class ProxyLogging:
         if pipeline_response is not None:
             response = pipeline_response  # rebind-ok: adopt the pipeline's replacement response, same contract as the callback loops below
 
-        pipeline_managed: Final = _pipeline_managed_guardrail_names(data)
+        pipeline_managed: Final = _pipeline_managed_guardrail_names(data, "post_call")
         guardrail_callbacks: Final[list[CustomGuardrail]] = []
         other_callbacks: Final[list[CustomLogger]] = []
         try:
diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
index c92f5fa6c55..4e1ccf71c5c 100644
--- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
+++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
@@ -977,6 +977,79 @@ async def test_post_call_pipeline_managed_default_on_guardrail_runs_exactly_once
     assert seen["count"] == 1
 
 
+@pytest.mark.asyncio
+async def test_post_call_hook_still_runs_guardrail_managed_only_by_pre_call_pipeline(
+    proxy_logging, make_user_api_key_auth, monkeypatch
+):
+    seen: Dict[str, Any] = {"count": 0}
+
+    class DualStageGuardrail(CustomGuardrail):
+        async def async_post_call_success_hook(self, data, user_api_key_dict, response):
+            seen["count"] += 1
+            return None
+
+    pre_call_pipeline = GuardrailPipeline(
+        mode="pre_call",
+        steps=[PipelineStep(guardrail="gr-dual", on_pass="allow", on_fail="block")],
+    )
+    monkeypatch.setattr(
+        litellm,
+        "callbacks",
+        [DualStageGuardrail(guardrail_name="gr-dual", event_hook=["pre_call", "post_call"], default_on=True)],
+    )
+    monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
+    data = {
+        "model": "m",
+        "messages": [{"role": "user", "content": "hi"}],
+        "metadata": {
+            "_guardrail_pipelines": [("request-governance", pre_call_pipeline)],
+            "_pipeline_managed_guardrails": {"gr-dual"},
+        },
+    }
+
+    await proxy_logging.post_call_success_hook(
+        data=data, response=litellm.ModelResponse(), user_api_key_dict=make_user_api_key_auth()
+    )
+
+    assert seen["count"] == 1
+
+
+@pytest.mark.asyncio
+async def test_pre_call_hook_still_runs_guardrail_managed_only_by_post_call_pipeline(
+    proxy_logging, make_user_api_key_auth, monkeypatch
+):
+    seen: Dict[str, Any] = {"count": 0}
+
+    class DualStageGuardrail(CustomGuardrail):
+        async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type):
+            seen["count"] += 1
+            return data
+
+    post_call_pipeline = GuardrailPipeline(
+        mode="post_call",
+        steps=[PipelineStep(guardrail="gr-dual", on_pass="allow", on_fail="block")],
+    )
+    monkeypatch.setattr(
+        litellm,
+        "callbacks",
+        [DualStageGuardrail(guardrail_name="gr-dual", event_hook=["pre_call", "post_call"], default_on=True)],
+    )
+    data = {
+        "model": "m",
+        "messages": [{"role": "user", "content": "hi"}],
+        "metadata": {
+            "_guardrail_pipelines": [("response-governance", post_call_pipeline)],
+            "_pipeline_managed_guardrails": {"gr-dual"},
+        },
+    }
+
+    await proxy_logging.pre_call_hook(
+        user_api_key_dict=make_user_api_key_auth(), data=data, call_type="completion"
+    )
+
+    assert seen["count"] == 1
+
+
 @pytest.mark.asyncio
 async def test_post_call_pipeline_replacement_response_reaches_caller(
     proxy_logging, make_user_api_key_auth, monkeypatch

From 996019cd23423c7b2a35dbd50f4b3b3571362cd3 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 29 Aug 2026 00:14:23 -0700
Subject: [PATCH 06/85] fix(policy_engine): keep post_call pipeline guardrail
 logging and reject background bypass

Post_call pipelines run step hooks against a copied request dict, so guardrail
writes into the metadata bucket (applied_guardrails for the response header,
standard_logging_guardrail_information for spend logs) were dropped when the
guardrail was the first writer. Merge those writes back onto the request on the
post_call allow path, keeping the request payload and the executor's per-step
guardrails activation flag out of it.

Background /v1/responses requests dodge the streaming 400: pre_call sees stream
unset, then the polling task forces stream=true with pre-call logic skipped and
the streaming branch returns before post_call_success_hook, silently bypassing
post_call pipelines. Reject background=true at pre_call the same way as
stream=true.

Also pin the run_in_parallel pipeline-managed exclusion in both hook loops with
regression tests.
---
 litellm/proxy/utils.py                        |  49 +++++-
 .../proxy_logging/test_guardrail_pipeline.py  | 148 +++++++++++++++++-
 2 files changed, 187 insertions(+), 10 deletions(-)

diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index d3f2e1d7301..0f97473312c 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -457,8 +457,37 @@ def _pipeline_managed_guardrail_names(
     )
 
 
+def _merge_pipeline_metadata_bucket(data: dict, bucket_key: str, modified_bucket_value: object) -> None:
+    if not isinstance(modified_bucket_value, dict):
+        return
+    modified_bucket: Final = cast("dict[str, object]", modified_bucket_value)  # cast-ok: metadata buckets are str-keyed
+    surviving_writes: Final = {key: value for key, value in modified_bucket.items() if key != "guardrails"}
+    existing_bucket: Final = data.get(bucket_key)
+    if isinstance(existing_bucket, dict):
+        cast("dict[str, object]", existing_bucket).update(surviving_writes)  # cast-ok: metadata buckets are str-keyed
+    else:
+        data[bucket_key] = surviving_writes
+
+
+def _merge_pipeline_metadata_writes(data: dict, modified_data: Mapping[str, object]) -> None:
+    """
+    Copy metadata-bucket writes from a pipeline's working copy back onto the request.
+
+    Post_call pipelines run step hooks against a copied request dict so the payload
+    already sent upstream stays untouched, but hooks record proxy-internal logging
+    state in the metadata buckets (``applied_guardrails`` for response headers,
+    ``standard_logging_guardrail_information`` for spend logs), and those writes
+    must reach the request dict the proxy keeps reading after the pipeline returns.
+
+    The ``guardrails`` key is the executor's per-step activation flag for
+    ``should_run_guardrail``, not a hook write, so it stays in the working copy.
+    """
+    for bucket_key in ("metadata", "litellm_metadata"):
+        _merge_pipeline_metadata_bucket(data, bucket_key, modified_data.get(bucket_key))
+
+
 def _raise_for_streaming_post_call_pipelines(data: Mapping[str, object]) -> None:
-    if data.get("stream") is not True:
+    if data.get("stream") is not True and data.get("background") is not True:
         return
     post_call_policies: Final = tuple(
         policy_name for policy_name, pipeline in _policy_pipelines(data) if pipeline.mode == "post_call"
@@ -470,9 +499,10 @@ def _raise_for_streaming_post_call_pipelines(data: Mapping[str, object]) -> None
         detail={
             "error": {
                 "message": (
-                    "Policies with post_call guardrail pipelines cannot govern streaming responses yet: "
-                    f"{', '.join(post_call_policies)}. Retry with stream=false, or move these policies' output "
-                    "guardrails from pipeline steps to guardrails.add, which scans streamed output."
+                    "Policies with post_call guardrail pipelines cannot govern streaming or background "
+                    f"responses yet: {', '.join(post_call_policies)}. Retry with stream=false and "
+                    "background=false, or move these policies' output guardrails from pipeline steps to "
+                    "guardrails.add, which scans streamed output."
                 ),
                 "type": "guardrail_pipeline_error",
                 "policies": list(post_call_policies),
@@ -1667,11 +1697,16 @@ class ProxyLogging:
         Returns data dict if allowed, raises on block/modify_response.
         ``original_response`` is set on the post_call path, where the request
         payload (already sent upstream) must stay untouched; a replacement
-        response carried in ``modified_data`` is adopted by the caller.
+        response carried in ``modified_data`` is adopted by the caller, and
+        metadata-bucket writes (applied guardrails, guardrail logging info)
+        are merged back so headers and spend logs still see them.
         """
         if result.terminal_action == "allow":
-            if result.modified_data is not None and original_response is None:
-                data.update(result.modified_data)
+            if result.modified_data is not None:
+                if original_response is None:
+                    data.update(result.modified_data)
+                else:
+                    _merge_pipeline_metadata_writes(data, result.modified_data)
             return data
 
         if result.terminal_action == "block":
diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
index 4e1ccf71c5c..34a25d75bc0 100644
--- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
+++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
@@ -23,6 +23,7 @@ from litellm.integrations.custom_guardrail import (
     ModifyResponseException,
 )
 from litellm.integrations.prometheus import PrometheusLogger
+from litellm.proxy.common_utils.callback_utils import add_guardrail_to_applied_guardrails_header
 from litellm.proxy.utils import ProxyLogging, _raise_for_streaming_post_call_pipelines
 from litellm.types.guardrails import GuardrailEventHooks
 from litellm.types.proxy.policy_engine.pipeline_types import (
@@ -1139,18 +1140,132 @@ def test_handle_pipeline_result_modify_response_carries_original_response():
     assert info.value.original_response is response
 
 
-def test_handle_pipeline_result_allow_discards_modifications_on_post_call():
+def test_handle_pipeline_result_allow_on_post_call_keeps_metadata_writes_only():
     data = {"a": 1, "metadata": {"guardrails": ["other"]}}
     result = MagicMock()
     result.terminal_action = "allow"
-    result.modified_data = {"metadata": {"guardrails": ["gr-post"]}, "response": object()}
+    result.modified_data = {
+        "a": 2,
+        "metadata": {"guardrails": ["other"], "applied_guardrails": ["gr-post"]},
+        "response": object(),
+    }
 
     out = ProxyLogging._handle_pipeline_result(
         result=result, data=data, policy_name="p", original_response=litellm.ModelResponse()
     )
 
     assert out is data
-    assert data == {"a": 1, "metadata": {"guardrails": ["other"]}}
+    assert data["a"] == 1
+    assert "response" not in data
+    assert data["metadata"] == {"guardrails": ["other"], "applied_guardrails": ["gr-post"]}
+
+
+@pytest.mark.asyncio
+async def test_post_call_pipeline_guardrail_metadata_writes_reach_request_data(
+    proxy_logging, make_user_api_key_auth, monkeypatch
+):
+    class HeaderWritingGuardrail(CustomGuardrail):
+        async def async_post_call_success_hook(self, data, user_api_key_dict, response):
+            add_guardrail_to_applied_guardrails_header(request_data=data, guardrail_name="gr-post")
+            self.add_standard_logging_guardrail_information_to_request_data(
+                guardrail_json_response={"verdict": "pass"},
+                request_data=data,
+                guardrail_status="success",
+            )
+            return None
+
+    monkeypatch.setattr(
+        litellm,
+        "callbacks",
+        [HeaderWritingGuardrail(guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=False)],
+    )
+    monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
+    data = _post_call_pipeline_data()
+
+    await proxy_logging.post_call_success_hook(
+        data=data, response=litellm.ModelResponse(), user_api_key_dict=make_user_api_key_auth()
+    )
+
+    assert data["metadata"]["applied_guardrails"] == ["gr-post"]
+    slg_entries = data["metadata"]["standard_logging_guardrail_information"]
+    assert len(slg_entries) == 1
+    assert slg_entries[0]["guardrail_name"] == "gr-post"
+
+
+@pytest.mark.asyncio
+async def test_post_call_pipeline_managed_parallel_guardrail_runs_exactly_once(
+    proxy_logging, make_user_api_key_auth, monkeypatch
+):
+    seen: Dict[str, Any] = {"count": 0}
+
+    class CountingGuardrail(CustomGuardrail):
+        async def async_post_call_success_hook(self, data, user_api_key_dict, response):
+            seen["count"] += 1
+            return None
+
+    monkeypatch.setattr(
+        litellm,
+        "callbacks",
+        [
+            CountingGuardrail(
+                guardrail_name="gr-post",
+                event_hook=GuardrailEventHooks.post_call,
+                default_on=True,
+                run_in_parallel=True,
+            )
+        ],
+    )
+    monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
+    data = _post_call_pipeline_data()
+
+    await proxy_logging.post_call_success_hook(
+        data=data, response=litellm.ModelResponse(), user_api_key_dict=make_user_api_key_auth()
+    )
+
+    assert seen["count"] == 1
+
+
+@pytest.mark.asyncio
+async def test_pre_call_pipeline_managed_parallel_guardrail_runs_exactly_once(
+    proxy_logging, make_user_api_key_auth, monkeypatch
+):
+    seen: Dict[str, Any] = {"count": 0}
+
+    class CountingGuardrail(CustomGuardrail):
+        async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type):
+            seen["count"] += 1
+            return data
+
+    pre_call_pipeline = GuardrailPipeline(
+        mode="pre_call",
+        steps=[PipelineStep(guardrail="gr-pre", on_pass="allow", on_fail="block")],
+    )
+    monkeypatch.setattr(
+        litellm,
+        "callbacks",
+        [
+            CountingGuardrail(
+                guardrail_name="gr-pre",
+                event_hook=GuardrailEventHooks.pre_call,
+                default_on=True,
+                run_in_parallel=True,
+            )
+        ],
+    )
+    data = {
+        "model": "m",
+        "messages": [{"role": "user", "content": "hi"}],
+        "metadata": {
+            "_guardrail_pipelines": [("request-governance", pre_call_pipeline)],
+            "_pipeline_managed_guardrails": {"gr-pre"},
+        },
+    }
+
+    await proxy_logging.pre_call_hook(
+        user_api_key_dict=make_user_api_key_auth(), data=data, call_type="completion"
+    )
+
+    assert seen["count"] == 1
 
 
 @pytest.mark.asyncio
@@ -1173,6 +1288,26 @@ async def test_pre_call_hook_rejects_streaming_request_with_post_call_pipeline(
     assert "stream=false" in info.value.detail["error"]["message"]
 
 
+@pytest.mark.asyncio
+async def test_pre_call_hook_rejects_background_request_with_post_call_pipeline(
+    proxy_logging, make_user_api_key_auth, monkeypatch
+):
+    monkeypatch.setattr(litellm, "callbacks", [])
+    data = _post_call_pipeline_data(background=True)
+
+    with pytest.raises(HTTPException) as info:
+        await proxy_logging.pre_call_hook(
+            user_api_key_dict=make_user_api_key_auth(),
+            data=data,
+            call_type="aresponses",
+            guardrails_only=True,
+        )
+
+    assert info.value.status_code == 400
+    assert info.value.detail["error"]["policies"] == ["response-governance"]
+    assert "background=false" in info.value.detail["error"]["message"]
+
+
 def test_raise_for_streaming_post_call_pipelines_ignores_non_streaming_and_pre_call():
     post_call = GuardrailPipeline(mode="post_call", steps=[PipelineStep(guardrail="g", on_fail="block")])
     pre_call = GuardrailPipeline(mode="pre_call", steps=[PipelineStep(guardrail="g", on_fail="block")])
@@ -1183,6 +1318,12 @@ def test_raise_for_streaming_post_call_pipelines_ignores_non_streaming_and_pre_c
         )
         is None
     )
+    assert (
+        _raise_for_streaming_post_call_pipelines(
+            {"background": False, "metadata": {"_guardrail_pipelines": [("p", post_call)]}}
+        )
+        is None
+    )
     assert _raise_for_streaming_post_call_pipelines({"metadata": {"_guardrail_pipelines": [("p", post_call)]}}) is None
     assert (
         _raise_for_streaming_post_call_pipelines(
@@ -1191,3 +1332,4 @@ def test_raise_for_streaming_post_call_pipelines_ignores_non_streaming_and_pre_c
         is None
     )
     assert _raise_for_streaming_post_call_pipelines({"stream": True}) is None
+    assert _raise_for_streaming_post_call_pipelines({"background": True}) is None

From c5bcf3a73594ce5fad662a47781d89dbe7955718 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 29 Aug 2026 12:06:43 -0700
Subject: [PATCH 07/85] feat(policy_engine): execute post_call guardrail
 pipelines on streaming responses

---
 .../unified_guardrail/unified_guardrail.py    |  46 ++++-
 .../proxy/policy_engine/pipeline_executor.py  |  47 ++++-
 litellm/proxy/utils.py                        | 187 ++++++++++++++++--
 .../test_unified_guardrail.py                 |   2 +-
 .../proxy_logging/test_guardrail_pipeline.py  | 168 +++++++++++++++-
 5 files changed, 417 insertions(+), 33 deletions(-)

diff --git a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py
index e95e97bfe74..60b4444e1d4 100644
--- a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py
+++ b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py
@@ -19,7 +19,7 @@ from litellm.cost_calculator import _infer_call_type
 from litellm.integrations.custom_guardrail import CustomGuardrail
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route
-from litellm.llms import load_guardrail_translation_mappings
+from litellm.llms import get_guardrail_translation_mapping, load_guardrail_translation_mappings
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.types.guardrails import GuardrailEventHooks
 from litellm.types.utils import (
@@ -62,6 +62,36 @@ def _as_endpoint_translation(translation: _EndpointTranslation) -> _EndpointTran
     return translation
 
 
+def resolve_endpoint_translation(
+    user_api_key_dict: UserAPIKeyAuth, first_response_item: object | None
+) -> "tuple[str, BaseTranslation] | None":
+    """
+    Resolve the endpoint guardrail translation for a streamed response: the
+    request route wins, falling back to inferring the call type from the first
+    response chunk (the same resolution order the streaming iterator hook uses).
+    Returns None when the call type is unresolvable or has no translation.
+    """
+    route_call_types: Final = (
+        get_call_types_for_route(user_api_key_dict.request_route) if user_api_key_dict.request_route else None
+    )
+    call_type: Final = (
+        route_call_types[0].value
+        if route_call_types
+        else (
+            _infer_call_type(call_type=None, completion_response=first_response_item)
+            if first_response_item is not None
+            else None
+        )
+    )
+    if call_type is None:
+        return None
+    try:
+        handler_cls: Final = get_guardrail_translation_mapping(CallTypes(call_type))
+    except ValueError:
+        return None
+    return call_type, handler_cls()
+
+
 def _chunk_choices(item: object) -> Sequence[object]:
     choices: Final[Sequence[object]] = getattr(item, "choices", None) or []
     return choices
@@ -346,7 +376,7 @@ class UnifiedLLMGuardrails(CustomLogger):
 
         return response
 
-    async def _handle_streaming_block(
+    async def handle_streaming_block(
         self,
         exc: "ModifyResponseException",
         endpoint_translation: _EndpointTranslation,
@@ -402,7 +432,7 @@ class UnifiedLLMGuardrails(CustomLogger):
             return None
         return call_type
 
-    async def _emit_streaming_http_error(
+    async def emit_streaming_http_error(
         self,
         exc: HTTPException,
         call_type: str | None,
@@ -577,7 +607,7 @@ class UnifiedLLMGuardrails(CustomLogger):
         except ModifyResponseException as e:
             if e.original_response is None:
                 e.original_response = responses_so_far
-            async for block_chunk in self._handle_streaming_block(
+            async for block_chunk in self.handle_streaming_block(
                 e,
                 endpoint_translation,
                 stream_started=bool(responses_yielded),
@@ -586,7 +616,7 @@ class UnifiedLLMGuardrails(CustomLogger):
                 yield block_chunk
             raise _StreamTerminated()
         except HTTPException as e:
-            async for error_item in self._emit_streaming_http_error(e, call_type, responses_so_far, request_data):
+            async for error_item in self.emit_streaming_http_error(e, call_type, responses_so_far, request_data):
                 yield error_item
             raise _StreamTerminated()
 
@@ -758,7 +788,7 @@ class UnifiedLLMGuardrails(CustomLogger):
         except ModifyResponseException as e:
             if e.original_response is None:
                 e.original_response = responses_so_far
-            async for block_chunk in self._handle_streaming_block(
+            async for block_chunk in self.handle_streaming_block(
                 e,
                 endpoint_translation,
                 stream_started=bool(responses_yielded),
@@ -1060,7 +1090,7 @@ class UnifiedLLMGuardrails(CustomLogger):
                     # The current chunk was appended to responses_so_far but not
                     # yet yielded, so exclude it: the continuation must reflect
                     # only what the client has actually received.
-                    async for block_chunk in self._handle_streaming_block(
+                    async for block_chunk in self.handle_streaming_block(
                         e,
                         endpoint_translation,
                         stream_started=chunks_yielded,
@@ -1124,7 +1154,7 @@ class UnifiedLLMGuardrails(CustomLogger):
                 # terminating SSE sequence with the block message rather than
                 # propagating into a bare error blob that truncates the stream.
                 # The withheld original chunks are never released.
-                async for block_chunk in self._handle_streaming_block(
+                async for block_chunk in self.handle_streaming_block(
                     e,
                     endpoint_translation,
                     stream_started=bool(responses_yielded),
diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py
index 190784a2a60..c422a7c0964 100644
--- a/litellm/proxy/policy_engine/pipeline_executor.py
+++ b/litellm/proxy/policy_engine/pipeline_executor.py
@@ -6,7 +6,7 @@ pass/fail actions (allow, block, next, modify_response) and data forwarding.
 """
 
 import time
-from typing import Any, Final, Literal
+from typing import TYPE_CHECKING, Any, Final, Literal
 
 import litellm
 from litellm._logging import verbose_proxy_logger
@@ -25,6 +25,11 @@ from litellm.types.proxy.policy_engine.pipeline_types import (
     PipelineStepResult,
 )
 
+if TYPE_CHECKING:
+    from litellm.llms.base_llm.guardrail_translation.base_translation import (
+        BaseTranslation,
+    )
+
 try:
     from fastapi.exceptions import HTTPException
 except ImportError:
@@ -43,6 +48,8 @@ class PipelineExecutor:
         call_type: str,
         policy_name: str,
         raw_request_snapshot: dict | None = None,  # mutable-ok: same request-payload shape as data
+        streaming_chunks: list[Any] | None = None,  # mutable-ok: shared buffered-stream chunks, read per step
+        endpoint_translation: "BaseTranslation | None" = None,
     ) -> PipelineExecutionResult:
         """
         Execute pipeline steps sequentially with conditional actions.
@@ -59,6 +66,12 @@ class PipelineExecutor:
                 step whose guardrail opted into ``scan_raw_request`` evaluates
                 the original request instead of whatever an earlier
                 ``pass_data`` step in this same pipeline already rewrote.
+            streaming_chunks: buffered chunks of a completed stream. When set
+                (with ``endpoint_translation``), post_call steps scan the
+                assembled streamed output through the endpoint translation
+                instead of calling ``async_post_call_success_hook``.
+            endpoint_translation: the guardrail translation for the streamed
+                endpoint, resolved by the caller.
 
         Returns:
             PipelineExecutionResult with terminal action and step results
@@ -83,6 +96,8 @@ class PipelineExecutor:
                 user_api_key_dict=user_api_key_dict,
                 call_type=call_type,
                 raw_request_snapshot=raw_request_snapshot,
+                streaming_chunks=streaming_chunks,
+                endpoint_translation=endpoint_translation,
             )
 
             duration = time.perf_counter() - start_time
@@ -154,6 +169,8 @@ class PipelineExecutor:
         user_api_key_dict: Any,
         call_type: str,
         raw_request_snapshot: dict | None = None,  # mutable-ok: same request-payload shape as data
+        streaming_chunks: list[Any] | None = None,  # mutable-ok: shared buffered-stream chunks, read per step
+        endpoint_translation: "BaseTranslation | None" = None,
     ) -> tuple[
         Literal["pass", "fail", "error"],
         dict | None,
@@ -198,10 +215,8 @@ class PipelineExecutor:
 
             # Use unified_guardrail path if callback implements apply_guardrail
             target: CustomLogger = callback
-            use_unified: Final = (
-                "apply_guardrail" in type(callback).__dict__ and not callback.use_native_lifecycle_hooks
-            )
-            if use_unified:
+            use_unified: Final = PipelineExecutor.supports_unified_execution(callback)
+            if use_unified and streaming_chunks is None:
                 hook_input["guardrail_to_apply"] = callback
                 target = UnifiedLLMGuardrails()
 
@@ -216,6 +231,22 @@ class PipelineExecutor:
                     callback.mark_pre_call_hook_ran(data)
                     if isinstance(response, dict):
                         callback.mark_pre_call_hook_ran(response)
+            elif mode == "post_call" and streaming_chunks is not None:
+                if not use_unified or endpoint_translation is None:
+                    return (
+                        "error",
+                        None,
+                        f"Guardrail '{step.guardrail}' does not support streaming pipeline execution",
+                        None,
+                    )
+                await endpoint_translation.process_output_streaming_response(
+                    responses_so_far=streaming_chunks,
+                    guardrail_to_apply=callback,
+                    litellm_logging_obj=data.get("litellm_logging_obj"),
+                    user_api_key_dict=user_api_key_dict,
+                    request_data=hook_input,
+                )
+                response = None
             elif mode == "post_call":
                 response = await target.async_post_call_success_hook(
                     user_api_key_dict=user_api_key_dict,
@@ -246,6 +277,12 @@ class PipelineExecutor:
                 verbose_proxy_logger.error("Pipeline: unexpected error from guardrail '%s': %s", step.guardrail, e)
                 return ("error", None, str(e), e)
 
+    @staticmethod
+    def supports_unified_execution(callback: CustomGuardrail) -> bool:
+        """Whether this guardrail runs through the unified apply_guardrail path,
+        the interface streaming pipeline execution requires."""
+        return "apply_guardrail" in type(callback).__dict__ and not callback.use_native_lifecycle_hooks
+
     @staticmethod
     def find_guardrail_callback(guardrail_name: str) -> CustomGuardrail | None:
         """Look up an initialized guardrail callback by name from litellm.callbacks."""
diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index 80e991640a9..5642eb5383e 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -19,7 +19,7 @@ from email.mime.text import MIMEText
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, Protocol, TypeVar, Union, cast, overload
 
-from typing_extensions import ReadOnly, TypedDict
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 from litellm import _custom_logger_compatible_callbacks_literal
 from litellm.constants import (
@@ -486,29 +486,80 @@ def _merge_pipeline_metadata_writes(data: dict, modified_data: Mapping[str, obje
         _merge_pipeline_metadata_bucket(data, bucket_key, modified_data.get(bucket_key))
 
 
+def _pipeline_step_supports_streaming(guardrail_name: str) -> bool:
+    callback: Final = PipelineExecutor.find_guardrail_callback(guardrail_name)
+    return callback is not None and PipelineExecutor.supports_unified_execution(callback)
+
+
+class _PipelineErrorBody(TypedDict):
+    message: ReadOnly[str]
+    type: ReadOnly[str]
+    policies: ReadOnly[tuple[str, ...]]
+    guardrails: NotRequired[ReadOnly[tuple[str, ...]]]
+
+
+class _PipelineErrorDetail(TypedDict):
+    error: ReadOnly[_PipelineErrorBody]
+
+
 def _raise_for_streaming_post_call_pipelines(data: Mapping[str, object]) -> None:
-    if data.get("stream") is not True and data.get("background") is not True:
+    """
+    Reject up front the requests whose post_call pipelines could never run.
+
+    Background responses skip the post_call hooks entirely, so a pipeline
+    governing one would silently never execute. Streaming responses execute
+    pipelines against the buffered stream through the endpoint guardrail
+    translations, which requires every step's guardrail to support the unified
+    apply_guardrail interface; steps that cannot (native-lifecycle guardrails,
+    or guardrails not registered at all) keep the 400 rather than letting
+    ungoverned output stream through.
+    """
+    is_stream: Final = data.get("stream") is True
+    is_background: Final = data.get("background") is True
+    if not is_stream and not is_background:
         return
-    post_call_policies: Final = tuple(
-        policy_name for policy_name, pipeline in _policy_pipelines(data) if pipeline.mode == "post_call"
+    post_call_pipelines: Final = tuple(
+        (policy_name, pipeline) for policy_name, pipeline in _policy_pipelines(data) if pipeline.mode == "post_call"
     )
-    if not post_call_policies:
+    if not post_call_pipelines:
         return
-    raise HTTPException(
-        status_code=400,
-        detail={
+    post_call_policies: Final = tuple(policy_name for policy_name, _pipeline in post_call_pipelines)
+    if is_background:
+        background_detail: Final[_PipelineErrorDetail] = {
             "error": {
                 "message": (
-                    "Policies with post_call guardrail pipelines cannot govern streaming or background "
-                    f"responses yet: {', '.join(post_call_policies)}. Retry with stream=false and "
-                    "background=false, or move these policies' output guardrails from pipeline steps to "
-                    "guardrails.add, which scans streamed output."
+                    "Policies with post_call guardrail pipelines cannot govern background "
+                    f"responses: {', '.join(post_call_policies)}. Retry with background=false."
                 ),
                 "type": "guardrail_pipeline_error",
-                "policies": list(post_call_policies),
+                "policies": post_call_policies,
             }
-        },
+        }
+        raise HTTPException(status_code=400, detail=background_detail)
+    unsupported_guardrails: Final = tuple(
+        dict.fromkeys(
+            step.guardrail
+            for _policy_name, pipeline in post_call_pipelines
+            for step in pipeline.steps
+            if not _pipeline_step_supports_streaming(step.guardrail)
+        )
     )
+    if not unsupported_guardrails:
+        return
+    unsupported_detail: Final[_PipelineErrorDetail] = {
+        "error": {
+            "message": (
+                "Policies with post_call guardrail pipelines cannot govern streaming responses "
+                "because these pipeline guardrails do not support the unified apply_guardrail "
+                f"interface: {', '.join(unsupported_guardrails)}. Retry with stream=false, or move "
+                "them from pipeline steps to guardrails.add, which scans streamed output."
+            ),
+            "type": "guardrail_pipeline_error",
+            "policies": post_call_policies,
+            "guardrails": unsupported_guardrails,
+        }
+    }
+    raise HTTPException(status_code=400, detail=unsupported_detail)
 
 
 def _prompt_block_text(block: object) -> str:
@@ -1689,7 +1740,7 @@ class ProxyLogging:
         result: PipelineExecutionResult,
         data: dict,
         policy_name: str,
-        original_response: LLMResponseTypes | None = None,
+        original_response: "LLMResponseTypes | Sequence[object] | None" = None,
     ) -> dict:
         """
         Handle a PipelineExecutionResult — allow, block, or modify_response.
@@ -1699,7 +1750,9 @@ class ProxyLogging:
         payload (already sent upstream) must stay untouched; a replacement
         response carried in ``modified_data`` is adopted by the caller, and
         metadata-bucket writes (applied guardrails, guardrail logging info)
-        are merged back so headers and spend logs still see them.
+        are merged back so headers and spend logs still see them. On the
+        streaming path it is the buffered chunk list, carried into
+        ``ModifyResponseException.original_response`` for usage reporting.
         """
         if result.terminal_action == "allow":
             if result.modified_data is not None:
@@ -3195,11 +3248,16 @@ class ProxyLogging:
             # dict lookups + llm_router.get_deployment() per callback per chunk.
             _cached_guardrail_data: dict | None = None
             _guardrail_data_computed = False
+            pipeline_managed: Final = (
+                _pipeline_managed_guardrail_names(data, "post_call") if caps.has_guardrail else frozenset()
+            )
 
             for callback in litellm.callbacks:
                 try:
                     _callback: CustomLogger | None = None
                     if isinstance(callback, CustomGuardrail):
+                        if callback.guardrail_name in pipeline_managed:
+                            continue
                         # Main - V2 Guardrails implementation
                         from litellm.types.guardrails import GuardrailEventHooks
 
@@ -3256,12 +3314,17 @@ class ProxyLogging:
         1. /chat/completions
         """
         caps: Final = ProxyLogging._callback_capabilities()
+        post_call_pipelines: Final = tuple(
+            (policy_name, pipeline)
+            for policy_name, pipeline in _policy_pipelines(request_data)
+            if pipeline.mode == "post_call"
+        )
         # Fast path: no real overrides. Internal proxy CustomLogger callbacks
         # (e.g. _PROXY_MaxBudgetLimiter, ManagedFiles) inherit the default
         # ``async for chunk: yield chunk`` body, so wrapping the iterator
         # through each of them adds N pass-through trampolines per chunk for
         # zero behavior change. Skip the chain entirely and stream through.
-        if not caps.iterator_overrides:
+        if not caps.iterator_overrides and not post_call_pipelines:
             try:
                 async for chunk in response:
                     yield chunk
@@ -3281,8 +3344,11 @@ class ProxyLogging:
         current_response = response
         stream_needs_translation: Final = ProxyLogging._stream_requires_guardrail_translation(user_api_key_dict)
 
+        pipeline_managed_names: Final = _pipeline_managed_guardrail_names(request_data, "post_call")
         for resolved_callback, kind in caps.iterator_overrides:
             if isinstance(resolved_callback, CustomGuardrail):
+                if resolved_callback.guardrail_name in pipeline_managed_names:
+                    continue
                 if (
                     resolved_callback.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call)
                     is not True
@@ -3322,6 +3388,17 @@ class ProxyLogging:
                     ),
                 )
 
+        # Policy pipelines run last, over the fully buffered stream, so a
+        # pipeline verdict covers whatever the flat guardrail chain above
+        # already let through.
+        if post_call_pipelines:
+            current_response = self._pipeline_gated_stream(
+                response=current_response,
+                user_api_key_dict=user_api_key_dict,
+                request_data=request_data,
+                pipelines=post_call_pipelines,
+            )
+
         try:
             async for chunk in current_response:
                 yield chunk
@@ -3337,6 +3414,82 @@ class ProxyLogging:
         # we reach this point the metadata is fully populated.
         ProxyLogging._fire_deferred_stream_logging(request_data)
 
+    async def _pipeline_gated_stream(
+        self,
+        response: "AsyncGenerator[object, None]",
+        user_api_key_dict: UserAPIKeyAuth,
+        request_data: dict,
+        pipelines: "tuple[tuple[str, GuardrailPipeline], ...]",
+    ) -> "AsyncGenerator[Any, None]":
+        """
+        Execute post_call policy pipelines against a streamed response.
+
+        Buffers the whole stream (nothing reaches the client until every
+        pipeline allows it), then runs each pipeline's steps against the
+        assembled output through the endpoint guardrail translation, the same
+        machinery flat post_call guardrails use at end of stream. An allow
+        releases the buffered chunks verbatim; a block or modify_response
+        terminates with the translation's block chunks or the raised error.
+        """
+        from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import (
+            resolve_endpoint_translation,
+        )
+
+        buffered: Final[list[object]] = []  # mutable-ok: accumulates the stream before the pipeline verdict
+        async for item in response:
+            buffered.append(item)
+        if not buffered:
+            return
+
+        resolved: Final = resolve_endpoint_translation(user_api_key_dict, buffered[0])
+        if resolved is None:
+            policy_names: Final = tuple(policy_name for policy_name, _pipeline in pipelines)
+            unresolvable_detail: Final[_PipelineErrorDetail] = {
+                "error": {
+                    "message": (
+                        "Policy pipelines could not govern this streaming response shape; "
+                        f"the response was withheld: {', '.join(policy_names)}."
+                    ),
+                    "type": "guardrail_pipeline_error",
+                    "policies": policy_names,
+                }
+            }
+            raise HTTPException(status_code=500, detail=unresolvable_detail)
+        call_type, endpoint_translation = resolved
+
+        for policy_name, pipeline in pipelines:
+            result: PipelineExecutionResult = await PipelineExecutor.execute_steps(
+                steps=pipeline.steps,
+                mode="post_call",
+                data=request_data,
+                user_api_key_dict=user_api_key_dict,
+                call_type=call_type,
+                policy_name=policy_name,
+                streaming_chunks=buffered,
+                endpoint_translation=endpoint_translation,
+            )
+            try:
+                ProxyLogging._handle_pipeline_result(
+                    result, data=request_data, policy_name=policy_name, original_response=buffered
+                )
+            except ModifyResponseException as e:
+                if e.original_response is None:
+                    e.original_response = buffered
+                async for block_chunk in unified_guardrail.handle_streaming_block(
+                    e, endpoint_translation, stream_started=False, responses_so_far=()
+                ):
+                    yield block_chunk
+                return
+            except HTTPException as e:
+                async for error_chunk in unified_guardrail.emit_streaming_http_error(
+                    e, call_type, buffered, request_data
+                ):
+                    yield error_chunk
+                return
+
+        for buffered_item in buffered:
+            yield buffered_item
+
     @staticmethod
     def _fire_deferred_stream_logging(request_data: dict) -> None:
         """
diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py
index 8b9ecfbbeee..fe4acbf8277 100644
--- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py
+++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/unified_guardrails/test_unified_guardrail.py
@@ -1026,7 +1026,7 @@ class TestStreamingTransform:
         )
 
         emitted = []
-        async for item in handler._emit_streaming_http_error(
+        async for item in handler.emit_streaming_http_error(
             exc,
             call_type=CallTypes.asend_message.value,
             responses_so_far=[{"id": "req-1"}],
diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
index 34a25d75bc0..2b36ae111dc 100644
--- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
+++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
@@ -1284,7 +1284,8 @@ async def test_pre_call_hook_rejects_streaming_request_with_post_call_pipeline(
         )
 
     assert info.value.status_code == 400
-    assert info.value.detail["error"]["policies"] == ["response-governance"]
+    assert info.value.detail["error"]["policies"] == ("response-governance",)
+    assert info.value.detail["error"]["guardrails"] == ("gr-post",)
     assert "stream=false" in info.value.detail["error"]["message"]
 
 
@@ -1304,7 +1305,7 @@ async def test_pre_call_hook_rejects_background_request_with_post_call_pipeline(
         )
 
     assert info.value.status_code == 400
-    assert info.value.detail["error"]["policies"] == ["response-governance"]
+    assert info.value.detail["error"]["policies"] == ("response-governance",)
     assert "background=false" in info.value.detail["error"]["message"]
 
 
@@ -1333,3 +1334,166 @@ def test_raise_for_streaming_post_call_pipelines_ignores_non_streaming_and_pre_c
     )
     assert _raise_for_streaming_post_call_pipelines({"stream": True}) is None
     assert _raise_for_streaming_post_call_pipelines({"background": True}) is None
+
+
+# ---------------------------------------------------------------------------
+# post_call pipelines on streaming responses
+# ---------------------------------------------------------------------------
+
+
+def _unified_stream_guardrail(seen: Dict[str, Any], block: bool = False) -> CustomGuardrail:
+    class UnifiedStreamGuardrail(CustomGuardrail):
+        async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None):
+            seen["count"] = seen.get("count", 0) + 1
+            seen["input_type"] = input_type
+            if block:
+                raise HTTPException(status_code=400, detail={"error": "output blocked"})
+            return inputs
+
+    return UnifiedStreamGuardrail(guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=False)
+
+
+def _stream_chunks() -> List[Any]:
+    return [
+        litellm.ModelResponseStream(choices=[{"index": 0, "delta": {"content": "hello "}, "finish_reason": None}]),
+        litellm.ModelResponseStream(choices=[{"index": 0, "delta": {"content": "world"}, "finish_reason": "stop"}]),
+    ]
+
+
+async def _async_chunk_iter(chunks: List[Any]):
+    for chunk in chunks:
+        yield chunk
+
+
+@pytest.mark.asyncio
+async def test_pre_call_hook_allows_streaming_when_pipeline_guardrail_supports_unified(
+    proxy_logging, make_user_api_key_auth, monkeypatch
+):
+    seen: Dict[str, Any] = {}
+    monkeypatch.setattr(litellm, "callbacks", [_unified_stream_guardrail(seen)])
+    data = _post_call_pipeline_data(stream=True)
+
+    out = await proxy_logging.pre_call_hook(
+        user_api_key_dict=make_user_api_key_auth(),
+        data=data,
+        call_type="completion",
+        guardrails_only=True,
+    )
+
+    assert out is not None
+    assert out.get("stream") is True
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("native_lifecycle", [False, True])
+async def test_pre_call_hook_rejects_streaming_when_pipeline_guardrail_lacks_unified_support(
+    proxy_logging, make_user_api_key_auth, monkeypatch, native_lifecycle
+):
+    if native_lifecycle:
+
+        class NativeOnlyGuardrail(CustomGuardrail):
+            use_native_lifecycle_hooks = True
+
+            async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None):
+                return inputs
+
+    else:
+
+        class NativeOnlyGuardrail(CustomGuardrail):
+            pass
+
+    monkeypatch.setattr(
+        litellm,
+        "callbacks",
+        [NativeOnlyGuardrail(guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=False)],
+    )
+    data = _post_call_pipeline_data(stream=True)
+
+    with pytest.raises(HTTPException) as info:
+        await proxy_logging.pre_call_hook(
+            user_api_key_dict=make_user_api_key_auth(),
+            data=data,
+            call_type="completion",
+            guardrails_only=True,
+        )
+
+    assert info.value.status_code == 400
+    assert info.value.detail["error"]["guardrails"] == ("gr-post",)
+    assert "apply_guardrail" in info.value.detail["error"]["message"]
+
+
+@pytest.mark.asyncio
+async def test_streaming_iterator_hook_pipeline_allow_releases_buffered_chunks(
+    proxy_logging, make_user_api_key_auth, monkeypatch
+):
+    seen: Dict[str, Any] = {}
+    monkeypatch.setattr(litellm, "callbacks", [_unified_stream_guardrail(seen)])
+    monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
+    data = _post_call_pipeline_data(stream=True)
+    chunks = _stream_chunks()
+
+    delivered = [
+        item
+        async for item in proxy_logging.async_post_call_streaming_iterator_hook(
+            user_api_key_dict=make_user_api_key_auth(request_route="/v1/chat/completions"),
+            response=_async_chunk_iter(chunks),
+            request_data=data,
+        )
+    ]
+
+    assert [id(item) for item in delivered] == [id(chunk) for chunk in chunks]
+    assert seen["count"] == 1
+    assert seen["input_type"] == "response"
+
+
+@pytest.mark.asyncio
+async def test_streaming_iterator_hook_pipeline_block_withholds_all_chunks(
+    proxy_logging, make_user_api_key_auth, monkeypatch
+):
+    seen: Dict[str, Any] = {}
+    monkeypatch.setattr(litellm, "callbacks", [_unified_stream_guardrail(seen, block=True)])
+    monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
+    data = _post_call_pipeline_data(stream=True)
+    delivered: List[Any] = []
+
+    async def _drain() -> None:
+        async for item in proxy_logging.async_post_call_streaming_iterator_hook(
+            user_api_key_dict=make_user_api_key_auth(request_route="/v1/chat/completions"),
+            response=_async_chunk_iter(_stream_chunks()),
+            request_data=data,
+        ):
+            delivered.append(item)
+
+    with pytest.raises(HTTPException) as info:
+        await _drain()
+
+    assert delivered == []
+    assert info.value.status_code == 400
+    assert "output blocked" in str(info.value.detail)
+
+
+@pytest.mark.asyncio
+async def test_streaming_iterator_hook_pipeline_withholds_unresolvable_response_shape(
+    proxy_logging, make_user_api_key_auth, monkeypatch
+):
+    seen: Dict[str, Any] = {}
+    monkeypatch.setattr(litellm, "callbacks", [_unified_stream_guardrail(seen)])
+    monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
+    data = _post_call_pipeline_data(stream=True)
+    delivered: List[Any] = []
+
+    async def _drain() -> None:
+        async for item in proxy_logging.async_post_call_streaming_iterator_hook(
+            user_api_key_dict=make_user_api_key_auth(),
+            response=_async_chunk_iter([object(), object()]),
+            request_data=data,
+        ):
+            delivered.append(item)
+
+    with pytest.raises(HTTPException) as info:
+        await _drain()
+
+    assert delivered == []
+    assert info.value.status_code == 500
+    assert "withheld" in info.value.detail["error"]["message"]
+    assert seen.get("count") is None

From fa5a10941e713968fbe7251a99d867ef0050dd69 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 29 Aug 2026 13:28:11 -0700
Subject: [PATCH 08/85] fix(policy_engine): fail closed on streaming for
 content-rewriting pipeline steps and untranslatable routes

---
 .../unified_guardrail/unified_guardrail.py    |  19 ++-
 litellm/proxy/utils.py                        | 111 ++++++++++++------
 .../proxy_logging/test_guardrail_pipeline.py  |  84 +++++++++++--
 3 files changed, 155 insertions(+), 59 deletions(-)

diff --git a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py
index 60b4444e1d4..89527c05eb6 100644
--- a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py
+++ b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py
@@ -876,6 +876,14 @@ class UnifiedLLMGuardrails(CustomLogger):
         choices: Final = _chunk_choices(item)
         return any(getattr(choice, "finish_reason", None) is not None for choice in choices)
 
+    def resolve_streaming_flag(self, guardrail_to_apply: CustomGuardrail | None, name: str, default: object) -> object:
+        """Streaming flag resolution order (later wins): default < guardrail
+        attribute < guardrail_config dict < this callback's optional_params."""
+        attribute_value: Final = default if guardrail_to_apply is None else getattr(guardrail_to_apply, name, default)
+        config: Final = None if guardrail_to_apply is None else getattr(guardrail_to_apply, "guardrail_config", None)
+        config_value: Final = config.get(name, attribute_value) if isinstance(config, dict) else attribute_value
+        return self.optional_params.get(name, config_value)
+
     async def async_post_call_streaming_iterator_hook(
         self,
         user_api_key_dict: UserAPIKeyAuth,
@@ -906,17 +914,8 @@ class UnifiedLLMGuardrails(CustomLogger):
         if guardrail_to_apply is None:
             guardrail_to_apply = request_data.pop("guardrail_to_apply", None)
 
-        # Get streaming configuration. Resolution order (later wins): default
-        # < guardrail attribute < guardrail_config dict < this callback's
-        # optional_params.
         def _streaming_flag(name: str, default: object) -> Any:
-            value = default
-            if guardrail_to_apply is not None:
-                value = getattr(guardrail_to_apply, name, value)
-                config: Final[Mapping[str, object]] = getattr(guardrail_to_apply, "guardrail_config", {})
-                if isinstance(config, dict):
-                    value = config.get(name, value)
-            return self.optional_params.get(name, value)
+            return self.resolve_streaming_flag(guardrail_to_apply, name, default)
 
         sampling_rate: Final[int] = _streaming_flag("streaming_sampling_rate", 5)
         # Only apply the guardrail at end of stream (not per chunk).
diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index 5642eb5383e..8b2c38d457f 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -139,6 +139,7 @@ from litellm.proxy.db.token_auth import (
 )
 from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import (
     UnifiedLLMGuardrails,
+    resolve_endpoint_translation,
 )
 from litellm.proxy.hooks import PROXY_HOOKS, get_proxy_hook
 from litellm.proxy.hooks.cache_control_check import _PROXY_CacheControlCheck
@@ -486,11 +487,19 @@ def _merge_pipeline_metadata_writes(data: dict, modified_data: Mapping[str, obje
         _merge_pipeline_metadata_bucket(data, bucket_key, modified_data.get(bucket_key))
 
 
-def _pipeline_step_supports_streaming(guardrail_name: str) -> bool:
+def _pipeline_step_supports_unified_streaming(guardrail_name: str) -> bool:
     callback: Final = PipelineExecutor.find_guardrail_callback(guardrail_name)
     return callback is not None and PipelineExecutor.supports_unified_execution(callback)
 
 
+def _pipeline_step_rewrites_streamed_content(guardrail_name: str) -> bool:
+    callback: Final = PipelineExecutor.find_guardrail_callback(guardrail_name)
+    if callback is None:
+        return False
+    transform_mode: Final = unified_guardrail.resolve_streaming_flag(callback, "streaming_transform_mode", "block_only")
+    return callback.mask_response_content or transform_mode == "incremental_diff"
+
+
 class _PipelineErrorBody(TypedDict):
     message: ReadOnly[str]
     type: ReadOnly[str]
@@ -502,17 +511,20 @@ class _PipelineErrorDetail(TypedDict):
     error: ReadOnly[_PipelineErrorBody]
 
 
-def _raise_for_streaming_post_call_pipelines(data: Mapping[str, object]) -> None:
+def _raise_for_streaming_post_call_pipelines(data: Mapping[str, object], user_api_key_dict: UserAPIKeyAuth) -> None:
     """
     Reject up front the requests whose post_call pipelines could never run.
 
     Background responses skip the post_call hooks entirely, so a pipeline
     governing one would silently never execute. Streaming responses execute
     pipelines against the buffered stream through the endpoint guardrail
-    translations, which requires every step's guardrail to support the unified
-    apply_guardrail interface; steps that cannot (native-lifecycle guardrails,
-    or guardrails not registered at all) keep the 400 rather than letting
-    ungoverned output stream through.
+    translation of the request route, releasing the buffered chunks verbatim
+    on allow. That needs every step's guardrail to support the unified
+    apply_guardrail interface and to only allow or block (a step that rewrites
+    streamed content, via mask_response_content or
+    streaming_transform_mode=incremental_diff, would have its rewrite silently
+    dropped), and needs the route to have a translation at all; anything else
+    keeps the 400 rather than letting ungoverned output stream through.
     """
     is_stream: Final = data.get("stream") is True
     is_background: Final = data.get("background") is True
@@ -536,30 +548,61 @@ def _raise_for_streaming_post_call_pipelines(data: Mapping[str, object]) -> None
             }
         }
         raise HTTPException(status_code=400, detail=background_detail)
-    unsupported_guardrails: Final = tuple(
-        dict.fromkeys(
-            step.guardrail
-            for _policy_name, pipeline in post_call_pipelines
-            for step in pipeline.steps
-            if not _pipeline_step_supports_streaming(step.guardrail)
-        )
+    step_guardrails: Final = tuple(
+        dict.fromkeys(step.guardrail for _policy_name, pipeline in post_call_pipelines for step in pipeline.steps)
     )
-    if not unsupported_guardrails:
+    unsupported_guardrails: Final = tuple(
+        guardrail for guardrail in step_guardrails if not _pipeline_step_supports_unified_streaming(guardrail)
+    )
+    if unsupported_guardrails:
+        unsupported_detail: Final[_PipelineErrorDetail] = {
+            "error": {
+                "message": (
+                    "Policies with post_call guardrail pipelines cannot govern streaming responses "
+                    "because these pipeline guardrails do not support the unified apply_guardrail "
+                    f"interface: {', '.join(unsupported_guardrails)}. Retry with stream=false, or drop "
+                    "them from the pipeline steps so guardrails.add scans them on streamed output."
+                ),
+                "type": "guardrail_pipeline_error",
+                "policies": post_call_policies,
+                "guardrails": unsupported_guardrails,
+            }
+        }
+        raise HTTPException(status_code=400, detail=unsupported_detail)
+    rewriting_guardrails: Final = tuple(
+        guardrail for guardrail in step_guardrails if _pipeline_step_rewrites_streamed_content(guardrail)
+    )
+    if rewriting_guardrails:
+        rewriting_detail: Final[_PipelineErrorDetail] = {
+            "error": {
+                "message": (
+                    "Policies with post_call guardrail pipelines cannot govern streaming responses "
+                    "because these pipeline guardrails rewrite streamed content (mask_response_content "
+                    "or streaming_transform_mode=incremental_diff), which pipeline steps would release "
+                    f"unmodified: {', '.join(rewriting_guardrails)}. Retry with stream=false, or drop "
+                    "them from the pipeline steps so guardrails.add applies them to streamed output."
+                ),
+                "type": "guardrail_pipeline_error",
+                "policies": post_call_policies,
+                "guardrails": rewriting_guardrails,
+            }
+        }
+        raise HTTPException(status_code=400, detail=rewriting_detail)
+    route: Final = user_api_key_dict.request_route
+    if not route or resolve_endpoint_translation(user_api_key_dict, None) is not None:
         return
-    unsupported_detail: Final[_PipelineErrorDetail] = {
+    route_detail: Final[_PipelineErrorDetail] = {
         "error": {
             "message": (
-                "Policies with post_call guardrail pipelines cannot govern streaming responses "
-                "because these pipeline guardrails do not support the unified apply_guardrail "
-                f"interface: {', '.join(unsupported_guardrails)}. Retry with stream=false, or move "
-                "them from pipeline steps to guardrails.add, which scans streamed output."
+                "Policies with post_call guardrail pipelines cannot govern streaming responses on "
+                f"route {route} because it has no endpoint guardrail translation to scan the stream "
+                f"through: {', '.join(post_call_policies)}. Retry with stream=false."
             ),
             "type": "guardrail_pipeline_error",
             "policies": post_call_policies,
-            "guardrails": unsupported_guardrails,
         }
     }
-    raise HTTPException(status_code=400, detail=unsupported_detail)
+    raise HTTPException(status_code=400, detail=route_detail)
 
 
 def _prompt_block_text(block: object) -> str:
@@ -1915,7 +1958,7 @@ class ProxyLogging:
         )
 
         try:
-            _raise_for_streaming_post_call_pipelines(data)
+            _raise_for_streaming_post_call_pipelines(data, user_api_key_dict)
 
             # Execute guardrail pipelines before the normal callback loop
             data, _ = await self._maybe_execute_pipelines(
@@ -3431,10 +3474,6 @@ class ProxyLogging:
         releases the buffered chunks verbatim; a block or modify_response
         terminates with the translation's block chunks or the raised error.
         """
-        from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import (
-            resolve_endpoint_translation,
-        )
-
         buffered: Final[list[object]] = []  # mutable-ok: accumulates the stream before the pipeline verdict
         async for item in response:
             buffered.append(item)
@@ -3444,17 +3483,15 @@ class ProxyLogging:
         resolved: Final = resolve_endpoint_translation(user_api_key_dict, buffered[0])
         if resolved is None:
             policy_names: Final = tuple(policy_name for policy_name, _pipeline in pipelines)
-            unresolvable_detail: Final[_PipelineErrorDetail] = {
-                "error": {
-                    "message": (
-                        "Policy pipelines could not govern this streaming response shape; "
-                        f"the response was withheld: {', '.join(policy_names)}."
-                    ),
-                    "type": "guardrail_pipeline_error",
-                    "policies": policy_names,
-                }
-            }
-            raise HTTPException(status_code=500, detail=unresolvable_detail)
+            raise ProxyException(
+                message=(
+                    "Policy pipelines could not govern this streaming response shape; "
+                    f"the response was withheld: {', '.join(policy_names)}."
+                ),
+                type="guardrail_pipeline_error",
+                param=None,
+                code=500,
+            )
         call_type, endpoint_translation = resolved
 
         for policy_name, pipeline in pipelines:
diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
index 2b36ae111dc..6c90637158e 100644
--- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
+++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
@@ -23,6 +23,7 @@ from litellm.integrations.custom_guardrail import (
     ModifyResponseException,
 )
 from litellm.integrations.prometheus import PrometheusLogger
+from litellm.proxy._types import ProxyException
 from litellm.proxy.common_utils.callback_utils import add_guardrail_to_applied_guardrails_header
 from litellm.proxy.utils import ProxyLogging, _raise_for_streaming_post_call_pipelines
 from litellm.types.guardrails import GuardrailEventHooks
@@ -1309,31 +1310,35 @@ async def test_pre_call_hook_rejects_background_request_with_post_call_pipeline(
     assert "background=false" in info.value.detail["error"]["message"]
 
 
-def test_raise_for_streaming_post_call_pipelines_ignores_non_streaming_and_pre_call():
+def test_raise_for_streaming_post_call_pipelines_ignores_non_streaming_and_pre_call(make_user_api_key_auth):
     post_call = GuardrailPipeline(mode="post_call", steps=[PipelineStep(guardrail="g", on_fail="block")])
     pre_call = GuardrailPipeline(mode="pre_call", steps=[PipelineStep(guardrail="g", on_fail="block")])
+    auth = make_user_api_key_auth(request_route="/custom/stream")
 
     assert (
         _raise_for_streaming_post_call_pipelines(
-            {"stream": False, "metadata": {"_guardrail_pipelines": [("p", post_call)]}}
+            {"stream": False, "metadata": {"_guardrail_pipelines": [("p", post_call)]}}, auth
         )
         is None
     )
     assert (
         _raise_for_streaming_post_call_pipelines(
-            {"background": False, "metadata": {"_guardrail_pipelines": [("p", post_call)]}}
+            {"background": False, "metadata": {"_guardrail_pipelines": [("p", post_call)]}}, auth
         )
         is None
     )
-    assert _raise_for_streaming_post_call_pipelines({"metadata": {"_guardrail_pipelines": [("p", post_call)]}}) is None
+    assert (
+        _raise_for_streaming_post_call_pipelines({"metadata": {"_guardrail_pipelines": [("p", post_call)]}}, auth)
+        is None
+    )
     assert (
         _raise_for_streaming_post_call_pipelines(
-            {"stream": True, "metadata": {"_guardrail_pipelines": [("p", pre_call)]}}
+            {"stream": True, "metadata": {"_guardrail_pipelines": [("p", pre_call)]}}, auth
         )
         is None
     )
-    assert _raise_for_streaming_post_call_pipelines({"stream": True}) is None
-    assert _raise_for_streaming_post_call_pipelines({"background": True}) is None
+    assert _raise_for_streaming_post_call_pipelines({"stream": True}, auth) is None
+    assert _raise_for_streaming_post_call_pipelines({"background": True}, auth) is None
 
 
 # ---------------------------------------------------------------------------
@@ -1366,15 +1371,16 @@ async def _async_chunk_iter(chunks: List[Any]):
 
 
 @pytest.mark.asyncio
+@pytest.mark.parametrize("request_route", [None, "/v1/chat/completions"])
 async def test_pre_call_hook_allows_streaming_when_pipeline_guardrail_supports_unified(
-    proxy_logging, make_user_api_key_auth, monkeypatch
+    proxy_logging, make_user_api_key_auth, monkeypatch, request_route
 ):
     seen: Dict[str, Any] = {}
     monkeypatch.setattr(litellm, "callbacks", [_unified_stream_guardrail(seen)])
     data = _post_call_pipeline_data(stream=True)
 
     out = await proxy_logging.pre_call_hook(
-        user_api_key_dict=make_user_api_key_auth(),
+        user_api_key_dict=make_user_api_key_auth(request_route=request_route),
         data=data,
         call_type="completion",
         guardrails_only=True,
@@ -1422,6 +1428,60 @@ async def test_pre_call_hook_rejects_streaming_when_pipeline_guardrail_lacks_uni
     assert "apply_guardrail" in info.value.detail["error"]["message"]
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "rewrite_attribute, value",
+    [
+        ("mask_response_content", True),
+        ("streaming_transform_mode", "incremental_diff"),
+        ("guardrail_config", {"streaming_transform_mode": "incremental_diff"}),
+    ],
+)
+async def test_pre_call_hook_rejects_streaming_when_pipeline_guardrail_rewrites_streamed_content(
+    proxy_logging, make_user_api_key_auth, monkeypatch, rewrite_attribute, value
+):
+    seen: Dict[str, Any] = {}
+    guardrail = _unified_stream_guardrail(seen)
+    setattr(guardrail, rewrite_attribute, value)
+    monkeypatch.setattr(litellm, "callbacks", [guardrail])
+    data = _post_call_pipeline_data(stream=True)
+
+    with pytest.raises(HTTPException) as info:
+        await proxy_logging.pre_call_hook(
+            user_api_key_dict=make_user_api_key_auth(request_route="/v1/chat/completions"),
+            data=data,
+            call_type="completion",
+            guardrails_only=True,
+        )
+
+    assert info.value.status_code == 400
+    assert info.value.detail["error"]["guardrails"] == ("gr-post",)
+    assert "rewrite streamed content" in info.value.detail["error"]["message"]
+    assert seen.get("count") is None
+
+
+@pytest.mark.asyncio
+async def test_pre_call_hook_rejects_streaming_when_route_has_no_guardrail_translation(
+    proxy_logging, make_user_api_key_auth, monkeypatch
+):
+    seen: Dict[str, Any] = {}
+    monkeypatch.setattr(litellm, "callbacks", [_unified_stream_guardrail(seen)])
+    data = _post_call_pipeline_data(stream=True)
+
+    with pytest.raises(HTTPException) as info:
+        await proxy_logging.pre_call_hook(
+            user_api_key_dict=make_user_api_key_auth(request_route="/custom/stream"),
+            data=data,
+            call_type="completion",
+            guardrails_only=True,
+        )
+
+    assert info.value.status_code == 400
+    assert info.value.detail["error"]["policies"] == ("response-governance",)
+    assert "/custom/stream" in info.value.detail["error"]["message"]
+    assert seen.get("count") is None
+
+
 @pytest.mark.asyncio
 async def test_streaming_iterator_hook_pipeline_allow_releases_buffered_chunks(
     proxy_logging, make_user_api_key_auth, monkeypatch
@@ -1490,10 +1550,10 @@ async def test_streaming_iterator_hook_pipeline_withholds_unresolvable_response_
         ):
             delivered.append(item)
 
-    with pytest.raises(HTTPException) as info:
+    with pytest.raises(ProxyException) as info:
         await _drain()
 
     assert delivered == []
-    assert info.value.status_code == 500
-    assert "withheld" in info.value.detail["error"]["message"]
+    assert info.value.code == "500"
+    assert "withheld" in info.value.message
     assert seen.get("count") is None

From 1bed9bae43a7c28e46d6b38d4b58d10d35d87c58 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 29 Aug 2026 13:41:33 -0700
Subject: [PATCH 09/85] chore(policy_engine): drop control-flow comment flagged
 in review

---
 litellm/proxy/utils.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index 8b2c38d457f..231d4f18aa0 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -3431,9 +3431,6 @@ class ProxyLogging:
                     ),
                 )
 
-        # Policy pipelines run last, over the fully buffered stream, so a
-        # pipeline verdict covers whatever the flat guardrail chain above
-        # already let through.
         if post_call_pipelines:
             current_response = self._pipeline_gated_stream(
                 response=current_response,

From d51198fdeb3ccc7fcf70fbb116f94d8f360ef4d6 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 29 Aug 2026 14:07:34 -0700
Subject: [PATCH 10/85] test(policy_engine): cover streaming pipeline gate
 branches

Adds regression tests for the modify_response block on the Anthropic route,
the gate with no iterator overrides, and the per-chunk hook skipping
pipeline-managed guardrails. Corrects the gate docstring: an allow releases
the chunks as the endpoint translation left them, not verbatim
---
 litellm/proxy/utils.py                        |   7 +-
 .../proxy_logging/test_guardrail_pipeline.py  | 110 ++++++++++++++++++
 2 files changed, 115 insertions(+), 2 deletions(-)

diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index 231d4f18aa0..a64b57c1388 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -3468,8 +3468,11 @@ class ProxyLogging:
         pipeline allows it), then runs each pipeline's steps against the
         assembled output through the endpoint guardrail translation, the same
         machinery flat post_call guardrails use at end of stream. An allow
-        releases the buffered chunks verbatim; a block or modify_response
-        terminates with the translation's block chunks or the raised error.
+        releases the buffered chunks as that machinery left them (the
+        Responses and A2A translations write guardrail output back into the
+        final chunk, exactly as they do for flat guardrails); a block or
+        modify_response terminates with the translation's block chunks or the
+        raised error.
         """
         buffered: Final[list[object]] = []  # mutable-ok: accumulates the stream before the pipeline verdict
         async for item in response:
diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
index 6c90637158e..a258442c79a 100644
--- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
+++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
@@ -10,6 +10,7 @@ Covers ``_should_use_guardrail_load_balancing``, ``_execute_guardrail_hook``,
 from __future__ import annotations
 
 import asyncio
+import json
 from typing import Any, Dict, List
 from unittest.mock import AsyncMock, MagicMock, patch
 
@@ -1557,3 +1558,112 @@ async def test_streaming_iterator_hook_pipeline_withholds_unresolvable_response_
     assert info.value.code == "500"
     assert "withheld" in info.value.message
     assert seen.get("count") is None
+
+
+def _anthropic_sse_chunks() -> List[bytes]:
+    events = [
+        ("message_start", {"type": "message_start", "message": {"id": "msg_1", "type": "message", "role": "assistant", "model": "m", "content": [], "stop_reason": None, "usage": {"input_tokens": 1, "output_tokens": 0}}}),
+        ("content_block_start", {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}),
+        ("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "hello world"}}),
+        ("content_block_stop", {"type": "content_block_stop", "index": 0}),
+        ("message_delta", {"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": None}, "usage": {"output_tokens": 2}}),
+        ("message_stop", {"type": "message_stop"}),
+    ]
+    return [f"event: {name}\ndata: {json.dumps(payload)}\n\n".encode() for name, payload in events]
+
+
+@pytest.mark.asyncio
+async def test_streaming_iterator_hook_pipeline_modify_response_emits_translated_block(
+    proxy_logging, make_user_api_key_auth, monkeypatch
+):
+    seen: Dict[str, Any] = {}
+    monkeypatch.setattr(litellm, "callbacks", [_unified_stream_guardrail(seen, block=True)])
+    monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
+    pipeline = GuardrailPipeline(
+        mode="post_call",
+        steps=[
+            PipelineStep(
+                guardrail="gr-post",
+                on_pass="allow",
+                on_fail="modify_response",
+                modify_response_message="content policy block",
+            )
+        ],
+    )
+    data = _post_call_pipeline_data(stream=True)
+    data["metadata"]["_guardrail_pipelines"] = [("response-governance", pipeline)]
+    chunks = _anthropic_sse_chunks()
+
+    delivered = [
+        item
+        async for item in proxy_logging.async_post_call_streaming_iterator_hook(
+            user_api_key_dict=make_user_api_key_auth(request_route="/v1/messages"),
+            response=_async_chunk_iter(chunks),
+            request_data=data,
+        )
+    ]
+
+    raw = b"".join(delivered).decode()
+    assert seen["count"] == 1
+    assert "content policy block" in raw
+    assert "hello world" not in raw
+    assert not any(item is chunk for item in delivered for chunk in chunks)
+
+
+@pytest.mark.asyncio
+async def test_streaming_iterator_hook_pipeline_gates_without_iterator_overrides(
+    proxy_logging, make_user_api_key_auth, monkeypatch
+):
+    monkeypatch.setattr(litellm, "callbacks", [])
+    monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
+    data = _post_call_pipeline_data(stream=True)
+    delivered: List[Any] = []
+
+    async def _drain() -> None:
+        async for item in proxy_logging.async_post_call_streaming_iterator_hook(
+            user_api_key_dict=make_user_api_key_auth(request_route="/v1/chat/completions"),
+            response=_async_chunk_iter(_stream_chunks()),
+            request_data=data,
+        ):
+            delivered.append(item)
+
+    with pytest.raises(HTTPException) as info:
+        await _drain()
+
+    assert delivered == []
+    assert info.value.status_code == 400
+    assert info.value.detail["error"]["pipeline_context"]["step_results"] == [
+        {"guardrail": "gr-post", "outcome": "error", "action": "block"}
+    ]
+
+
+@pytest.mark.asyncio
+async def test_per_chunk_streaming_hook_skips_pipeline_managed_guardrail(
+    proxy_logging, make_user_api_key_auth, monkeypatch
+):
+    seen: Dict[str, Any] = {}
+
+    class RecordingGuardrail(CustomGuardrail):
+        async def async_post_call_streaming_hook(self, user_api_key_dict, response):
+            seen[self.guardrail_name] = seen.get(self.guardrail_name, 0) + 1
+            return None
+
+    managed = RecordingGuardrail(
+        guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=True
+    )
+    free = RecordingGuardrail(
+        guardrail_name="gr-free", event_hook=GuardrailEventHooks.post_call, default_on=True
+    )
+    monkeypatch.setattr(litellm, "callbacks", [managed, free])
+    monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
+    data = _post_call_pipeline_data(stream=True)
+
+    result = await proxy_logging.async_post_call_streaming_hook(
+        data=data,
+        response=_stream_chunks()[0],
+        user_api_key_dict=make_user_api_key_auth(request_route="/v1/chat/completions"),
+    )
+
+    assert result is not None
+    assert seen.get("gr-post") is None
+    assert seen["gr-free"] == 1

From 0c1f33dff7084559cf1612a38b5dde4ea0afe9b8 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 29 Aug 2026 14:19:21 -0700
Subject: [PATCH 11/85] fix(policy_engine): fail closed on content filter MASK
 steps for streaming pipelines

A litellm_content_filter step with a MASK action masks chat streams through
its own iterator hook under guardrails.add, which pipeline-managed guardrails
skip, so the pipeline path released the stream unmasked. CustomGuardrail now
declares rewrites_streamed_output (mask_response_content by default, any MASK
action for the content filter) and the upfront streaming check names such
steps in the same 400 it gives mask_response_content and incremental_diff
---
 litellm/integrations/custom_guardrail.py      |  3 ++
 .../litellm_content_filter/content_filter.py  |  7 ++++
 litellm/proxy/utils.py                        | 14 ++++----
 .../content_filter/test_content_filter.py     | 36 +++++++++++++++++++
 .../proxy_logging/test_guardrail_pipeline.py  | 34 +++++++++++++++++-
 5 files changed, 86 insertions(+), 8 deletions(-)

diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py
index 8dc6881d23e..a6e3d000120 100644
--- a/litellm/integrations/custom_guardrail.py
+++ b/litellm/integrations/custom_guardrail.py
@@ -762,6 +762,9 @@ class CustomGuardrail(CustomLogger):
     def uses_apply_guardrail_interface(self) -> bool:
         return type(self).apply_guardrail is not CustomGuardrail.apply_guardrail
 
+    def rewrites_streamed_output(self) -> bool:
+        return self.mask_response_content
+
     def _deployment_pre_call_target(self) -> "CustomLogger":
         if not self.uses_apply_guardrail_interface() or self.use_native_lifecycle_hooks:
             return self
diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py
index 722f96ef814..bd31882841e 100644
--- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py
+++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py
@@ -1947,6 +1947,13 @@ class ContentFilterGuardrail(CustomGuardrail):
                 exception_str=exception_str,
             )
 
+    def rewrites_streamed_output(self) -> bool:
+        return (
+            super().rewrites_streamed_output()
+            or any(entry["action"] == ContentFilterAction.MASK for entry in self.compiled_patterns)
+            or any(action == ContentFilterAction.MASK for action, _ in self.blocked_words.values())
+        )
+
     async def async_post_call_streaming_iterator_hook(
         self,
         user_api_key_dict: UserAPIKeyAuth,
diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index a64b57c1388..138f272af42 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -497,7 +497,7 @@ def _pipeline_step_rewrites_streamed_content(guardrail_name: str) -> bool:
     if callback is None:
         return False
     transform_mode: Final = unified_guardrail.resolve_streaming_flag(callback, "streaming_transform_mode", "block_only")
-    return callback.mask_response_content or transform_mode == "incremental_diff"
+    return callback.rewrites_streamed_output() or transform_mode == "incremental_diff"
 
 
 class _PipelineErrorBody(TypedDict):
@@ -518,10 +518,10 @@ def _raise_for_streaming_post_call_pipelines(data: Mapping[str, object], user_ap
     Background responses skip the post_call hooks entirely, so a pipeline
     governing one would silently never execute. Streaming responses execute
     pipelines against the buffered stream through the endpoint guardrail
-    translation of the request route, releasing the buffered chunks verbatim
-    on allow. That needs every step's guardrail to support the unified
-    apply_guardrail interface and to only allow or block (a step that rewrites
-    streamed content, via mask_response_content or
+    translation of the request route, releasing the buffered chunks on allow.
+    That needs every step's guardrail to support the unified apply_guardrail
+    interface and to only allow or block (a step that rewrites streamed
+    content, via mask_response_content, a MASK action, or
     streaming_transform_mode=incremental_diff, would have its rewrite silently
     dropped), and needs the route to have a translation at all; anything else
     keeps the 400 rather than letting ungoverned output stream through.
@@ -577,8 +577,8 @@ def _raise_for_streaming_post_call_pipelines(data: Mapping[str, object], user_ap
             "error": {
                 "message": (
                     "Policies with post_call guardrail pipelines cannot govern streaming responses "
-                    "because these pipeline guardrails rewrite streamed content (mask_response_content "
-                    "or streaming_transform_mode=incremental_diff), which pipeline steps would release "
+                    "because these pipeline guardrails rewrite streamed content (mask_response_content, "
+                    "a MASK action, or streaming_transform_mode=incremental_diff), which pipeline steps would release "
                     f"unmodified: {', '.join(rewriting_guardrails)}. Retry with stream=false, or drop "
                     "them from the pipeline steps so guardrails.add applies them to streamed output."
                 ),
diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py
index be55ac47bde..ffbedfa43ff 100644
--- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py
+++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py
@@ -3068,3 +3068,39 @@ class TestContentFilterToolCallArguments:
                 request_data={},
                 input_type="response",
             )
+
+
+class TestRewritesStreamedOutput:
+    def test_block_only_rules_do_not_rewrite(self):
+        guardrail = ContentFilterGuardrail(
+            guardrail_name="cf",
+            patterns=[ContentFilterPattern(pattern_type="prebuilt", pattern_name="us_ssn", action=ContentFilterAction.BLOCK)],
+            blocked_words=[BlockedWord(keyword="kumquat", action=ContentFilterAction.BLOCK)],
+        )
+
+        assert guardrail.rewrites_streamed_output() is False
+
+    def test_mask_blocked_word_rewrites(self):
+        guardrail = ContentFilterGuardrail(
+            guardrail_name="cf",
+            blocked_words=[BlockedWord(keyword="persimmon", action=ContentFilterAction.MASK)],
+        )
+
+        assert guardrail.rewrites_streamed_output() is True
+
+    def test_mask_pattern_rewrites(self):
+        guardrail = ContentFilterGuardrail(
+            guardrail_name="cf",
+            patterns=[ContentFilterPattern(pattern_type="prebuilt", pattern_name="us_ssn", action=ContentFilterAction.MASK)],
+        )
+
+        assert guardrail.rewrites_streamed_output() is True
+
+    def test_mask_response_content_rewrites(self):
+        guardrail = ContentFilterGuardrail(
+            guardrail_name="cf",
+            blocked_words=[BlockedWord(keyword="kumquat", action=ContentFilterAction.BLOCK)],
+            mask_response_content=True,
+        )
+
+        assert guardrail.rewrites_streamed_output() is True
diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
index a258442c79a..ee0a9a10172 100644
--- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
+++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
@@ -27,7 +27,8 @@ from litellm.integrations.prometheus import PrometheusLogger
 from litellm.proxy._types import ProxyException
 from litellm.proxy.common_utils.callback_utils import add_guardrail_to_applied_guardrails_header
 from litellm.proxy.utils import ProxyLogging, _raise_for_streaming_post_call_pipelines
-from litellm.types.guardrails import GuardrailEventHooks
+from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ContentFilterGuardrail
+from litellm.types.guardrails import BlockedWord, ContentFilterAction, GuardrailEventHooks
 from litellm.types.proxy.policy_engine.pipeline_types import (
     GuardrailPipeline,
     PipelineStep,
@@ -1461,6 +1462,37 @@ async def test_pre_call_hook_rejects_streaming_when_pipeline_guardrail_rewrites_
     assert seen.get("count") is None
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("action, rejected", [(ContentFilterAction.MASK, True), (ContentFilterAction.BLOCK, False)])
+async def test_pre_call_hook_rejects_streaming_only_when_content_filter_step_masks(
+    proxy_logging, make_user_api_key_auth, monkeypatch, action, rejected
+):
+    guardrail = ContentFilterGuardrail(
+        guardrail_name="gr-post",
+        event_hook=GuardrailEventHooks.post_call,
+        blocked_words=[BlockedWord(keyword="persimmon", action=action)],
+    )
+    monkeypatch.setattr(litellm, "callbacks", [guardrail])
+    data = _post_call_pipeline_data(stream=True)
+    user_api_key_dict = make_user_api_key_auth(request_route="/v1/chat/completions")
+
+    if not rejected:
+        out = await proxy_logging.pre_call_hook(
+            user_api_key_dict=user_api_key_dict, data=data, call_type="completion", guardrails_only=True
+        )
+        assert out is not None and out.get("stream") is True
+        return
+
+    with pytest.raises(HTTPException) as info:
+        await proxy_logging.pre_call_hook(
+            user_api_key_dict=user_api_key_dict, data=data, call_type="completion", guardrails_only=True
+        )
+
+    assert info.value.status_code == 400
+    assert info.value.detail["error"]["guardrails"] == ("gr-post",)
+    assert "a MASK action" in info.value.detail["error"]["message"]
+
+
 @pytest.mark.asyncio
 async def test_pre_call_hook_rejects_streaming_when_route_has_no_guardrail_translation(
     proxy_logging, make_user_api_key_auth, monkeypatch

From 90c8031dd76c5565c25b2adfe301a4ce715a6964 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 29 Aug 2026 14:32:35 -0700
Subject: [PATCH 12/85] fix(policy_engine): fail closed on content filter
 category MASK steps for streaming pipelines

---
 .../litellm_content_filter/content_filter.py  |  2 ++
 .../content_filter/test_content_filter.py     | 20 +++++++++++++++++
 .../proxy_logging/test_guardrail_pipeline.py  | 22 +++++++++++++++++++
 3 files changed, 44 insertions(+)

diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py
index bd31882841e..85eb50c78e7 100644
--- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py
+++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py
@@ -1952,6 +1952,8 @@ class ContentFilterGuardrail(CustomGuardrail):
             super().rewrites_streamed_output()
             or any(entry["action"] == ContentFilterAction.MASK for entry in self.compiled_patterns)
             or any(action == ContentFilterAction.MASK for action, _ in self.blocked_words.values())
+            or any(action == ContentFilterAction.MASK for _, _, action in self.category_keywords.values())
+            or any(action == ContentFilterAction.MASK for _, _, action in self.always_block_category_keywords.values())
         )
 
     async def async_post_call_streaming_iterator_hook(
diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py
index ffbedfa43ff..73020fe3e6f 100644
--- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py
+++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py
@@ -3104,3 +3104,23 @@ class TestRewritesStreamedOutput:
         )
 
         assert guardrail.rewrites_streamed_output() is True
+
+    @pytest.mark.parametrize("action, expected", [("MASK", True), ("BLOCK", False)])
+    def test_category_keywords_follow_the_category_action(self, action, expected):
+        guardrail = ContentFilterGuardrail(
+            guardrail_name="cf",
+            categories=[{"category": "bias_gender", "enabled": True, "action": action}],
+        )
+
+        assert guardrail.category_keywords and not guardrail.always_block_category_keywords
+        assert guardrail.rewrites_streamed_output() is expected
+
+    @pytest.mark.parametrize("action, expected", [("MASK", True), ("BLOCK", False)])
+    def test_always_block_category_keywords_follow_the_category_action(self, action, expected):
+        guardrail = ContentFilterGuardrail(
+            guardrail_name="cf",
+            categories=[{"category": "age_discrimination", "enabled": True, "action": action}],
+        )
+
+        assert guardrail.always_block_category_keywords and not guardrail.category_keywords
+        assert guardrail.rewrites_streamed_output() is expected
diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
index ee0a9a10172..895a986f348 100644
--- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
+++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
@@ -1493,6 +1493,28 @@ async def test_pre_call_hook_rejects_streaming_only_when_content_filter_step_mas
     assert "a MASK action" in info.value.detail["error"]["message"]
 
 
+@pytest.mark.asyncio
+async def test_pre_call_hook_rejects_streaming_when_content_filter_category_masks(
+    proxy_logging, make_user_api_key_auth, monkeypatch
+):
+    guardrail = ContentFilterGuardrail(
+        guardrail_name="gr-post",
+        event_hook=GuardrailEventHooks.post_call,
+        categories=[{"category": "bias_gender", "enabled": True, "action": "MASK"}],
+    )
+    monkeypatch.setattr(litellm, "callbacks", [guardrail])
+    data = _post_call_pipeline_data(stream=True)
+    user_api_key_dict = make_user_api_key_auth(request_route="/v1/chat/completions")
+
+    with pytest.raises(HTTPException) as info:
+        await proxy_logging.pre_call_hook(
+            user_api_key_dict=user_api_key_dict, data=data, call_type="completion", guardrails_only=True
+        )
+
+    assert info.value.status_code == 400
+    assert info.value.detail["error"]["guardrails"] == ("gr-post",)
+
+
 @pytest.mark.asyncio
 async def test_pre_call_hook_rejects_streaming_when_route_has_no_guardrail_translation(
     proxy_logging, make_user_api_key_auth, monkeypatch

From 2247fbc66df9769c3e3661b457b0b132513c1bd5 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 29 Aug 2026 15:44:08 -0700
Subject: [PATCH 13/85] fix(policy_engine): withhold streams when a pipeline
 guardrail rewrites output at runtime

---
 .../proxy/policy_engine/pipeline_executor.py  | 114 +++++++++++++++---
 litellm/proxy/utils.py                        |  60 ++++++---
 .../policy_engine/test_pipeline_executor.py   |  63 +++++++++-
 .../proxy_logging/test_guardrail_pipeline.py  | 103 +++++++++++++++-
 4 files changed, 302 insertions(+), 38 deletions(-)

diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py
index c422a7c0964..acd2c2c973a 100644
--- a/litellm/proxy/policy_engine/pipeline_executor.py
+++ b/litellm/proxy/policy_engine/pipeline_executor.py
@@ -6,8 +6,11 @@ pass/fail actions (allow, block, next, modify_response) and data forwarding.
 """
 
 import time
+from collections.abc import Mapping, Sequence
 from typing import TYPE_CHECKING, Any, Final, Literal
 
+from pydantic import BaseModel
+
 import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.integrations.custom_guardrail import (
@@ -24,8 +27,10 @@ from litellm.types.proxy.policy_engine.pipeline_types import (
     PipelineStep,
     PipelineStepResult,
 )
+from litellm.types.utils import GenericGuardrailAPIInputs
 
 if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
     from litellm.llms.base_llm.guardrail_translation.base_translation import (
         BaseTranslation,
     )
@@ -36,6 +41,90 @@ except ImportError:
     HTTPException = None
 
 
+class UndeliverableStreamRewrite(Exception):
+    def __init__(self, guardrail_name: str) -> None:
+        super().__init__(
+            f"Guardrail '{guardrail_name}' rewrote the streamed response, which streaming pipelines cannot deliver"
+        )
+        self.guardrail_name: Final = guardrail_name
+
+
+def _tool_call_shape(tool_call: object) -> tuple[object, object]:
+    plain: Final = tool_call.model_dump() if isinstance(tool_call, BaseModel) else tool_call
+    function: Final = plain.get("function") if isinstance(plain, Mapping) else None
+    if not isinstance(function, Mapping):
+        return (None, None)
+    return (function.get("name"), function.get("arguments"))
+
+
+def _rewrote_texts(sent: Sequence[str] | None, returned: Sequence[str] | None) -> bool:
+    return sent is not None and returned is not None and list(returned) != list(sent)
+
+
+def _rewrote_tool_calls(sent: Sequence[object] | None, returned: Sequence[object] | None) -> bool:
+    if sent is None or returned is None:
+        return False
+    return [_tool_call_shape(tool_call) for tool_call in returned] != [
+        _tool_call_shape(tool_call) for tool_call in sent
+    ]
+
+
+class _StreamRewriteObserver(CustomGuardrail):
+    """Stand-in handed to the endpoint translation in place of a streaming pipeline step's
+    guardrail. Translations cannot rewrite every buffered chunk consistently, so the gate
+    withholds the stream whenever the guardrail returned different output than it was given,
+    which for guardrails like Bedrock's ANONYMIZED action is only known at runtime."""
+
+    def __init__(self, inner: CustomGuardrail) -> None:
+        super().__init__(guardrail_name=inner.guardrail_name)
+        self.inner: Final = inner
+        self.rewrote = False
+
+    def structured_messages_cover_full_request(self) -> bool:
+        return self.inner.structured_messages_cover_full_request()
+
+    async def apply_guardrail(
+        self,
+        inputs: GenericGuardrailAPIInputs,
+        request_data: dict,  # mutable-ok: matches CustomGuardrail.apply_guardrail
+        input_type: Literal["request", "response"],
+        logging_obj: "LiteLLMLoggingObj | None" = None,
+    ) -> GenericGuardrailAPIInputs:
+        outputs: Final = await self.inner.apply_guardrail(
+            inputs=inputs, request_data=request_data, input_type=input_type, logging_obj=logging_obj
+        )
+        self.rewrote = (
+            self.rewrote
+            or _rewrote_texts(inputs.get("texts"), outputs.get("texts"))
+            or _rewrote_tool_calls(inputs.get("tool_calls"), outputs.get("tool_calls"))
+        )
+        return outputs
+
+
+def _prepare_hook_input(
+    step: PipelineStep,
+    callback: CustomLogger,
+    data: dict,  # mutable-ok: same request-payload shape the hooks mutate
+    raw_request_snapshot: dict | None,  # mutable-ok: same request-payload shape as data
+) -> tuple[dict, bool]:  # mutable-ok: returns that same request-payload dict
+    """Inject the step's guardrail name into metadata so should_run_guardrail() allows it,
+    and pick the payload the step scans: a scan_raw_request step evaluates the pristine
+    pre-pipeline snapshot instead of `data` (which earlier pass_data steps in this same
+    pipeline may have already rewritten), same reason the normal sequential/parallel
+    guardrail loops do this."""
+    if "metadata" not in data:
+        data["metadata"] = {}
+    data["metadata"]["guardrails"] = [step.guardrail]
+
+    scans_raw_request: Final = getattr(callback, "scan_raw_request", False)
+    hook_input: Final[dict] = (  # mutable-ok: same request-payload shape as data
+        independent_snapshot(raw_request_snapshot) if scans_raw_request and raw_request_snapshot is not None else data
+    )
+    if hook_input is not data:
+        hook_input.setdefault("metadata", {})["guardrails"] = [step.guardrail]
+    return hook_input, scans_raw_request
+
+
 class PipelineExecutor:
     """Executes guardrail pipelines with ordered, conditional step logic."""
 
@@ -195,23 +284,7 @@ class PipelineExecutor:
             return ("error", None, f"Guardrail '{step.guardrail}' not found", None)
 
         try:
-            # Inject guardrail name into metadata so should_run_guardrail() allows it
-            if "metadata" not in data:
-                data["metadata"] = {}
-            data["metadata"]["guardrails"] = [step.guardrail]
-
-            # A scan_raw_request step evaluates the pristine pre-pipeline
-            # snapshot instead of `data` (which earlier pass_data steps in
-            # this same pipeline may have already rewritten), same reason
-            # the normal sequential/parallel guardrail loops do this.
-            scans_raw_request: Final = getattr(callback, "scan_raw_request", False)
-            hook_input: Final[dict] = (  # mutable-ok: same request-payload shape as data
-                independent_snapshot(raw_request_snapshot)
-                if scans_raw_request and raw_request_snapshot is not None
-                else data
-            )
-            if hook_input is not data:
-                hook_input.setdefault("metadata", {})["guardrails"] = [step.guardrail]
+            hook_input, scans_raw_request = _prepare_hook_input(step, callback, data, raw_request_snapshot)
 
             # Use unified_guardrail path if callback implements apply_guardrail
             target: CustomLogger = callback
@@ -239,13 +312,16 @@ class PipelineExecutor:
                         f"Guardrail '{step.guardrail}' does not support streaming pipeline execution",
                         None,
                     )
+                observer: Final = _StreamRewriteObserver(callback)
                 await endpoint_translation.process_output_streaming_response(
                     responses_so_far=streaming_chunks,
-                    guardrail_to_apply=callback,
+                    guardrail_to_apply=observer,
                     litellm_logging_obj=data.get("litellm_logging_obj"),
                     user_api_key_dict=user_api_key_dict,
                     request_data=hook_input,
                 )
+                if observer.rewrote:
+                    raise UndeliverableStreamRewrite(step.guardrail)
                 response = None
             elif mode == "post_call":
                 response = await target.async_post_call_success_hook(
@@ -269,6 +345,8 @@ class PipelineExecutor:
                 return ("pass", {"response": response}, None, None)
             return ("pass", response if isinstance(response, dict) else None, None, None)
 
+        except UndeliverableStreamRewrite:
+            raise
         except Exception as e:
             if CustomGuardrail._is_guardrail_intervention(e):
                 error_msg: Final = _extract_error_message(e)
diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index 138f272af42..6c990222e51 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -155,7 +155,7 @@ from litellm.proxy.hooks.sensitive_data_routing import (
 )
 from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
 from litellm.proxy.management_helpers.key_settings_audit import with_settings_updated_at
-from litellm.proxy.policy_engine.pipeline_executor import PipelineExecutor
+from litellm.proxy.policy_engine.pipeline_executor import PipelineExecutor, UndeliverableStreamRewrite
 from litellm.repositories.budget_repository import BudgetRepository
 from litellm.repositories.config_repository import ConfigRepository
 from litellm.repositories.table_repositories import (
@@ -511,6 +511,23 @@ class _PipelineErrorDetail(TypedDict):
     error: ReadOnly[_PipelineErrorBody]
 
 
+def _undeliverable_stream_rewrite_error(policy_name: str, guardrail_name: str) -> HTTPException:
+    detail: Final[_PipelineErrorDetail] = {
+        "error": {
+            "message": (
+                f"Streaming response withheld by policy pipeline '{policy_name}' because guardrail "
+                f"'{guardrail_name}' rewrote the streamed output, and streaming pipelines cannot deliver "
+                "rewrites. Retry with stream=false, or drop it from the pipeline steps so guardrails.add "
+                "applies it to streamed output."
+            ),
+            "type": "guardrail_pipeline_error",
+            "policies": (policy_name,),
+            "guardrails": (guardrail_name,),
+        }
+    }
+    return HTTPException(status_code=400, detail=detail)
+
+
 def _raise_for_streaming_post_call_pipelines(data: Mapping[str, object], user_api_key_dict: UserAPIKeyAuth) -> None:
     """
     Reject up front the requests whose post_call pipelines could never run.
@@ -3468,11 +3485,12 @@ class ProxyLogging:
         pipeline allows it), then runs each pipeline's steps against the
         assembled output through the endpoint guardrail translation, the same
         machinery flat post_call guardrails use at end of stream. An allow
-        releases the buffered chunks as that machinery left them (the
-        Responses and A2A translations write guardrail output back into the
-        final chunk, exactly as they do for flat guardrails); a block or
-        modify_response terminates with the translation's block chunks or the
-        raised error.
+        releases the buffered chunks verbatim; a step whose guardrail rewrote
+        the output withholds the stream with a 400 instead, since no
+        translation rewrites every buffered chunk consistently and some
+        rewrites (Bedrock's ANONYMIZED action, for one) are only decided at
+        runtime; a block or modify_response terminates with the translation's
+        block chunks or the raised error.
         """
         buffered: Final[list[object]] = []  # mutable-ok: accumulates the stream before the pipeline verdict
         async for item in response:
@@ -3495,16 +3513,26 @@ class ProxyLogging:
         call_type, endpoint_translation = resolved
 
         for policy_name, pipeline in pipelines:
-            result: PipelineExecutionResult = await PipelineExecutor.execute_steps(
-                steps=pipeline.steps,
-                mode="post_call",
-                data=request_data,
-                user_api_key_dict=user_api_key_dict,
-                call_type=call_type,
-                policy_name=policy_name,
-                streaming_chunks=buffered,
-                endpoint_translation=endpoint_translation,
-            )
+            try:
+                result: PipelineExecutionResult = await PipelineExecutor.execute_steps(
+                    steps=pipeline.steps,
+                    mode="post_call",
+                    data=request_data,
+                    user_api_key_dict=user_api_key_dict,
+                    call_type=call_type,
+                    policy_name=policy_name,
+                    streaming_chunks=buffered,
+                    endpoint_translation=endpoint_translation,
+                )
+            except UndeliverableStreamRewrite as rewrite:
+                async for error_chunk in unified_guardrail.emit_streaming_http_error(
+                    _undeliverable_stream_rewrite_error(policy_name, rewrite.guardrail_name),
+                    call_type,
+                    buffered,
+                    request_data,
+                ):
+                    yield error_chunk
+                return
             try:
                 ProxyLogging._handle_pipeline_result(
                     result, data=request_data, policy_name=policy_name, original_response=buffered
diff --git a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py
index 054a5af4148..52fd8777a19 100644
--- a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py
+++ b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py
@@ -13,7 +13,7 @@ from litellm.integrations.custom_guardrail import CustomGuardrail
 from litellm.proxy.guardrails.guardrail_hooks.custom_code.custom_code_guardrail import (
     CustomCodeGuardrail,
 )
-from litellm.proxy.policy_engine.pipeline_executor import PipelineExecutor
+from litellm.proxy.policy_engine.pipeline_executor import PipelineExecutor, UndeliverableStreamRewrite
 from litellm.types.proxy.policy_engine.pipeline_types import (
     GuardrailPipeline,
     PipelineStep,
@@ -811,3 +811,64 @@ async def test_pipeline_step_keeps_native_hook_when_opted_out(monkeypatch):
     assert outcome == "pass"
     assert guardrail.native_pre_call_ran is True
     assert "guardrail_to_apply" not in data
+
+
+class _TextReturningGuardrail(CustomGuardrail):
+    def __init__(self, returned_texts):
+        super().__init__(guardrail_name="masker", event_hook="post_call", default_on=True)
+        self.returned_texts = returned_texts
+
+    async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None):
+        return {**inputs, "texts": self.returned_texts}
+
+
+class _TextTranslation:
+    def __init__(self):
+        self.seen_guardrail_names = []
+
+    async def process_output_streaming_response(
+        self, responses_so_far, guardrail_to_apply, litellm_logging_obj=None, user_api_key_dict=None, request_data=None
+    ):
+        self.seen_guardrail_names.append(guardrail_to_apply.guardrail_name)
+        await guardrail_to_apply.apply_guardrail(
+            inputs={"texts": ["hello world"]},
+            request_data=request_data or {},
+            input_type="response",
+            logging_obj=litellm_logging_obj,
+        )
+        return responses_so_far
+
+
+async def _run_streaming_step(returned_texts, translation):
+    return await PipelineExecutor.execute_steps(
+        steps=[PipelineStep(guardrail="masker", on_pass="allow", on_fail="next", on_error="next")],
+        mode="post_call",
+        data={"model": "m"},
+        user_api_key_dict=MagicMock(),
+        call_type="completion",
+        policy_name="p",
+        streaming_chunks=[object()],
+        endpoint_translation=translation,
+    )
+
+
+@pytest.mark.asyncio
+async def test_streaming_step_rewrite_escapes_execute_steps_regardless_of_step_actions(monkeypatch):
+    monkeypatch.setattr(litellm, "callbacks", [_TextReturningGuardrail(["hello [MASKED]"])])
+    translation = _TextTranslation()
+
+    with pytest.raises(UndeliverableStreamRewrite) as info:
+        await _run_streaming_step(["hello [MASKED]"], translation)
+
+    assert info.value.guardrail_name == "masker"
+    assert translation.seen_guardrail_names == ["masker"]
+
+
+@pytest.mark.asyncio
+async def test_streaming_step_unchanged_texts_in_another_container_allow(monkeypatch):
+    monkeypatch.setattr(litellm, "callbacks", [_TextReturningGuardrail(("hello world",))])
+
+    result = await _run_streaming_step(("hello world",), _TextTranslation())
+
+    assert result.terminal_action == "allow"
+    assert [step.outcome for step in result.step_results] == ["pass"]
diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
index 895a986f348..74bd2e483cc 100644
--- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
+++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
@@ -11,7 +11,7 @@ from __future__ import annotations
 
 import asyncio
 import json
-from typing import Any, Dict, List
+from typing import Any, Callable, Dict, List
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
@@ -878,10 +878,12 @@ async def test_process_prompt_template_aresponses_swaps_model_and_merges_input(p
 # ---------------------------------------------------------------------------
 
 
-def _post_call_pipeline_data(guardrail: str = "gr-post", **extra: Any) -> Dict[str, Any]:
+def _post_call_pipeline_data(
+    guardrail: str = "gr-post", step: PipelineStep | None = None, **extra: Any
+) -> Dict[str, Any]:
     pipeline = GuardrailPipeline(
         mode="post_call",
-        steps=[PipelineStep(guardrail=guardrail, on_pass="allow", on_fail="block")],
+        steps=[step or PipelineStep(guardrail=guardrail, on_pass="allow", on_fail="block")],
     )
     return {
         "model": "m",
@@ -1587,6 +1589,101 @@ async def test_streaming_iterator_hook_pipeline_block_withholds_all_chunks(
     assert "output blocked" in str(info.value.detail)
 
 
+def _rewriting_stream_guardrail(transform: Callable[[Dict[str, Any]], Dict[str, Any]]) -> CustomGuardrail:
+    class RewritingStreamGuardrail(CustomGuardrail):
+        async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None):
+            return {**inputs, **transform(inputs)}
+
+    return RewritingStreamGuardrail(guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=False)
+
+
+def _tool_call_stream_chunks() -> List[Any]:
+    tool_call = {
+        "index": 0,
+        "id": "call_1",
+        "type": "function",
+        "function": {"name": "lookup", "arguments": '{"ssn": "123"}'},
+    }
+    return [
+        litellm.ModelResponseStream(
+            choices=[{"index": 0, "delta": {"tool_calls": [tool_call]}, "finish_reason": None}]
+        ),
+        litellm.ModelResponseStream(choices=[{"index": 0, "delta": {}, "finish_reason": "tool_calls"}]),
+    ]
+
+
+def _echoed_tool_call_dicts(arguments: str) -> List[Dict[str, Any]]:
+    return [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": arguments}}]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("on_fail, on_error", [("block", None), ("next", "next")])
+@pytest.mark.parametrize(
+    "make_chunks, transform",
+    [
+        (_stream_chunks, lambda inputs: {"texts": ["hello [MASKED]"]}),
+        (_tool_call_stream_chunks, lambda inputs: {"tool_calls": _echoed_tool_call_dicts('{"ssn": "[MASKED]"}')}),
+    ],
+    ids=["texts", "tool_calls"],
+)
+async def test_streaming_iterator_hook_pipeline_withholds_runtime_rewrite(
+    proxy_logging, make_user_api_key_auth, monkeypatch, make_chunks, transform, on_fail, on_error
+):
+    monkeypatch.setattr(litellm, "callbacks", [_rewriting_stream_guardrail(transform)])
+    monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
+    step = PipelineStep(guardrail="gr-post", on_pass="allow", on_fail=on_fail, on_error=on_error)
+    data = _post_call_pipeline_data(step=step, stream=True)
+    delivered: List[Any] = []
+
+    async def _drain() -> None:
+        async for item in proxy_logging.async_post_call_streaming_iterator_hook(
+            user_api_key_dict=make_user_api_key_auth(request_route="/v1/chat/completions"),
+            response=_async_chunk_iter(make_chunks()),
+            request_data=data,
+        ):
+            delivered.append(item)
+
+    with pytest.raises(HTTPException) as info:
+        await _drain()
+
+    error = info.value.detail["error"]
+    assert delivered == []
+    assert info.value.status_code == 400
+    assert error["type"] == "guardrail_pipeline_error"
+    assert error["policies"] == ("response-governance",)
+    assert error["guardrails"] == ("gr-post",)
+    assert "stream=false" in error["message"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "make_chunks, transform",
+    [
+        (_stream_chunks, lambda inputs: {"texts": tuple(inputs["texts"])}),
+        (_tool_call_stream_chunks, lambda inputs: {"tool_calls": _echoed_tool_call_dicts('{"ssn": "123"}')}),
+    ],
+    ids=["texts_as_tuple", "tool_calls_as_dicts"],
+)
+async def test_streaming_iterator_hook_pipeline_releases_stream_echoed_in_another_shape(
+    proxy_logging, make_user_api_key_auth, monkeypatch, make_chunks, transform
+):
+    monkeypatch.setattr(litellm, "callbacks", [_rewriting_stream_guardrail(transform)])
+    monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
+    data = _post_call_pipeline_data(stream=True)
+    chunks = make_chunks()
+
+    delivered = [
+        item
+        async for item in proxy_logging.async_post_call_streaming_iterator_hook(
+            user_api_key_dict=make_user_api_key_auth(request_route="/v1/chat/completions"),
+            response=_async_chunk_iter(chunks),
+            request_data=data,
+        )
+    ]
+
+    assert [id(item) for item in delivered] == [id(chunk) for chunk in chunks]
+
+
 @pytest.mark.asyncio
 async def test_streaming_iterator_hook_pipeline_withholds_unresolvable_response_shape(
     proxy_logging, make_user_api_key_auth, monkeypatch

From badefa395cbfea283e2bac3d7ba545638a0792d0 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 29 Aug 2026 21:30:31 -0700
Subject: [PATCH 14/85] fix(policy_engine): snapshot guardrail inputs before
 apply_guardrail so in-place stream rewrites are withheld

---
 .../proxy/policy_engine/pipeline_executor.py  | 22 ++++++++++---------
 .../policy_engine/test_pipeline_executor.py   | 22 +++++++++++++++++++
 2 files changed, 34 insertions(+), 10 deletions(-)

diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py
index 21f3aca3f58..4c192a50096 100644
--- a/litellm/proxy/policy_engine/pipeline_executor.py
+++ b/litellm/proxy/policy_engine/pipeline_executor.py
@@ -57,16 +57,16 @@ def _tool_call_shape(tool_call: object) -> tuple[object, object]:
     return (function.get("name"), function.get("arguments"))
 
 
-def _rewrote_texts(sent: Sequence[str] | None, returned: Sequence[str] | None) -> bool:
-    return sent is not None and returned is not None and tuple(returned) != tuple(sent)
+def _text_snapshot(texts: Sequence[str] | None) -> tuple[str, ...] | None:
+    return None if texts is None else tuple(texts)
 
 
-def _rewrote_tool_calls(sent: Sequence[object] | None, returned: Sequence[object] | None) -> bool:
-    if sent is None or returned is None:
-        return False
-    return tuple(_tool_call_shape(tool_call) for tool_call in returned) != tuple(
-        _tool_call_shape(tool_call) for tool_call in sent
-    )
+def _tool_call_shapes(tool_calls: Sequence[object] | None) -> tuple[tuple[object, object], ...] | None:
+    return None if tool_calls is None else tuple(_tool_call_shape(tool_call) for tool_call in tool_calls)
+
+
+def _rewrote(sent: tuple[object, ...] | None, returned: tuple[object, ...] | None) -> bool:
+    return sent is not None and returned is not None and returned != sent
 
 
 class _StreamRewriteObserver(CustomGuardrail):
@@ -90,13 +90,15 @@ class _StreamRewriteObserver(CustomGuardrail):
         input_type: Literal["request", "response"],
         logging_obj: "LiteLLMLoggingObj | None" = None,
     ) -> GenericGuardrailAPIInputs:
+        sent_texts: Final = _text_snapshot(inputs.get("texts"))
+        sent_tool_shapes: Final = _tool_call_shapes(inputs.get("tool_calls"))
         outputs: Final = await self.inner.apply_guardrail(
             inputs=inputs, request_data=request_data, input_type=input_type, logging_obj=logging_obj
         )
         self.rewrote = (
             self.rewrote
-            or _rewrote_texts(inputs.get("texts"), outputs.get("texts"))
-            or _rewrote_tool_calls(inputs.get("tool_calls"), outputs.get("tool_calls"))
+            or _rewrote(sent_texts, _text_snapshot(outputs.get("texts")))
+            or _rewrote(sent_tool_shapes, _tool_call_shapes(outputs.get("tool_calls")))
         )
         return outputs
 
diff --git a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py
index 52fd8777a19..ef5d206f1d8 100644
--- a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py
+++ b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py
@@ -872,3 +872,25 @@ async def test_streaming_step_unchanged_texts_in_another_container_allow(monkeyp
 
     assert result.terminal_action == "allow"
     assert [step.outcome for step in result.step_results] == ["pass"]
+
+
+class _InPlaceMutatingGuardrail(CustomGuardrail):
+    """Rewrites like bedrock/presidio do: rebinds inputs["texts"] on the dict it was handed
+    and returns that same dict, so a post-call comparison against inputs sees no change."""
+
+    def __init__(self):
+        super().__init__(guardrail_name="masker", event_hook="post_call", default_on=True)
+
+    async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None):
+        inputs["texts"] = ["hello [MASKED]"]
+        return inputs
+
+
+@pytest.mark.asyncio
+async def test_streaming_step_in_place_rewrite_still_withholds_stream(monkeypatch):
+    monkeypatch.setattr(litellm, "callbacks", [_InPlaceMutatingGuardrail()])
+
+    with pytest.raises(UndeliverableStreamRewrite) as info:
+        await _run_streaming_step(["hello [MASKED]"], _TextTranslation())
+
+    assert info.value.guardrail_name == "masker"

From bcee01a7a7a3a29c5f6e54a0045ff3688d2dbdef Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 29 Aug 2026 21:52:52 -0700
Subject: [PATCH 15/85] fix(policy_engine): merge guardrail metadata writes
 back on block and modify_response so failure spend records keep guardrail
 cost and status

---
 .../proxy/policy_engine/pipeline_executor.py  |  2 +
 litellm/proxy/utils.py                        |  7 +++-
 .../proxy_logging/test_guardrail_pipeline.py  | 38 +++++++++++++++++++
 3 files changed, 46 insertions(+), 1 deletion(-)

diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py
index 4c192a50096..0c3ceb53707 100644
--- a/litellm/proxy/policy_engine/pipeline_executor.py
+++ b/litellm/proxy/policy_engine/pipeline_executor.py
@@ -236,6 +236,7 @@ class PipelineExecutor:
                     step_results=step_results,
                     error_message=error_detail,
                     original_exception=original_exception,
+                    modified_data=working_data if working_data != data else None,
                 )
 
             if action == "modify_response":
@@ -243,6 +244,7 @@ class PipelineExecutor:
                     terminal_action="modify_response",
                     step_results=step_results,
                     modify_response_message=step.modify_response_message or error_detail,
+                    modified_data=working_data if working_data != data else None,
                 )
 
             # action == "next" → continue to next step
diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index f9d018bc452..dbcea834d50 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -1838,7 +1838,9 @@ class ProxyLogging:
         payload (already sent upstream) must stay untouched; a replacement
         response carried in ``modified_data`` is adopted by the caller, and
         metadata-bucket writes (applied guardrails, guardrail logging info)
-        are merged back so headers and spend logs still see them. On the
+        are merged back so headers and spend logs still see them, on block
+        and modify_response too, so failure spend records keep guardrail
+        cost and status. On the
         streaming path it is the buffered chunk list, carried into
         ``ModifyResponseException.original_response`` for usage reporting.
         """
@@ -1850,6 +1852,9 @@ class ProxyLogging:
                     _merge_pipeline_metadata_writes(data, result.modified_data)
             return data
 
+        if result.modified_data is not None:
+            _merge_pipeline_metadata_writes(data, result.modified_data)
+
         if result.terminal_action == "block":
             original_exception: Final = result.original_exception
             if original_exception is not None and not _exception_changes_request_flow(original_exception):
diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
index 74bd2e483cc..d9b3578c966 100644
--- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
+++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
@@ -1197,6 +1197,44 @@ async def test_post_call_pipeline_guardrail_metadata_writes_reach_request_data(
     assert slg_entries[0]["guardrail_name"] == "gr-post"
 
 
+@pytest.mark.asyncio
+async def test_post_call_pipeline_block_keeps_guardrail_metadata_writes(
+    proxy_logging, make_user_api_key_auth, monkeypatch
+):
+    class BlockingWriterGuardrail(CustomGuardrail):
+        async def async_post_call_success_hook(self, data, user_api_key_dict, response):
+            add_guardrail_to_applied_guardrails_header(request_data=data, guardrail_name="gr-post")
+            self.add_standard_logging_guardrail_information_to_request_data(
+                guardrail_json_response={"verdict": "fail"},
+                request_data=data,
+                guardrail_status="guardrail_intervened",
+            )
+            raise HTTPException(status_code=400, detail={"error": "output blocked"})
+
+    monkeypatch.setattr(
+        litellm,
+        "callbacks",
+        [
+            BlockingWriterGuardrail(
+                guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=False
+            )
+        ],
+    )
+    monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
+    data = _post_call_pipeline_data()
+
+    with pytest.raises(HTTPException):
+        await proxy_logging.post_call_success_hook(
+            data=data, response=litellm.ModelResponse(), user_api_key_dict=make_user_api_key_auth()
+        )
+
+    assert data["metadata"]["applied_guardrails"] == ["gr-post"]
+    slg_entries = data["metadata"]["standard_logging_guardrail_information"]
+    assert len(slg_entries) == 1
+    assert slg_entries[0]["guardrail_name"] == "gr-post"
+    assert slg_entries[0]["guardrail_status"] == "guardrail_intervened"
+
+
 @pytest.mark.asyncio
 async def test_post_call_pipeline_managed_parallel_guardrail_runs_exactly_once(
     proxy_logging, make_user_api_key_auth, monkeypatch

From 673d1743a66363022777f0b3b261142ef77964ab Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 29 Aug 2026 22:12:59 -0700
Subject: [PATCH 16/85] fix(policy_engine): apply post_call pipeline text
 rewrites on streams

Buffered streams governed by post_call policy pipelines now deliver text
rewrites back into the stream per surface (chat SSE, responses SSE,
anthropic messages SSE) instead of rejecting the request with a 400
upfront. Rewrites chain across pipeline steps; tool-call rewrites and
translations without stream write-back still withhold the stream.
---
 .../chat/guardrail_translation/handler.py     |  74 +++++-
 .../guardrail_translation/base_translation.py |  15 +-
 .../chat/guardrail_translation/handler.py     |  91 ++++++-
 .../guardrail_translation/handler.py          |  83 ++++++-
 .../proxy/policy_engine/pipeline_executor.py  |  90 +++++--
 litellm/proxy/utils.py                        |  61 ++---
 .../test_anthropic_guardrail_handler.py       |  65 +++++
 .../test_openai_guardrail_handler.py          |  55 +++++
 ...test_openai_responses_guardrail_handler.py |  83 +++++++
 .../policy_engine/test_pipeline_executor.py   |   2 +
 .../proxy_logging/test_guardrail_pipeline.py  | 222 ++++++++++++++----
 11 files changed, 711 insertions(+), 130 deletions(-)

diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py
index b9ca18c7843..89c8431dfe4 100644
--- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py
+++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py
@@ -13,9 +13,10 @@ Pattern Overview:
 """
 
 import json
-from collections.abc import Mapping, Sequence
+from collections.abc import Iterator, Mapping, Sequence
 from copy import deepcopy
 from dataclasses import dataclass
+from itertools import chain, repeat
 from typing import TYPE_CHECKING, Any, Final, cast
 
 from typing_extensions import assert_never
@@ -120,6 +121,8 @@ class AnthropicMessagesHandler(BaseTranslation):
     them through guardrail rewrites; downstream provider handling is out of scope.
     """
 
+    delivers_ended_stream_text_rewrites = True
+
     def __init__(self):
         super().__init__()
         self.adapter = LiteLLMAnthropicMessagesAdapter()
@@ -931,11 +934,14 @@ class AnthropicMessagesHandler(BaseTranslation):
         litellm_logging_obj: "LiteLLMLoggingObj | None" = None,
         user_api_key_dict: "UserAPIKeyAuth | None" = None,
         request_data: dict | None = None,
+        deliver_ended_stream_rewrites: bool = False,
     ) -> list[Any]:
         """
         Process output streaming response by applying guardrails to text content.
 
         Get the string so far, check the apply guardrail to the string so far, and return the list of responses so far.
+        With ``deliver_ended_stream_rewrites``, an ended stream whose guardrail rewrote the text gets the rewrite
+        written back across the buffered chunks (full rewritten text in the first ``text_delta``, the rest blanked).
         """
         from litellm.integrations.custom_guardrail import ModifyResponseException
 
@@ -982,6 +988,15 @@ class AnthropicMessagesHandler(BaseTranslation):
                             responses_so_far, request_data
                         )
                     raise
+                guardrailed_texts: Final = _guardrailed_inputs.get("texts")
+                if (
+                    deliver_ended_stream_rewrites
+                    and isinstance(string_so_far, str)
+                    and string_so_far
+                    and guardrailed_texts
+                    and guardrailed_texts[0] != string_so_far
+                ):
+                    self._write_ended_stream_text_rewrite(responses_so_far, guardrailed_texts[0])
             else:
                 verbose_proxy_logger.debug("Skipping output guardrail - model response has no choices")
             return responses_so_far
@@ -1093,6 +1108,63 @@ class AnthropicMessagesHandler(BaseTranslation):
             inputs["model"] = response_model
         return inputs
 
+    @staticmethod
+    def _write_ended_stream_text_rewrite(
+        responses_so_far: list[Any],  # mutable-ok: rewrites the caller's buffered chunks in place
+        rewritten_text: str,
+    ) -> None:
+        """Deliver an ended-stream guardrail text rewrite by rewriting the
+        buffered chunks in place: the first ``text_delta`` carries the full
+        rewritten text and every later one is blanked, leaving the surrounding
+        message and content-block framing untouched. Handles both chunk formats
+        this stream carries (parsed event dicts and raw SSE bytes)."""
+        replacements: Final = chain((rewritten_text,), repeat(""))
+        for idx, item in enumerate(responses_so_far):
+            if isinstance(item, dict):
+                delta = item.get("delta")
+                if item.get("type") == "content_block_delta" and isinstance(delta, dict):
+                    if delta.get("type") == "text_delta":
+                        delta["text"] = next(replacements)
+            elif isinstance(item, (bytes, bytearray)):
+                responses_so_far[idx] = (  # rebind-ok: delivers the rewrite into the caller's buffer
+                    AnthropicMessagesHandler._rewrite_sse_text_deltas(bytes(item), replacements)
+                )
+
+    @staticmethod
+    def _rewrite_sse_text_deltas(sse_bytes: bytes, replacements: "Iterator[str]") -> bytes:
+        """Rewrite every ``text_delta`` data line in one SSE chunk with the next
+        replacement text, leaving all other events and framing byte-identical."""
+        try:
+            decoded: Final = sse_bytes.decode("utf-8")
+        except UnicodeDecodeError:
+            return sse_bytes
+        return "\n\n".join(
+            AnthropicMessagesHandler._rewrite_sse_block(block, replacements) for block in decoded.split("\n\n")
+        ).encode("utf-8")
+
+    @staticmethod
+    def _rewrite_sse_block(block: str, replacements: "Iterator[str]") -> str:
+        return "\n".join(AnthropicMessagesHandler._rewrite_sse_line(line, replacements) for line in block.split("\n"))
+
+    @staticmethod
+    def _rewrite_sse_line(line: str, replacements: "Iterator[str]") -> str:
+        if not line.startswith("data:"):
+            return line
+        try:
+            data: Final[str | int | float | bool | None | Sequence[object] | Mapping[str, object]] = json.loads(
+                line[len("data:") :].strip()
+            )
+        except json.JSONDecodeError:
+            return line
+        if not isinstance(data, dict) or data.get("type") != "content_block_delta":
+            return line
+        delta: Final = data.get("delta")
+        if not isinstance(delta, dict) or delta.get("type") != "text_delta":
+            return line
+        return "data: " + json.dumps(
+            {**data, "delta": {**delta, "text": next(replacements)}}  # mutable-ok: json.dumps needs plain dicts
+        )
+
     def get_streaming_string_so_far(self, responses_so_far: list[Any]) -> str:
         """
         Parse streaming responses and extract accumulated text content.
diff --git a/litellm/llms/base_llm/guardrail_translation/base_translation.py b/litellm/llms/base_llm/guardrail_translation/base_translation.py
index ba96ab3dc99..4b0cc0fd97c 100644
--- a/litellm/llms/base_llm/guardrail_translation/base_translation.py
+++ b/litellm/llms/base_llm/guardrail_translation/base_translation.py
@@ -1,6 +1,6 @@
 from abc import ABC, abstractmethod
 from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, Any, Final, Optional
+from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional
 
 if TYPE_CHECKING:
     from litellm.integrations.custom_guardrail import (
@@ -33,6 +33,13 @@ class StreamTransformSink:
 
 
 class BaseTranslation(ABC):
+    delivers_ended_stream_text_rewrites: ClassVar[bool] = False
+    """Whether ``process_output_streaming_response`` accepts
+    ``deliver_ended_stream_rewrites=True`` and, on an ended (fully buffered)
+    stream, writes guardrail text rewrites back across ``responses_so_far`` so
+    a buffered pipeline can release rewritten chunks instead of withholding the
+    stream. Tool-call rewrites stay undeliverable everywhere."""
+
     @staticmethod
     def transform_user_api_key_dict_to_metadata(
         user_api_key_dict: Any | None,
@@ -113,6 +120,7 @@ class BaseTranslation(ABC):
         user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
         request_data: dict | None = None,
         stream_transform_sink: StreamTransformSink | None = None,
+        deliver_ended_stream_rewrites: bool = False,
     ) -> Any:
         """
         Process output streaming response with guardrails.
@@ -120,6 +128,11 @@ class BaseTranslation(ABC):
         Optional to override in subclasses. ``stream_transform_sink`` is the
         out-parameter used by handlers that support streaming text
         transformations (see ``StreamTransformSink``); base handlers ignore it.
+        ``deliver_ended_stream_rewrites`` is passed True only when the caller
+        holds the whole buffered stream and the subclass declares
+        ``delivers_ended_stream_text_rewrites``: the handler then writes
+        guardrail text rewrites back across ``responses_so_far`` instead of
+        discarding them.
         """
         return responses_so_far
 
diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py
index 54673c77f80..1358cf7c37a 100644
--- a/litellm/llms/openai/chat/guardrail_translation/handler.py
+++ b/litellm/llms/openai/chat/guardrail_translation/handler.py
@@ -61,6 +61,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
     Methods can be overridden to customize behavior for different message formats.
     """
 
+    delivers_ended_stream_text_rewrites = True
+
     def get_structured_messages(self, data: dict) -> list[AllMessageValues] | None:
         """
         Convert chat completions request data to OpenAI-spec structured messages.
@@ -440,6 +442,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         user_api_key_dict: Any | None = None,
         request_data: dict | None = None,
         stream_transform_sink: StreamTransformSink | None = None,
+        deliver_ended_stream_rewrites: bool = False,
     ) -> list["ModelResponseStream"]:
         """
         Process output streaming responses by applying guardrails to text content.
@@ -454,6 +457,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
                 accumulated text (``responses_so_far`` is left untouched so it stays
                 a correct raw accumulator across rounds) and the guardrailed text
                 plus requested holdback are reported per choice on the sink.
+            deliver_ended_stream_rewrites: When True and the buffered stream has
+                ended, guardrail text rewrites are written back across
+                ``responses_so_far`` (full rewritten text in each choice's first
+                content-carrying chunk, the rest blanked) instead of discarded.
 
         Returns:
             The (unmodified) list of responses.
@@ -479,6 +486,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
             litellm_logging_obj=litellm_logging_obj,
             user_api_key_dict=user_api_key_dict,
             request_data=request_data,
+            deliver_ended_stream_rewrites=deliver_ended_stream_rewrites,
         )
 
     async def _process_streaming_block_only(
@@ -489,10 +497,12 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         litellm_logging_obj: "LiteLLMLoggingObj | None",
         user_api_key_dict: Any | None,
         request_data: dict | None,
+        deliver_ended_stream_rewrites: bool = False,
     ) -> list["ModelResponseStream"]:
         """Block-only streaming path: run the guardrail so an in-flight BLOCK can
         terminate the stream. Text rewrites are not propagated to the client here
-        (see ``_process_streaming_transform`` for the incremental_diff path)."""
+        (see ``_process_streaming_transform`` for the incremental_diff path) unless
+        ``deliver_ended_stream_rewrites`` opts the ended-stream branch in."""
         # check if the stream has ended
         has_stream_ended = False
         for chunk in responses_so_far:
@@ -501,20 +511,14 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
                 break
 
         if has_stream_ended:
-            # convert to model response
-            model_response: Final = cast(
-                ModelResponse,
-                stream_chunk_builder(chunks=responses_so_far, logging_obj=litellm_logging_obj),
-            )
-            # run process_output_response
-            await self.process_output_response(
-                response=model_response,
+            await self._process_ended_stream(
+                responses_so_far=responses_so_far,
                 guardrail_to_apply=guardrail_to_apply,
                 litellm_logging_obj=litellm_logging_obj,
                 user_api_key_dict=user_api_key_dict,
                 request_data=request_data,
+                deliver_ended_stream_rewrites=deliver_ended_stream_rewrites,
             )
-
             return responses_so_far
 
         # Step 0: Check if any response has text content to process
@@ -591,6 +595,38 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
 
         return responses_so_far
 
+    async def _process_ended_stream(
+        self,
+        *,
+        responses_so_far: list["ModelResponseStream"],  # mutable-ok: rewrites the caller's buffered chunks in place
+        guardrail_to_apply: "CustomGuardrail",
+        litellm_logging_obj: "LiteLLMLoggingObj | None",
+        user_api_key_dict: object,
+        request_data: dict[str, object] | None,  # mutable-ok: same request-payload shape the hooks take
+        deliver_ended_stream_rewrites: bool,
+    ) -> None:
+        """Ended-stream path: rebuild the full response, run the non-streaming
+        output guardrail against it, and (when opted in) write any text rewrite
+        back across the buffered chunks."""
+        model_response: Final = cast(
+            ModelResponse,
+            stream_chunk_builder(chunks=responses_so_far, logging_obj=litellm_logging_obj),
+        )
+        pre_guardrail_texts: Final = self._string_choice_contents(model_response)
+        await self.process_output_response(
+            response=model_response,
+            guardrail_to_apply=guardrail_to_apply,
+            litellm_logging_obj=litellm_logging_obj,
+            user_api_key_dict=user_api_key_dict,
+            request_data=request_data,
+        )
+        if deliver_ended_stream_rewrites:
+            await self._write_ended_stream_text_rewrites(
+                responses_so_far=responses_so_far,
+                guardrailed_response=model_response,
+                pre_guardrail_texts=pre_guardrail_texts,
+            )
+
     @staticmethod
     def _accumulate_string_content_by_choice_index(
         responses_so_far: list["ModelResponseStream"],
@@ -922,6 +958,41 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
                             if "name" in func_dict:
                                 existing_tool_call.function.name = func_dict["name"]
 
+    @staticmethod
+    def _string_choice_contents(response: "ModelResponse") -> tuple[str | None, ...]:
+        return tuple(
+            choice.message.content if isinstance(choice.message.content, str) else None for choice in response.choices
+        )
+
+    async def _write_ended_stream_text_rewrites(
+        self,
+        responses_so_far: list["ModelResponseStream"],  # mutable-ok: rewrites the caller's buffered chunks in place
+        guardrailed_response: "ModelResponse",
+        pre_guardrail_texts: tuple[str | None, ...],
+    ) -> None:
+        """Write ended-stream guardrail text rewrites back across the buffered
+        chunks: each rewritten choice's full text lands in its first
+        content-carrying chunk and the rest are blanked, the same shape the
+        in-flight write-back uses. Chunks carrying only finish_reason or usage
+        stay untouched."""
+        post_guardrail_texts: Final = self._string_choice_contents(guardrailed_response)
+        changed: Final = tuple(
+            (choice_idx, after)
+            for choice_idx, (before, after) in enumerate(zip(pre_guardrail_texts, post_guardrail_texts))
+            if before is not None and after is not None and after != before
+        )
+        if not changed:
+            return
+        await self._apply_guardrail_responses_to_output_streaming(
+            responses=responses_so_far,
+            guardrailed_texts=[
+                after for _choice_idx, after in changed
+            ],  # mutable-ok: the callee's signature predates this change and takes lists
+            task_mappings=[
+                (choice_idx, None) for choice_idx, _after in changed
+            ],  # mutable-ok: the callee's signature predates this change and takes lists
+        )
+
     async def _apply_guardrail_responses_to_output_streaming(
         self,
         responses: list["ModelResponseStream"],
diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py
index 7c5d8ac99ad..0f475aa04c8 100644
--- a/litellm/llms/openai/responses/guardrail_translation/handler.py
+++ b/litellm/llms/openai/responses/guardrail_translation/handler.py
@@ -28,7 +28,9 @@ Output: response.output is List[GenericResponseOutputItem] where each has:
     - text: str
 """
 
-from collections.abc import Sequence
+from collections.abc import Mapping, Sequence
+from itertools import chain, repeat
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Union, cast
 
 from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall
@@ -91,6 +93,8 @@ class OpenAIResponsesHandler(BaseTranslation):
     Methods can be overridden to customize behavior for different message formats.
     """
 
+    delivers_ended_stream_text_rewrites = True
+
     def get_structured_messages(self, data: dict) -> list[AllMessageValues] | None:
         """
         Convert Responses API request data to OpenAI-spec structured messages.
@@ -482,6 +486,7 @@ class OpenAIResponsesHandler(BaseTranslation):
         litellm_logging_obj: "LiteLLMLoggingObj | None" = None,
         user_api_key_dict: "UserAPIKeyAuth | None" = None,
         request_data: dict | None = None,
+        deliver_ended_stream_rewrites: bool = False,
     ) -> list[Any]:
         """
         Process output streaming response by applying guardrails to text content.
@@ -493,7 +498,11 @@ class OpenAIResponsesHandler(BaseTranslation):
         For ``response.completed`` events (the normal end-of-stream signal) we
         use the same per-item extraction + task-mapping approach as
         ``process_output_response`` so that unmasking / blocking works correctly
-        for every output item.
+        for every output item. With ``deliver_ended_stream_rewrites`` the earlier
+        text-carrying events (``response.output_text.delta`` / ``.done``,
+        ``response.content_part.done``, ``response.output_item.done``) are synced
+        to the rewritten completed response too, so a client reading deltas sees
+        the rewrite instead of the raw model output.
         """
         if not responses_so_far:
             return responses_so_far
@@ -562,6 +571,19 @@ class OpenAIResponsesHandler(BaseTranslation):
                     responses=guardrailed_texts,
                     task_mappings=task_mappings,
                 )
+                if deliver_ended_stream_rewrites:
+                    rewrites_by_position: Final = MappingProxyType(
+                        {
+                            task_mappings[task_idx]: rewritten
+                            for task_idx, rewritten in enumerate(guardrailed_texts)
+                            if task_idx < len(texts_to_check) and rewritten != texts_to_check[task_idx]
+                        }
+                    )
+                    if rewrites_by_position:
+                        self._sync_stream_events_with_rewrites(
+                            stream_events=responses_so_far[:-1],
+                            rewrites_by_position=rewrites_by_position,
+                        )
 
             return responses_so_far
 
@@ -607,6 +629,63 @@ class OpenAIResponsesHandler(BaseTranslation):
             )
         return responses_so_far
 
+    @staticmethod
+    def _write_event_field(event: object, field: str, value: str) -> None:
+        if isinstance(event, dict):
+            event[field] = value  # rebind-ok: delivering the rewrite means editing the buffered event in place
+        else:
+            setattr(event, field, value)
+
+    def _sync_stream_events_with_rewrites(
+        self,
+        stream_events: Sequence[Any],
+        rewrites_by_position: Mapping[tuple[int, int], str],
+    ) -> None:
+        """Sync pre-completion stream events with the rewritten completed
+        response, keyed by ``(output_index, content_index)``: the first
+        ``output_text.delta`` for a rewritten item carries the full rewritten
+        text and the rest are blanked, while ``output_text.done``,
+        ``content_part.done``, and ``output_item.done`` events carry the full
+        rewritten text, so every event a client may read agrees with the
+        rewritten ``response.completed`` payload."""
+        delta_replacements: Final = MappingProxyType(
+            {position: chain((rewritten,), repeat("")) for position, rewritten in rewrites_by_position.items()}
+        )
+        for event in stream_events:
+            if not (isinstance(event, dict) or hasattr(event, "get")):
+                continue
+            event_type = event.get("type")
+            output_index = event.get("output_index")
+            content_index = event.get("content_index")
+            if event_type == "response.output_item.done" and isinstance(output_index, int):
+                self._sync_output_item_done_event(event.get("item"), output_index, rewrites_by_position)
+                continue
+            if not isinstance(output_index, int) or not isinstance(content_index, int):
+                continue
+            position = (output_index, content_index)
+            if event_type == "response.output_text.delta" and position in delta_replacements:
+                self._write_event_field(event, "delta", next(delta_replacements[position]))
+            elif event_type == "response.output_text.done" and position in rewrites_by_position:
+                self._write_event_field(event, "text", rewrites_by_position[position])
+            elif event_type == "response.content_part.done" and position in rewrites_by_position:
+                part = event.get("part")
+                if isinstance(part, dict) or hasattr(part, "text"):
+                    self._write_event_field(part, "text", rewrites_by_position[position])
+
+    @staticmethod
+    def _sync_output_item_done_event(
+        item: object,
+        output_index: int,
+        rewrites_by_position: Mapping[tuple[int, int], str],
+    ) -> None:
+        content: Final = item.get("content") if isinstance(item, dict) else getattr(item, "content", None)
+        if not isinstance(content, list):
+            return
+        for (item_idx, content_idx), rewritten in rewrites_by_position.items():
+            if item_idx != output_index or content_idx >= len(content):
+                continue
+            OpenAIResponsesHandler._write_event_field(content[content_idx], "text", rewritten)
+
     def _check_streaming_has_ended(self, responses_so_far: Sequence[ResponsesStreamChunk]) -> bool:
         """
         Check if the streaming has ended.
diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py
index acd2c2c973a..1264cdfc14a 100644
--- a/litellm/proxy/policy_engine/pipeline_executor.py
+++ b/litellm/proxy/policy_engine/pipeline_executor.py
@@ -34,6 +34,7 @@ if TYPE_CHECKING:
     from litellm.llms.base_llm.guardrail_translation.base_translation import (
         BaseTranslation,
     )
+    from litellm.proxy._types import UserAPIKeyAuth
 
 try:
     from fastapi.exceptions import HTTPException
@@ -44,7 +45,8 @@ except ImportError:
 class UndeliverableStreamRewrite(Exception):
     def __init__(self, guardrail_name: str) -> None:
         super().__init__(
-            f"Guardrail '{guardrail_name}' rewrote the streamed response, which streaming pipelines cannot deliver"
+            f"Guardrail '{guardrail_name}' rewrote the streamed response in a way this endpoint's "
+            "streaming pipeline cannot deliver"
         )
         self.guardrail_name: Final = guardrail_name
 
@@ -57,28 +59,31 @@ def _tool_call_shape(tool_call: object) -> tuple[object, object]:
     return (function.get("name"), function.get("arguments"))
 
 
-def _rewrote_texts(sent: Sequence[str] | None, returned: Sequence[str] | None) -> bool:
-    return sent is not None and returned is not None and list(returned) != list(sent)
+def _text_snapshot(texts: Sequence[str] | None) -> tuple[str, ...] | None:
+    return None if texts is None else tuple(texts)
 
 
-def _rewrote_tool_calls(sent: Sequence[object] | None, returned: Sequence[object] | None) -> bool:
-    if sent is None or returned is None:
-        return False
-    return [_tool_call_shape(tool_call) for tool_call in returned] != [
-        _tool_call_shape(tool_call) for tool_call in sent
-    ]
+def _tool_call_shapes(tool_calls: Sequence[object] | None) -> tuple[tuple[object, object], ...] | None:
+    return None if tool_calls is None else tuple(_tool_call_shape(tool_call) for tool_call in tool_calls)
+
+
+def _rewrote(sent: tuple[object, ...] | None, returned: tuple[object, ...] | None) -> bool:
+    return sent is not None and returned is not None and returned != sent
 
 
 class _StreamRewriteObserver(CustomGuardrail):
     """Stand-in handed to the endpoint translation in place of a streaming pipeline step's
-    guardrail. Translations cannot rewrite every buffered chunk consistently, so the gate
-    withholds the stream whenever the guardrail returned different output than it was given,
-    which for guardrails like Bedrock's ANONYMIZED action is only known at runtime."""
+    guardrail. It records whether the guardrail returned different output than it was given,
+    which for guardrails like Bedrock's ANONYMIZED action is only known at runtime. Text
+    rewrites are deliverable on translations that write them back across the buffered chunks
+    (``delivers_ended_stream_text_rewrites``); tool-call rewrites and text rewrites on any
+    other translation make the gate withhold the stream."""
 
     def __init__(self, inner: CustomGuardrail) -> None:
         super().__init__(guardrail_name=inner.guardrail_name)
         self.inner: Final = inner
-        self.rewrote = False
+        self.rewrote_texts = False
+        self.rewrote_tool_calls = False
 
     def structured_messages_cover_full_request(self) -> bool:
         return self.inner.structured_messages_cover_full_request()
@@ -90,13 +95,14 @@ class _StreamRewriteObserver(CustomGuardrail):
         input_type: Literal["request", "response"],
         logging_obj: "LiteLLMLoggingObj | None" = None,
     ) -> GenericGuardrailAPIInputs:
+        sent_texts: Final = _text_snapshot(inputs.get("texts"))
+        sent_tool_shapes: Final = _tool_call_shapes(inputs.get("tool_calls"))
         outputs: Final = await self.inner.apply_guardrail(
             inputs=inputs, request_data=request_data, input_type=input_type, logging_obj=logging_obj
         )
-        self.rewrote = (
-            self.rewrote
-            or _rewrote_texts(inputs.get("texts"), outputs.get("texts"))
-            or _rewrote_tool_calls(inputs.get("tool_calls"), outputs.get("tool_calls"))
+        self.rewrote_texts = self.rewrote_texts or _rewrote(sent_texts, _text_snapshot(outputs.get("texts")))
+        self.rewrote_tool_calls = self.rewrote_tool_calls or _rewrote(
+            sent_tool_shapes, _tool_call_shapes(outputs.get("tool_calls"))
         )
         return outputs
 
@@ -250,6 +256,41 @@ class PipelineExecutor:
             modified_data=working_data if working_data != data else None,
         )
 
+    @staticmethod
+    async def _run_streaming_step(
+        step: PipelineStep,
+        callback: CustomGuardrail,
+        endpoint_translation: "BaseTranslation",
+        streaming_chunks: list[object],  # mutable-ok: shared buffered-stream chunks the translation rewrites in place
+        hook_input: dict[str, object],  # mutable-ok: same request-payload shape as data
+        user_api_key_dict: "UserAPIKeyAuth | None",
+        litellm_logging_obj: "LiteLLMLoggingObj | None",
+    ) -> None:
+        """Run one streaming post_call step through the endpoint translation, delivering
+        text rewrites on translations that support ended-stream write-back and raising
+        ``UndeliverableStreamRewrite`` for any rewrite that cannot reach the client."""
+        observer: Final = _StreamRewriteObserver(callback)
+        deliver_rewrites: Final = type(endpoint_translation).delivers_ended_stream_text_rewrites
+        if deliver_rewrites:
+            await endpoint_translation.process_output_streaming_response(
+                responses_so_far=streaming_chunks,
+                guardrail_to_apply=observer,
+                litellm_logging_obj=litellm_logging_obj,
+                user_api_key_dict=user_api_key_dict,
+                request_data=hook_input,
+                deliver_ended_stream_rewrites=True,
+            )
+        else:
+            await endpoint_translation.process_output_streaming_response(
+                responses_so_far=streaming_chunks,
+                guardrail_to_apply=observer,
+                litellm_logging_obj=litellm_logging_obj,
+                user_api_key_dict=user_api_key_dict,
+                request_data=hook_input,
+            )
+        if observer.rewrote_tool_calls or (observer.rewrote_texts and not deliver_rewrites):
+            raise UndeliverableStreamRewrite(step.guardrail)
+
     @staticmethod
     async def _run_step(
         step: PipelineStep,
@@ -312,16 +353,15 @@ class PipelineExecutor:
                         f"Guardrail '{step.guardrail}' does not support streaming pipeline execution",
                         None,
                     )
-                observer: Final = _StreamRewriteObserver(callback)
-                await endpoint_translation.process_output_streaming_response(
-                    responses_so_far=streaming_chunks,
-                    guardrail_to_apply=observer,
-                    litellm_logging_obj=data.get("litellm_logging_obj"),
+                await PipelineExecutor._run_streaming_step(
+                    step=step,
+                    callback=callback,
+                    endpoint_translation=endpoint_translation,
+                    streaming_chunks=streaming_chunks,
+                    hook_input=hook_input,
                     user_api_key_dict=user_api_key_dict,
-                    request_data=hook_input,
+                    litellm_logging_obj=data.get("litellm_logging_obj"),
                 )
-                if observer.rewrote:
-                    raise UndeliverableStreamRewrite(step.guardrail)
                 response = None
             elif mode == "post_call":
                 response = await target.async_post_call_success_hook(
diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index 6c990222e51..7f3c3aecd87 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -492,14 +492,6 @@ def _pipeline_step_supports_unified_streaming(guardrail_name: str) -> bool:
     return callback is not None and PipelineExecutor.supports_unified_execution(callback)
 
 
-def _pipeline_step_rewrites_streamed_content(guardrail_name: str) -> bool:
-    callback: Final = PipelineExecutor.find_guardrail_callback(guardrail_name)
-    if callback is None:
-        return False
-    transform_mode: Final = unified_guardrail.resolve_streaming_flag(callback, "streaming_transform_mode", "block_only")
-    return callback.rewrites_streamed_output() or transform_mode == "incremental_diff"
-
-
 class _PipelineErrorBody(TypedDict):
     message: ReadOnly[str]
     type: ReadOnly[str]
@@ -516,9 +508,10 @@ def _undeliverable_stream_rewrite_error(policy_name: str, guardrail_name: str) -
         "error": {
             "message": (
                 f"Streaming response withheld by policy pipeline '{policy_name}' because guardrail "
-                f"'{guardrail_name}' rewrote the streamed output, and streaming pipelines cannot deliver "
-                "rewrites. Retry with stream=false, or drop it from the pipeline steps so guardrails.add "
-                "applies it to streamed output."
+                f"'{guardrail_name}' rewrote the streamed output in a way this endpoint's streaming "
+                "pipeline cannot deliver (a tool-call rewrite, or a text rewrite on a route without "
+                "stream write-back). Retry with stream=false, or drop it from the pipeline steps so "
+                "guardrails.add applies it to streamed output."
             ),
             "type": "guardrail_pipeline_error",
             "policies": (policy_name,),
@@ -535,13 +528,13 @@ def _raise_for_streaming_post_call_pipelines(data: Mapping[str, object], user_ap
     Background responses skip the post_call hooks entirely, so a pipeline
     governing one would silently never execute. Streaming responses execute
     pipelines against the buffered stream through the endpoint guardrail
-    translation of the request route, releasing the buffered chunks on allow.
-    That needs every step's guardrail to support the unified apply_guardrail
-    interface and to only allow or block (a step that rewrites streamed
-    content, via mask_response_content, a MASK action, or
-    streaming_transform_mode=incremental_diff, would have its rewrite silently
-    dropped), and needs the route to have a translation at all; anything else
-    keeps the 400 rather than letting ungoverned output stream through.
+    translation of the request route, releasing the buffered chunks on allow
+    (rewritten in place when a guardrail rewrote text and the translation
+    delivers ended-stream rewrites; a rewrite the translation cannot deliver
+    fails closed at runtime instead). That needs every step's guardrail to
+    support the unified apply_guardrail interface, and needs the route to have
+    a translation at all; anything else keeps the 400 rather than letting
+    ungoverned output stream through.
     """
     is_stream: Final = data.get("stream") is True
     is_background: Final = data.get("background") is True
@@ -586,25 +579,6 @@ def _raise_for_streaming_post_call_pipelines(data: Mapping[str, object], user_ap
             }
         }
         raise HTTPException(status_code=400, detail=unsupported_detail)
-    rewriting_guardrails: Final = tuple(
-        guardrail for guardrail in step_guardrails if _pipeline_step_rewrites_streamed_content(guardrail)
-    )
-    if rewriting_guardrails:
-        rewriting_detail: Final[_PipelineErrorDetail] = {
-            "error": {
-                "message": (
-                    "Policies with post_call guardrail pipelines cannot govern streaming responses "
-                    "because these pipeline guardrails rewrite streamed content (mask_response_content, "
-                    "a MASK action, or streaming_transform_mode=incremental_diff), which pipeline steps would release "
-                    f"unmodified: {', '.join(rewriting_guardrails)}. Retry with stream=false, or drop "
-                    "them from the pipeline steps so guardrails.add applies them to streamed output."
-                ),
-                "type": "guardrail_pipeline_error",
-                "policies": post_call_policies,
-                "guardrails": rewriting_guardrails,
-            }
-        }
-        raise HTTPException(status_code=400, detail=rewriting_detail)
     route: Final = user_api_key_dict.request_route
     if not route or resolve_endpoint_translation(user_api_key_dict, None) is not None:
         return
@@ -3485,12 +3459,13 @@ class ProxyLogging:
         pipeline allows it), then runs each pipeline's steps against the
         assembled output through the endpoint guardrail translation, the same
         machinery flat post_call guardrails use at end of stream. An allow
-        releases the buffered chunks verbatim; a step whose guardrail rewrote
-        the output withholds the stream with a 400 instead, since no
-        translation rewrites every buffered chunk consistently and some
-        rewrites (Bedrock's ANONYMIZED action, for one) are only decided at
-        runtime; a block or modify_response terminates with the translation's
-        block chunks or the raised error.
+        releases the buffered chunks: verbatim when no guardrail rewrote the
+        output, rewritten in place when one rewrote text and the translation
+        delivers ended-stream rewrites (later steps then re-scan the rewritten
+        chunks, so rewrites chain). A rewrite the translation cannot deliver
+        (a tool-call rewrite, or a text rewrite on a route without write-back)
+        withholds the stream with a 400; a block or modify_response terminates
+        with the translation's block chunks or the raised error.
         """
         buffered: Final[list[object]] = []  # mutable-ok: accumulates the stream before the pipeline verdict
         async for item in response:
diff --git a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py
index af3ccd65b11..ba26da50bc8 100644
--- a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py
+++ b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py
@@ -263,6 +263,71 @@ class TestAnthropicMessagesHandlerStreamingOutputProcessing:
             # Should return the responses unchanged
             assert result == responses_so_far
 
+    @staticmethod
+    def _ended_sse_chunks() -> list:
+        events = [
+            ("message_start", {"type": "message_start", "message": {"id": "msg_1", "type": "message", "role": "assistant", "model": "claude-sonnet-4-5", "content": [], "stop_reason": None, "usage": {"input_tokens": 1, "output_tokens": 0}}}),
+            ("content_block_start", {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}),
+            ("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "hello "}}),
+            ("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "world"}}),
+            ("content_block_stop", {"type": "content_block_stop", "index": 0}),
+            ("message_delta", {"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": None}, "usage": {"output_tokens": 2}}),
+            ("message_stop", {"type": "message_stop"}),
+        ]
+        return [f"event: {name}\ndata: {json.dumps(payload)}\n\n".encode() for name, payload in events]
+
+    @staticmethod
+    def _masking_guardrail() -> CustomGuardrail:
+        class MaskWorld(CustomGuardrail):
+            async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None):
+                return {**inputs, "texts": [text.replace("world", "[MASKED]") for text in inputs.get("texts", [])]}
+
+        return MaskWorld(guardrail_name="test")
+
+    @staticmethod
+    def _delta_texts(chunks: list) -> list:
+        texts = []
+        for chunk in chunks:
+            for line in chunk.decode().split("\n"):
+                if not line.startswith("data:"):
+                    continue
+                data = json.loads(line[len("data:") :].strip())
+                if data.get("type") == "content_block_delta":
+                    texts.append(data["delta"]["text"])
+        return texts
+
+    @pytest.mark.asyncio
+    async def test_deliver_ended_stream_rewrites_writes_text_back_into_sse_chunks(self):
+        handler = AnthropicMessagesHandler()
+        chunks = self._ended_sse_chunks()
+
+        result = await handler.process_output_streaming_response(
+            responses_so_far=chunks,
+            guardrail_to_apply=self._masking_guardrail(),
+            litellm_logging_obj=MagicMock(),
+            deliver_ended_stream_rewrites=True,
+        )
+
+        assert result is chunks
+        assert self._delta_texts(chunks) == ["hello [MASKED]", ""]
+        raw = b"".join(chunks).decode()
+        assert "event: message_start" in raw and "event: message_stop" in raw
+        assert '"stop_reason": "end_turn"' in raw
+
+    @pytest.mark.asyncio
+    async def test_ended_stream_rewrite_leaves_chunks_untouched_by_default(self):
+        handler = AnthropicMessagesHandler()
+        chunks = self._ended_sse_chunks()
+        original = [bytes(chunk) for chunk in chunks]
+
+        await handler.process_output_streaming_response(
+            responses_so_far=chunks,
+            guardrail_to_apply=self._masking_guardrail(),
+            litellm_logging_obj=MagicMock(),
+        )
+
+        assert chunks == original
+
 
 class TestAnthropicMessagesHandlerInputProcessing:
     """Test input processing preserves litellm_metadata for dynamic guardrails."""
diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py
index a29e0be4655..26442a4a6ed 100644
--- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py
+++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py
@@ -1073,6 +1073,61 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
         # Should return the responses
         assert result == responses_so_far
 
+    @staticmethod
+    def _ended_stream_chunks() -> list:
+        from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
+
+        return [
+            ModelResponseStream(
+                id="chatcmpl-123",
+                created=1234567890,
+                model="gpt-4",
+                object="chat.completion.chunk",
+                choices=[StreamingChoices(index=0, delta=Delta(content="Hello"), finish_reason=None)],
+            ),
+            ModelResponseStream(
+                id="chatcmpl-123",
+                created=1234567890,
+                model="gpt-4",
+                object="chat.completion.chunk",
+                choices=[StreamingChoices(index=0, delta=Delta(content=" world"), finish_reason="stop")],
+            ),
+        ]
+
+    @pytest.mark.asyncio
+    async def test_deliver_ended_stream_rewrites_writes_text_back_into_chunks(self):
+        handler = OpenAIChatCompletionsHandler()
+        guardrail = MockGuardrail(guardrail_name="test")
+        chunks = self._ended_stream_chunks()
+
+        result = await handler.process_output_streaming_response(
+            responses_so_far=chunks,
+            guardrail_to_apply=guardrail,
+            litellm_logging_obj=None,
+            deliver_ended_stream_rewrites=True,
+        )
+
+        assert result is chunks
+        assert chunks[0].choices[0].delta.content == "HELLO WORLD"
+        assert chunks[1].choices[0].delta.content in (None, "")
+        assert chunks[1].choices[0].finish_reason == "stop"
+
+    @pytest.mark.asyncio
+    async def test_ended_stream_rewrite_leaves_chunks_untouched_by_default(self):
+        handler = OpenAIChatCompletionsHandler()
+        guardrail = MockGuardrail(guardrail_name="test")
+        chunks = self._ended_stream_chunks()
+
+        await handler.process_output_streaming_response(
+            responses_so_far=chunks,
+            guardrail_to_apply=guardrail,
+            litellm_logging_obj=None,
+        )
+
+        assert chunks[0].choices[0].delta.content == "Hello"
+        assert chunks[1].choices[0].delta.content == " world"
+        assert chunks[1].choices[0].finish_reason == "stop"
+
 
 class TestGetStructuredMessages:
     """Test the get_structured_messages method."""
diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py
index 447175b09a6..4f95e08cb71 100644
--- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py
+++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py
@@ -1104,6 +1104,89 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing:
         output_text = result[-1]["response"]["output"][0]["content"][0]["text"]
         assert output_text == original_text
 
+    @staticmethod
+    def _ended_stream_events() -> List[dict]:
+        content = [{"type": "output_text", "text": "hello world"}]
+        item = {
+            "type": "message",
+            "id": "msg_123",
+            "status": "completed",
+            "role": "assistant",
+            "content": content,
+        }
+        return [
+            {"type": "response.output_text.delta", "output_index": 0, "content_index": 0, "delta": "hello "},
+            {"type": "response.output_text.delta", "output_index": 0, "content_index": 0, "delta": "world"},
+            {"type": "response.output_text.done", "output_index": 0, "content_index": 0, "text": "hello world"},
+            {
+                "type": "response.content_part.done",
+                "output_index": 0,
+                "content_index": 0,
+                "part": {"type": "output_text", "text": "hello world"},
+            },
+            {"type": "response.output_item.done", "output_index": 0, "item": {**item, "content": [dict(c) for c in content]}},
+            {
+                "type": "response.completed",
+                "response": {
+                    "id": "resp_123",
+                    "model": "gpt-4o",
+                    "output": [{**item, "content": [dict(c) for c in content]}],
+                    "status": "completed",
+                },
+            },
+        ]
+
+    @staticmethod
+    def _masking_guardrail() -> CustomGuardrail:
+        class MaskWorld(CustomGuardrail):
+            async def apply_guardrail(
+                self,
+                inputs: GenericGuardrailAPIInputs,
+                request_data: dict,
+                input_type: Literal["request", "response"],
+                logging_obj: Optional[Any] = None,
+            ) -> GenericGuardrailAPIInputs:
+                texts = inputs.get("texts", [])
+                return {**inputs, "texts": [t.replace("world", "[MASKED]") for t in texts]}
+
+        return MaskWorld(guardrail_name="test-mask")
+
+    @pytest.mark.asyncio
+    async def test_deliver_ended_stream_rewrites_syncs_all_stream_events(self):
+        handler = OpenAIResponsesHandler()
+        events = self._ended_stream_events()
+
+        result = await handler.process_output_streaming_response(
+            responses_so_far=events,
+            guardrail_to_apply=self._masking_guardrail(),
+            litellm_logging_obj=None,
+            deliver_ended_stream_rewrites=True,
+        )
+
+        assert result is events
+        assert events[0]["delta"] == "hello [MASKED]"
+        assert events[1]["delta"] == ""
+        assert events[2]["text"] == "hello [MASKED]"
+        assert events[3]["part"]["text"] == "hello [MASKED]"
+        assert events[4]["item"]["content"][0]["text"] == "hello [MASKED]"
+        assert events[5]["response"]["output"][0]["content"][0]["text"] == "hello [MASKED]"
+
+    @pytest.mark.asyncio
+    async def test_ended_stream_rewrite_leaves_delta_events_untouched_by_default(self):
+        handler = OpenAIResponsesHandler()
+        events = self._ended_stream_events()
+
+        await handler.process_output_streaming_response(
+            responses_so_far=events,
+            guardrail_to_apply=self._masking_guardrail(),
+            litellm_logging_obj=None,
+        )
+
+        assert events[0]["delta"] == "hello "
+        assert events[1]["delta"] == "world"
+        assert events[2]["text"] == "hello world"
+        assert events[5]["response"]["output"][0]["content"][0]["text"] == "hello [MASKED]"
+
 
 class TestGetStructuredMessages:
     """Test the get_structured_messages method for Responses API handler."""
diff --git a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py
index 52fd8777a19..908c9f12c9e 100644
--- a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py
+++ b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py
@@ -823,6 +823,8 @@ class _TextReturningGuardrail(CustomGuardrail):
 
 
 class _TextTranslation:
+    delivers_ended_stream_text_rewrites = False
+
     def __init__(self):
         self.seen_guardrail_names = []
 
diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
index 74bd2e483cc..73270ec5671 100644
--- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
+++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
@@ -24,7 +24,7 @@ from litellm.integrations.custom_guardrail import (
     ModifyResponseException,
 )
 from litellm.integrations.prometheus import PrometheusLogger
-from litellm.proxy._types import ProxyException
+from litellm.proxy._types import ProxyException, UserAPIKeyAuth
 from litellm.proxy.common_utils.callback_utils import add_guardrail_to_applied_guardrails_header
 from litellm.proxy.utils import ProxyLogging, _raise_for_streaming_post_call_pipelines
 from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ContentFilterGuardrail
@@ -1441,7 +1441,7 @@ async def test_pre_call_hook_rejects_streaming_when_pipeline_guardrail_lacks_uni
         ("guardrail_config", {"streaming_transform_mode": "incremental_diff"}),
     ],
 )
-async def test_pre_call_hook_rejects_streaming_when_pipeline_guardrail_rewrites_streamed_content(
+async def test_pre_call_hook_allows_streaming_when_pipeline_guardrail_rewrites_streamed_content(
     proxy_logging, make_user_api_key_auth, monkeypatch, rewrite_attribute, value
 ):
     seen: Dict[str, Any] = {}
@@ -1450,24 +1450,21 @@ async def test_pre_call_hook_rejects_streaming_when_pipeline_guardrail_rewrites_
     monkeypatch.setattr(litellm, "callbacks", [guardrail])
     data = _post_call_pipeline_data(stream=True)
 
-    with pytest.raises(HTTPException) as info:
-        await proxy_logging.pre_call_hook(
-            user_api_key_dict=make_user_api_key_auth(request_route="/v1/chat/completions"),
-            data=data,
-            call_type="completion",
-            guardrails_only=True,
-        )
+    out = await proxy_logging.pre_call_hook(
+        user_api_key_dict=make_user_api_key_auth(request_route="/v1/chat/completions"),
+        data=data,
+        call_type="completion",
+        guardrails_only=True,
+    )
 
-    assert info.value.status_code == 400
-    assert info.value.detail["error"]["guardrails"] == ("gr-post",)
-    assert "rewrite streamed content" in info.value.detail["error"]["message"]
-    assert seen.get("count") is None
+    assert out is not None
+    assert out.get("stream") is True
 
 
 @pytest.mark.asyncio
-@pytest.mark.parametrize("action, rejected", [(ContentFilterAction.MASK, True), (ContentFilterAction.BLOCK, False)])
-async def test_pre_call_hook_rejects_streaming_only_when_content_filter_step_masks(
-    proxy_logging, make_user_api_key_auth, monkeypatch, action, rejected
+@pytest.mark.parametrize("action", [ContentFilterAction.MASK, ContentFilterAction.BLOCK])
+async def test_pre_call_hook_allows_streaming_when_content_filter_step_masks_or_blocks(
+    proxy_logging, make_user_api_key_auth, monkeypatch, action
 ):
     guardrail = ContentFilterGuardrail(
         guardrail_name="gr-post",
@@ -1478,25 +1475,15 @@ async def test_pre_call_hook_rejects_streaming_only_when_content_filter_step_mas
     data = _post_call_pipeline_data(stream=True)
     user_api_key_dict = make_user_api_key_auth(request_route="/v1/chat/completions")
 
-    if not rejected:
-        out = await proxy_logging.pre_call_hook(
-            user_api_key_dict=user_api_key_dict, data=data, call_type="completion", guardrails_only=True
-        )
-        assert out is not None and out.get("stream") is True
-        return
+    out = await proxy_logging.pre_call_hook(
+        user_api_key_dict=user_api_key_dict, data=data, call_type="completion", guardrails_only=True
+    )
 
-    with pytest.raises(HTTPException) as info:
-        await proxy_logging.pre_call_hook(
-            user_api_key_dict=user_api_key_dict, data=data, call_type="completion", guardrails_only=True
-        )
-
-    assert info.value.status_code == 400
-    assert info.value.detail["error"]["guardrails"] == ("gr-post",)
-    assert "a MASK action" in info.value.detail["error"]["message"]
+    assert out is not None and out.get("stream") is True
 
 
 @pytest.mark.asyncio
-async def test_pre_call_hook_rejects_streaming_when_content_filter_category_masks(
+async def test_pre_call_hook_allows_streaming_when_content_filter_category_masks(
     proxy_logging, make_user_api_key_auth, monkeypatch
 ):
     guardrail = ContentFilterGuardrail(
@@ -1508,13 +1495,11 @@ async def test_pre_call_hook_rejects_streaming_when_content_filter_category_mask
     data = _post_call_pipeline_data(stream=True)
     user_api_key_dict = make_user_api_key_auth(request_route="/v1/chat/completions")
 
-    with pytest.raises(HTTPException) as info:
-        await proxy_logging.pre_call_hook(
-            user_api_key_dict=user_api_key_dict, data=data, call_type="completion", guardrails_only=True
-        )
+    out = await proxy_logging.pre_call_hook(
+        user_api_key_dict=user_api_key_dict, data=data, call_type="completion", guardrails_only=True
+    )
 
-    assert info.value.status_code == 400
-    assert info.value.detail["error"]["guardrails"] == ("gr-post",)
+    assert out is not None and out.get("stream") is True
 
 
 @pytest.mark.asyncio
@@ -1618,17 +1603,10 @@ def _echoed_tool_call_dicts(arguments: str) -> List[Dict[str, Any]]:
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize("on_fail, on_error", [("block", None), ("next", "next")])
-@pytest.mark.parametrize(
-    "make_chunks, transform",
-    [
-        (_stream_chunks, lambda inputs: {"texts": ["hello [MASKED]"]}),
-        (_tool_call_stream_chunks, lambda inputs: {"tool_calls": _echoed_tool_call_dicts('{"ssn": "[MASKED]"}')}),
-    ],
-    ids=["texts", "tool_calls"],
-)
-async def test_streaming_iterator_hook_pipeline_withholds_runtime_rewrite(
-    proxy_logging, make_user_api_key_auth, monkeypatch, make_chunks, transform, on_fail, on_error
+async def test_streaming_iterator_hook_pipeline_withholds_runtime_tool_call_rewrite(
+    proxy_logging, make_user_api_key_auth, monkeypatch, on_fail, on_error
 ):
+    transform = lambda inputs: {"tool_calls": _echoed_tool_call_dicts('{"ssn": "[MASKED]"}')}  # noqa: E731
     monkeypatch.setattr(litellm, "callbacks", [_rewriting_stream_guardrail(transform)])
     monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
     step = PipelineStep(guardrail="gr-post", on_pass="allow", on_fail=on_fail, on_error=on_error)
@@ -1638,7 +1616,7 @@ async def test_streaming_iterator_hook_pipeline_withholds_runtime_rewrite(
     async def _drain() -> None:
         async for item in proxy_logging.async_post_call_streaming_iterator_hook(
             user_api_key_dict=make_user_api_key_auth(request_route="/v1/chat/completions"),
-            response=_async_chunk_iter(make_chunks()),
+            response=_async_chunk_iter(_tool_call_stream_chunks()),
             request_data=data,
         ):
             delivered.append(item)
@@ -1655,6 +1633,81 @@ async def test_streaming_iterator_hook_pipeline_withholds_runtime_rewrite(
     assert "stream=false" in error["message"]
 
 
+@pytest.mark.asyncio
+async def test_streaming_iterator_hook_pipeline_delivers_runtime_text_rewrite(
+    proxy_logging, make_user_api_key_auth, monkeypatch
+):
+    transform = lambda inputs: {"texts": ["hello [MASKED]"]}  # noqa: E731
+    monkeypatch.setattr(litellm, "callbacks", [_rewriting_stream_guardrail(transform)])
+    monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
+    data = _post_call_pipeline_data(stream=True)
+    chunks = _stream_chunks()
+
+    delivered = [
+        item
+        async for item in proxy_logging.async_post_call_streaming_iterator_hook(
+            user_api_key_dict=make_user_api_key_auth(request_route="/v1/chat/completions"),
+            response=_async_chunk_iter(chunks),
+            request_data=data,
+        )
+    ]
+
+    assert [id(item) for item in delivered] == [id(chunk) for chunk in chunks]
+    assert delivered[0].choices[0].delta.content == "hello [MASKED]"
+    assert delivered[1].choices[0].delta.content in (None, "")
+    assert delivered[1].choices[0].finish_reason == "stop"
+
+
+@pytest.mark.asyncio
+async def test_streaming_iterator_hook_pipeline_chains_text_rewrites_across_steps(
+    proxy_logging, make_user_api_key_auth, monkeypatch
+):
+    second_step_saw: Dict[str, Any] = {}
+
+    class FirstMask(CustomGuardrail):
+        async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None):
+            return {**inputs, "texts": [text.replace("world", "[MASKED]") for text in inputs["texts"]]}
+
+    class SecondMask(CustomGuardrail):
+        async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None):
+            second_step_saw["texts"] = list(inputs["texts"])
+            return {**inputs, "texts": [text.replace("hello", "[GREETING]") for text in inputs["texts"]]}
+
+    monkeypatch.setattr(
+        litellm,
+        "callbacks",
+        [
+            FirstMask(guardrail_name="gr-first", event_hook=GuardrailEventHooks.post_call, default_on=False),
+            SecondMask(guardrail_name="gr-second", event_hook=GuardrailEventHooks.post_call, default_on=False),
+        ],
+    )
+    monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
+    pipeline = GuardrailPipeline(
+        mode="post_call",
+        steps=[
+            PipelineStep(guardrail="gr-first", on_pass="next", on_fail="block"),
+            PipelineStep(guardrail="gr-second", on_pass="allow", on_fail="block"),
+        ],
+    )
+    data = _post_call_pipeline_data(stream=True)
+    data["metadata"]["_guardrail_pipelines"] = [("response-governance", pipeline)]
+    chunks = _stream_chunks()
+
+    delivered = [
+        item
+        async for item in proxy_logging.async_post_call_streaming_iterator_hook(
+            user_api_key_dict=make_user_api_key_auth(request_route="/v1/chat/completions"),
+            response=_async_chunk_iter(chunks),
+            request_data=data,
+        )
+    ]
+
+    assert second_step_saw["texts"] == ["hello [MASKED]"]
+    assert delivered[0].choices[0].delta.content == "[GREETING] [MASKED]"
+    assert delivered[1].choices[0].delta.content in (None, "")
+    assert delivered[1].choices[0].finish_reason == "stop"
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
     "make_chunks, transform",
@@ -1761,6 +1814,79 @@ async def test_streaming_iterator_hook_pipeline_modify_response_emits_translated
     assert not any(item is chunk for item in delivered for chunk in chunks)
 
 
+@pytest.mark.asyncio
+async def test_streaming_iterator_hook_pipeline_delivers_text_rewrite_on_anthropic_sse(
+    proxy_logging, make_user_api_key_auth, monkeypatch
+):
+    transform = lambda inputs: {"texts": ["hello [MASKED]"]}  # noqa: E731
+    monkeypatch.setattr(litellm, "callbacks", [_rewriting_stream_guardrail(transform)])
+    monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
+    data = _post_call_pipeline_data(stream=True)
+    chunks = _anthropic_sse_chunks()
+
+    delivered = [
+        item
+        async for item in proxy_logging.async_post_call_streaming_iterator_hook(
+            user_api_key_dict=make_user_api_key_auth(request_route="/v1/messages"),
+            response=_async_chunk_iter(chunks),
+            request_data=data,
+        )
+    ]
+
+    raw = b"".join(delivered).decode()
+    assert "hello [MASKED]" in raw
+    assert "hello world" not in raw
+    assert raw.count("event: content_block_delta") == 1
+    for expected_event in ("message_start", "content_block_start", "content_block_stop", "message_delta", "message_stop"):
+        assert f"event: {expected_event}" in raw
+
+
+@pytest.mark.asyncio
+async def test_pipeline_executor_withholds_text_rewrite_when_translation_lacks_write_back(monkeypatch):
+    from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
+    from litellm.proxy.policy_engine.pipeline_executor import PipelineExecutor, UndeliverableStreamRewrite
+
+    class NoWriteBackTranslation(BaseTranslation):
+        async def process_input_messages(self, data, guardrail_to_apply, litellm_logging_obj):
+            return data
+
+        async def process_output_response(self, response, guardrail_to_apply, litellm_logging_obj, **kwargs):
+            return response
+
+        async def process_output_streaming_response(
+            self,
+            responses_so_far,
+            guardrail_to_apply,
+            litellm_logging_obj,
+            user_api_key_dict=None,
+            request_data=None,
+            stream_transform_sink=None,
+            deliver_ended_stream_rewrites=False,
+        ):
+            assert deliver_ended_stream_rewrites is False
+            await guardrail_to_apply.apply_guardrail(
+                inputs={"texts": ["hello world"]},
+                request_data=request_data or {},
+                input_type="response",
+            )
+            return responses_so_far
+
+    transform = lambda inputs: {"texts": ["hello [MASKED]"]}  # noqa: E731
+    monkeypatch.setattr(litellm, "callbacks", [_rewriting_stream_guardrail(transform)])
+
+    with pytest.raises(UndeliverableStreamRewrite):
+        await PipelineExecutor.execute_steps(
+            steps=[PipelineStep(guardrail="gr-post", on_pass="allow", on_fail="block")],
+            mode="post_call",
+            data={"metadata": {}},
+            user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
+            call_type="acompletion",
+            policy_name="response-governance",
+            streaming_chunks=_stream_chunks(),
+            endpoint_translation=NoWriteBackTranslation(),
+        )
+
+
 @pytest.mark.asyncio
 async def test_streaming_iterator_hook_pipeline_gates_without_iterator_overrides(
     proxy_logging, make_user_api_key_auth, monkeypatch

From 85fea1a6752ac5f0c4c0415c5d3da4f6bc542911 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 1 Sep 2026 17:15:11 -0700
Subject: [PATCH 17/85] fix(policy_engine): move mutable-ok suppressions onto
 the flagged lines

---
 litellm/llms/openai/chat/guardrail_translation/handler.py | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py
index fcb447d5277..02206a36b0e 100644
--- a/litellm/llms/openai/chat/guardrail_translation/handler.py
+++ b/litellm/llms/openai/chat/guardrail_translation/handler.py
@@ -989,12 +989,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
             return
         await self._apply_guardrail_responses_to_output_streaming(
             responses=responses_so_far,
-            guardrailed_texts=[
-                after for _choice_idx, after in changed
-            ],  # mutable-ok: the callee's signature predates this change and takes lists
-            task_mappings=[
-                (choice_idx, None) for choice_idx, _after in changed
-            ],  # mutable-ok: the callee's signature predates this change and takes lists
+            guardrailed_texts=[after for _choice_idx, after in changed],  # mutable-ok: callee takes lists
+            task_mappings=[(choice_idx, None) for choice_idx, _after in changed],  # mutable-ok: callee takes lists
         )
 
     async def _apply_guardrail_responses_to_output_streaming(

From 4fbe4ce2e225940b009d390543bd76694eece343 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 1 Sep 2026 17:29:52 -0700
Subject: [PATCH 18/85] fix(guardrail_translation): deliver stream rewrites on
 incomplete and failed responses terminals

---
 .../guardrail_translation/handler.py          | 60 ++++++++++++-------
 ...test_openai_responses_guardrail_handler.py | 56 +++++++++++++++++
 2 files changed, 93 insertions(+), 23 deletions(-)

diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py
index ac894401628..3c902f1b829 100644
--- a/litellm/llms/openai/responses/guardrail_translation/handler.py
+++ b/litellm/llms/openai/responses/guardrail_translation/handler.py
@@ -86,6 +86,15 @@ class ResponsesStreamChunk(TypedDict, total=False):
     text: ReadOnly[str]
 
 
+_TERMINAL_ENVELOPE_EVENT_TYPES: Final = frozenset(
+    {
+        ResponsesAPIStreamEvents.RESPONSE_COMPLETED.value,
+        ResponsesAPIStreamEvents.RESPONSE_FAILED.value,
+        ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE.value,
+    }
+)
+
+
 def _next_stream_sequence_number(responses_so_far: Sequence[Any] | None) -> int:
     sequence_numbers: Final = (
         item.get("sequence_number") if isinstance(item, dict) else getattr(item, "sequence_number", None)
@@ -507,14 +516,17 @@ class OpenAIResponsesHandler(BaseTranslation):
         chunk, apply the guardrail, then write the result back in-place so the
         caller sees the modified content (e.g. PII tokens replaced).
 
-        For ``response.completed`` events (the normal end-of-stream signal) we
-        use the same per-item extraction + task-mapping approach as
-        ``process_output_response`` so that unmasking / blocking works correctly
-        for every output item. With ``deliver_ended_stream_rewrites`` the earlier
-        text-carrying events (``response.output_text.delta`` / ``.done``,
+        For terminal envelope events (``response.completed``, and equally
+        ``response.incomplete`` / ``response.failed``, whose envelopes carry the
+        partial output) we use the same per-item extraction + task-mapping
+        approach as ``process_output_response`` so that unmasking / blocking
+        works correctly for every output item. With
+        ``deliver_ended_stream_rewrites`` the earlier text-carrying events
+        (``response.output_text.delta`` / ``.done``,
         ``response.content_part.done``, ``response.output_item.done``) are synced
-        to the rewritten completed response too, so a client reading deltas sees
-        the rewrite instead of the raw model output.
+        to the rewritten envelope too, so a client reading deltas sees the
+        rewrite instead of the raw model output; a rewrite observed where no
+        write-back is possible fails closed instead of releasing raw output.
         """
         if not responses_so_far:
             return responses_so_far
@@ -526,14 +538,16 @@ class OpenAIResponsesHandler(BaseTranslation):
             return responses_so_far
 
         # ------------------------------------------------------------------ #
-        # Case 1: response.completed — full response is available in the      #
-        # final chunk; iterate output items, apply guardrail, write back.     #
+        # Case 1: terminal envelope events (completed/incomplete/failed).     #
+        # the accumulated response is available in the final chunk; iterate   #
+        # output items, apply guardrail, write back. Falls through to the     #
+        # string fallback when the envelope yields nothing to check.          #
         # ------------------------------------------------------------------ #
-        if final_chunk.get("type") == "response.completed":
+        if final_chunk.get("type") in _TERMINAL_ENVELOPE_EVENT_TYPES:
             response_obj: Final[ResponseOutputEnvelope] = final_chunk.get("response") or {}
-            if not hasattr(response_obj, "get"):
-                return responses_so_far
-            outputs: Final[Sequence[object]] = response_obj.get("output") or []
+            outputs: Final[Sequence[object]] = (
+                (response_obj.get("output") or []) if hasattr(response_obj, "get") else []
+            )
 
             texts_to_check: Final[list[str]] = []
             tool_calls_to_check: Final[list[ChatCompletionToolCallChunk]] = []
@@ -596,8 +610,7 @@ class OpenAIResponsesHandler(BaseTranslation):
                             stream_events=responses_so_far[:-1],
                             rewrites_by_position=rewrites_by_position,
                         )
-
-            return responses_so_far
+                return responses_so_far
 
         # ------------------------------------------------------------------ #
         # Case 2: response.output_item.done — extract tool calls only.        #
@@ -623,7 +636,8 @@ class OpenAIResponsesHandler(BaseTranslation):
         # ------------------------------------------------------------------ #
         # Fallback: apply guardrail to the accumulated text string.           #
         # No structured write-back is possible here; guardrails that only     #
-        # need to block/flag (not rewrite) still work correctly.             #
+        # need to block/flag (not rewrite) still work correctly, and a        #
+        # rewrite a caller expects delivered fails closed instead.            #
         # ------------------------------------------------------------------ #
         string_so_far: Final = self.get_streaming_string_so_far(responses_so_far)
         if string_so_far:
@@ -633,12 +647,17 @@ class OpenAIResponsesHandler(BaseTranslation):
             )
             if response_model:
                 fallback_inputs["model"] = response_model
-            await guardrail_to_apply.apply_guardrail(
+            fallback_outputs: Final = await guardrail_to_apply.apply_guardrail(
                 inputs=fallback_inputs,
                 request_data=request_data if request_data is not None else {},
                 input_type="response",
                 logging_obj=litellm_logging_obj,
             )
+            fallback_texts: Final = fallback_outputs.get("texts")
+            if deliver_ended_stream_rewrites and fallback_texts and tuple(fallback_texts) != (string_so_far,):
+                from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
+
+                raise UndeliverableStreamRewrite(guardrail_to_apply.guardrail_name or "unknown")
         return responses_so_far
 
     @staticmethod
@@ -704,12 +723,7 @@ class OpenAIResponsesHandler(BaseTranslation):
         """
         if not responses_so_far:
             return False
-        terminal_types: Final = {
-            ResponsesAPIStreamEvents.RESPONSE_COMPLETED.value,
-            ResponsesAPIStreamEvents.RESPONSE_FAILED.value,
-            ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE.value,
-        }
-        return responses_so_far[-1].get("type") in terminal_types
+        return responses_so_far[-1].get("type") in _TERMINAL_ENVELOPE_EVENT_TYPES
 
     def build_stream_error_items(
         self,
diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py
index 4f95e08cb71..dfd6352f9ec 100644
--- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py
+++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py
@@ -1171,6 +1171,62 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing:
         assert events[4]["item"]["content"][0]["text"] == "hello [MASKED]"
         assert events[5]["response"]["output"][0]["content"][0]["text"] == "hello [MASKED]"
 
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("terminal_type", ["response.incomplete", "response.failed"])
+    async def test_deliver_ended_stream_rewrites_syncs_non_completed_terminals(self, terminal_type):
+        handler = OpenAIResponsesHandler()
+        events = self._ended_stream_events()
+        events[-1]["type"] = terminal_type
+        events[-1]["response"]["status"] = terminal_type.split(".")[-1]
+
+        result = await handler.process_output_streaming_response(
+            responses_so_far=events,
+            guardrail_to_apply=self._masking_guardrail(),
+            litellm_logging_obj=None,
+            deliver_ended_stream_rewrites=True,
+        )
+
+        assert result is events
+        assert events[0]["delta"] == "hello [MASKED]"
+        assert events[1]["delta"] == ""
+        assert events[2]["text"] == "hello [MASKED]"
+        assert events[3]["part"]["text"] == "hello [MASKED]"
+        assert events[4]["item"]["content"][0]["text"] == "hello [MASKED]"
+        assert events[5]["response"]["output"][0]["content"][0]["text"] == "hello [MASKED]"
+
+    @pytest.mark.asyncio
+    async def test_fallback_rewrite_with_delivery_expected_fails_closed(self):
+        from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
+
+        handler = OpenAIResponsesHandler()
+        events = [
+            {"type": "response.output_text.delta", "output_index": 0, "content_index": 0, "delta": "hello "},
+            {"type": "response.output_text.done", "output_index": 0, "content_index": 0, "text": "hello world"},
+        ]
+
+        with pytest.raises(UndeliverableStreamRewrite):
+            await handler.process_output_streaming_response(
+                responses_so_far=events,
+                guardrail_to_apply=self._masking_guardrail(),
+                litellm_logging_obj=None,
+                deliver_ended_stream_rewrites=True,
+            )
+
+    @pytest.mark.asyncio
+    async def test_fallback_rewrite_without_delivery_expected_does_not_raise(self):
+        handler = OpenAIResponsesHandler()
+        events = [
+            {"type": "response.output_text.done", "output_index": 0, "content_index": 0, "text": "hello world"},
+        ]
+
+        result = await handler.process_output_streaming_response(
+            responses_so_far=events,
+            guardrail_to_apply=self._masking_guardrail(),
+            litellm_logging_obj=None,
+        )
+
+        assert result is events
+
     @pytest.mark.asyncio
     async def test_ended_stream_rewrite_leaves_delta_events_untouched_by_default(self):
         handler = OpenAIResponsesHandler()

From c9435b5ff3f72c6d9c5ebfa30175498bc5928daf Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 1 Sep 2026 18:00:42 -0700
Subject: [PATCH 19/85] fix(guardrails): key stream rewrites by choice index
 and scan delta-only responses buffers

Chat streaming write-backs now match chunks by the choice's index field
instead of its list position, delivering rewrites to the right choice on
n>1 streams; an ended-stream rewrite on a multi-choice buffer fails
closed since stream_chunk_builder collapses the choices. The Responses
fallback joins output_text.delta events when delivery is expected, so a
delta-only buffer is guardrail-checked instead of released raw.
---
 .../chat/guardrail_translation/handler.py     | 50 ++++++----
 .../guardrail_translation/handler.py          | 31 +++++--
 .../test_openai_guardrail_handler.py          | 93 +++++++++++++++++++
 ...test_openai_responses_guardrail_handler.py | 71 ++++++++++++++
 4 files changed, 222 insertions(+), 23 deletions(-)

diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py
index 02206a36b0e..182dec81937 100644
--- a/litellm/llms/openai/chat/guardrail_translation/handler.py
+++ b/litellm/llms/openai/chat/guardrail_translation/handler.py
@@ -620,6 +620,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
                 responses_so_far=responses_so_far,
                 guardrailed_response=model_response,
                 pre_guardrail_texts=pre_guardrail_texts,
+                guardrail_name=guardrail_to_apply.guardrail_name or "unknown",
             )
 
     def build_stream_error_items(
@@ -747,8 +748,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         """
         combined_texts: Final[dict[tuple[int, int | None], str]] = {}
 
-        for response_idx, response in enumerate(responses_so_far):
-            for choice_idx, choice in enumerate(response.choices):
+        for response in responses_so_far:
+            for choice in response.choices:
                 if isinstance(choice, litellm.StreamingChoices):
                     content = choice.delta.content
                 elif isinstance(choice, litellm.Choices):
@@ -761,7 +762,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
 
                 if isinstance(content, str):
                     # String content - accumulate for this choice
-                    str_key: tuple[int, int | None] = (choice_idx, None)
+                    str_key: tuple[int, int | None] = (choice.index, None)
                     if str_key not in combined_texts:
                         combined_texts[str_key] = ""
                     combined_texts[str_key] += content
@@ -772,7 +773,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
                         text_str = content_item.get("text")
                         if text_str:
                             list_key: tuple[int, int | None] = (
-                                choice_idx,
+                                choice.index,
                                 content_idx,
                             )
                             if list_key not in combined_texts:
@@ -973,24 +974,38 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         responses_so_far: list["ModelResponseStream"],  # mutable-ok: rewrites the caller's buffered chunks in place
         guardrailed_response: "ModelResponse",
         pre_guardrail_texts: tuple[str | None, ...],
+        guardrail_name: str,
     ) -> None:
         """Write ended-stream guardrail text rewrites back across the buffered
-        chunks: each rewritten choice's full text lands in its first
+        chunks: the full rewritten text lands in the choice's first
         content-carrying chunk and the rest are blanked, the same shape the
         in-flight write-back uses. Chunks carrying only finish_reason or usage
-        stay untouched."""
+        stay untouched. A rewrite on a stream carrying more than one distinct
+        choice index fails closed."""
         post_guardrail_texts: Final = self._string_choice_contents(guardrailed_response)
         changed: Final = tuple(
-            (choice_idx, after)
-            for choice_idx, (before, after) in enumerate(zip(pre_guardrail_texts, post_guardrail_texts))
+            after
+            for before, after in zip(pre_guardrail_texts, post_guardrail_texts)
             if before is not None and after is not None and after != before
         )
         if not changed:
             return
+        stream_choice_indices: Final = frozenset(
+            choice.index for response in responses_so_far for choice in response.choices
+        )
+        if len(stream_choice_indices) != 1:
+            # stream_chunk_builder collapses every choice into one index-0
+            # choice, so a rewrite of the rebuilt response cannot be attributed
+            # back to a single choice on an n>1 stream: withhold the stream
+            # rather than deliver the rewrite on the wrong choice
+            from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
+
+            raise UndeliverableStreamRewrite(guardrail_name)
+        target_choice_index: Final = next(iter(stream_choice_indices))
         await self._apply_guardrail_responses_to_output_streaming(
             responses=responses_so_far,
-            guardrailed_texts=[after for _choice_idx, after in changed],  # mutable-ok: callee takes lists
-            task_mappings=[(choice_idx, None) for choice_idx, _after in changed],  # mutable-ok: callee takes lists
+            guardrailed_texts=list(changed),  # mutable-ok: callee takes lists
+            task_mappings=[(target_choice_index, None) for _ in changed],  # mutable-ok: callee takes lists
         )
 
     async def _apply_guardrail_responses_to_output_streaming(
@@ -1008,7 +1023,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         Args:
             responses: List of ModelResponseStream objects to modify
             guardrailed_texts: List of guardrailed text responses (combined from all chunks)
-            task_mappings: List of tuples (choice_idx, content_idx)
+            task_mappings: List of tuples (choice_idx, content_idx), where choice_idx
+                is the choice's ``index`` field, not its position in a chunk's list
 
         Override this method to customize how responses are applied to streaming responses.
         """
@@ -1024,9 +1040,11 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         # Key: (choice_idx, content_idx), Value: boolean (True if already set)
         already_set: Final[dict[tuple[int, int | None], bool]] = {}
 
-        # Iterate through all responses and update content
-        for response_idx, response in enumerate(responses):
-            for choice_idx_in_response, choice in enumerate(response.choices):
+        # Iterate through all responses and update content, matching each chunk's
+        # choice by its index field: on n>1 streams a chunk usually carries one
+        # choice at list position 0 whose index names the logical choice.
+        for response in responses:
+            for choice in response.choices:
                 if isinstance(choice, litellm.StreamingChoices):
                     content = choice.delta.content
                 elif isinstance(choice, litellm.Choices):
@@ -1039,7 +1057,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
 
                 if isinstance(content, str):
                     # String content
-                    str_key: tuple[int, int | None] = (choice_idx_in_response, None)
+                    str_key: tuple[int, int | None] = (choice.index, None)
                     if str_key in guardrail_map:
                         if str_key not in already_set:
                             # First chunk - set the complete guardrailed text
@@ -1060,7 +1078,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
                     for content_idx, content_item in enumerate(content):
                         if "text" in content_item:
                             list_key: tuple[int, int | None] = (
-                                choice_idx_in_response,
+                                choice.index,
                                 content_idx,
                             )
                             if list_key in guardrail_map:
diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py
index 3c902f1b829..ebd1070a8e8 100644
--- a/litellm/llms/openai/responses/guardrail_translation/handler.py
+++ b/litellm/llms/openai/responses/guardrail_translation/handler.py
@@ -634,14 +634,20 @@ class OpenAIResponsesHandler(BaseTranslation):
             return responses_so_far
 
         # ------------------------------------------------------------------ #
-        # Fallback: apply guardrail to the accumulated text string.           #
-        # No structured write-back is possible here; guardrails that only     #
-        # need to block/flag (not rewrite) still work correctly, and a        #
-        # rewrite a caller expects delivered fails closed instead.            #
+        # Fallback: apply guardrail to the accumulated text string. When a    #
+        # caller expects rewrites delivered and only output_text.delta events #
+        # carried the text (a stream cut off before any .done or terminal     #
+        # envelope), the delta text is scanned instead so nothing escapes     #
+        # unchecked. No structured write-back is possible here; guardrails    #
+        # that only need to block/flag (not rewrite) still work correctly,    #
+        # and a rewrite a caller expects delivered fails closed instead.      #
         # ------------------------------------------------------------------ #
         string_so_far: Final = self.get_streaming_string_so_far(responses_so_far)
-        if string_so_far:
-            fallback_inputs: Final = GenericGuardrailAPIInputs(texts=[string_so_far])
+        text_to_check: Final = string_so_far or (
+            self._delta_text_so_far(responses_so_far) if deliver_ended_stream_rewrites else ""
+        )
+        if text_to_check:
+            fallback_inputs: Final = GenericGuardrailAPIInputs(texts=[text_to_check])
             response_model = (
                 final_chunk.get("response", {}).get("model") if isinstance(final_chunk.get("response"), dict) else None
             )
@@ -654,7 +660,7 @@ class OpenAIResponsesHandler(BaseTranslation):
                 logging_obj=litellm_logging_obj,
             )
             fallback_texts: Final = fallback_outputs.get("texts")
-            if deliver_ended_stream_rewrites and fallback_texts and tuple(fallback_texts) != (string_so_far,):
+            if deliver_ended_stream_rewrites and fallback_texts and tuple(fallback_texts) != (text_to_check,):
                 from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
 
                 raise UndeliverableStreamRewrite(guardrail_to_apply.guardrail_name or "unknown")
@@ -754,6 +760,17 @@ class OpenAIResponsesHandler(BaseTranslation):
         """
         return "".join([response.get("text", "") for response in responses_so_far])
 
+    @staticmethod
+    def _delta_text_so_far(responses_so_far: Sequence[ResponsesStreamChunk]) -> str:
+        """Accumulate the text carried by ``response.output_text.delta`` events,
+        for buffers where no ``.done`` event or terminal envelope repeats it."""
+        deltas: Final = (
+            response.get("delta")
+            for response in responses_so_far
+            if response.get("type") == ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA.value
+        )
+        return "".join(delta for delta in deltas if isinstance(delta, str))
+
     def _has_text_content(self, response: "ResponsesAPIResponse") -> bool:
         """
         Check if response has any text content to process.
diff --git a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py
index 26442a4a6ed..b177d4a73d4 100644
--- a/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py
+++ b/tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py
@@ -1128,6 +1128,99 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
         assert chunks[1].choices[0].delta.content == " world"
         assert chunks[1].choices[0].finish_reason == "stop"
 
+    @staticmethod
+    def _two_choice_stream_chunks() -> list:
+        from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
+
+        def chunk(index: int, content: str, finish_reason: Optional[str] = None) -> ModelResponseStream:
+            return ModelResponseStream(
+                id="chatcmpl-123",
+                created=1234567890,
+                model="gpt-4",
+                object="chat.completion.chunk",
+                choices=[StreamingChoices(index=index, delta=Delta(content=content), finish_reason=finish_reason)],
+            )
+
+        return [
+            chunk(0, "safe "),
+            chunk(1, "hello "),
+            chunk(0, "text", "stop"),
+            chunk(1, "world", "stop"),
+        ]
+
+    @staticmethod
+    def _world_masking_guardrail() -> CustomGuardrail:
+        class MaskWorld(CustomGuardrail):
+            async def apply_guardrail(
+                self,
+                inputs: GenericGuardrailAPIInputs,
+                request_data: dict,
+                input_type: Literal["request", "response"],
+                logging_obj: Optional[Any] = None,
+            ) -> GenericGuardrailAPIInputs:
+                texts = inputs.get("texts", [])
+                return {**inputs, "texts": [t.replace("world", "[MASKED]") for t in texts]}
+
+        return MaskWorld(guardrail_name="test-mask")
+
+    @pytest.mark.asyncio
+    async def test_deliver_ended_stream_rewrite_on_multi_choice_stream_fails_closed(self):
+        from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
+
+        handler = OpenAIChatCompletionsHandler()
+        chunks = self._two_choice_stream_chunks()
+
+        with pytest.raises(UndeliverableStreamRewrite):
+            await handler.process_output_streaming_response(
+                responses_so_far=chunks,
+                guardrail_to_apply=self._world_masking_guardrail(),
+                litellm_logging_obj=None,
+                deliver_ended_stream_rewrites=True,
+            )
+
+    @pytest.mark.asyncio
+    async def test_deliver_ended_stream_clean_multi_choice_stream_released_untouched(self):
+        handler = OpenAIChatCompletionsHandler()
+        chunks = self._two_choice_stream_chunks()
+
+        result = await handler.process_output_streaming_response(
+            responses_so_far=chunks,
+            guardrail_to_apply=MockPassThroughGuardrail(guardrail_name="test"),
+            litellm_logging_obj=None,
+            deliver_ended_stream_rewrites=True,
+        )
+
+        assert result is chunks
+        assert [c.choices[0].delta.content for c in chunks] == ["safe ", "hello ", "text", "world"]
+
+    @pytest.mark.asyncio
+    async def test_deliver_ended_stream_rewrite_lands_on_nonzero_choice_index(self):
+        from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
+
+        handler = OpenAIChatCompletionsHandler()
+
+        def chunk(content: str, finish_reason: Optional[str]) -> ModelResponseStream:
+            return ModelResponseStream(
+                id="chatcmpl-123",
+                created=1234567890,
+                model="gpt-4",
+                object="chat.completion.chunk",
+                choices=[StreamingChoices(index=1, delta=Delta(content=content), finish_reason=finish_reason)],
+            )
+
+        chunks = [chunk("hello ", None), chunk("world", "stop")]
+
+        result = await handler.process_output_streaming_response(
+            responses_so_far=chunks,
+            guardrail_to_apply=self._world_masking_guardrail(),
+            litellm_logging_obj=None,
+            deliver_ended_stream_rewrites=True,
+        )
+
+        assert result is chunks
+        assert chunks[0].choices[0].delta.content == "hello [MASKED]"
+        assert chunks[1].choices[0].delta.content in (None, "")
+
 
 class TestGetStructuredMessages:
     """Test the get_structured_messages method."""
diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py
index dfd6352f9ec..ae3700b1123 100644
--- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py
+++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py
@@ -1212,6 +1212,77 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing:
                 deliver_ended_stream_rewrites=True,
             )
 
+    @staticmethod
+    def _recording_guardrail() -> "tuple[CustomGuardrail, List[List[str]]]":
+        seen: List[List[str]] = []
+
+        class Recorder(CustomGuardrail):
+            async def apply_guardrail(
+                self,
+                inputs: GenericGuardrailAPIInputs,
+                request_data: dict,
+                input_type: Literal["request", "response"],
+                logging_obj: Optional[Any] = None,
+            ) -> GenericGuardrailAPIInputs:
+                seen.append(list(inputs.get("texts", [])))
+                return inputs
+
+        return Recorder(guardrail_name="recorder"), seen
+
+    @pytest.mark.asyncio
+    async def test_fallback_delta_only_rewrite_with_delivery_expected_fails_closed(self):
+        from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
+
+        handler = OpenAIResponsesHandler()
+        events = [
+            {"type": "response.output_text.delta", "output_index": 0, "content_index": 0, "delta": "hello "},
+            {"type": "response.output_text.delta", "output_index": 0, "content_index": 0, "delta": "world"},
+        ]
+
+        with pytest.raises(UndeliverableStreamRewrite):
+            await handler.process_output_streaming_response(
+                responses_so_far=events,
+                guardrail_to_apply=self._masking_guardrail(),
+                litellm_logging_obj=None,
+                deliver_ended_stream_rewrites=True,
+            )
+
+    @pytest.mark.asyncio
+    async def test_fallback_scans_delta_text_when_delivery_expected(self):
+        handler = OpenAIResponsesHandler()
+        guardrail, seen = self._recording_guardrail()
+        events = [
+            {"type": "response.output_text.delta", "output_index": 0, "content_index": 0, "delta": "hello "},
+            {"type": "response.output_text.delta", "output_index": 0, "content_index": 0, "delta": "there"},
+        ]
+
+        result = await handler.process_output_streaming_response(
+            responses_so_far=events,
+            guardrail_to_apply=guardrail,
+            litellm_logging_obj=None,
+            deliver_ended_stream_rewrites=True,
+        )
+
+        assert result is events
+        assert seen == [["hello there"]]
+
+    @pytest.mark.asyncio
+    async def test_fallback_ignores_delta_text_without_delivery_expected(self):
+        handler = OpenAIResponsesHandler()
+        guardrail, seen = self._recording_guardrail()
+        events = [
+            {"type": "response.output_text.delta", "output_index": 0, "content_index": 0, "delta": "hello world"},
+        ]
+
+        result = await handler.process_output_streaming_response(
+            responses_so_far=events,
+            guardrail_to_apply=guardrail,
+            litellm_logging_obj=None,
+        )
+
+        assert result is events
+        assert seen == []
+
     @pytest.mark.asyncio
     async def test_fallback_rewrite_without_delivery_expected_does_not_raise(self):
         handler = OpenAIResponsesHandler()

From c09db7c7a3d77d3ccd6a1a2163778b6f22f29a8f Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 1 Sep 2026 22:01:04 -0700
Subject: [PATCH 20/85] Fail closed on rewrites for buffers that never reached
 their terminal event

An Anthropic buffer without a stop_reason only ran the flat text scan, so a
rewrite there was dropped while the executor trusted the translation to have
delivered it. A Responses buffer ending at response.output_item.done returned
after the tool-call scan without ever checking the text. Both now reach the
flat scan and raise UndeliverableStreamRewrite when a caller expects the
rewrite delivered, matching the existing Responses no-envelope fallback.
---
 .../chat/guardrail_translation/handler.py     |  8 +++-
 .../guardrail_translation/handler.py          |  7 ++-
 .../test_anthropic_guardrail_handler.py       | 46 +++++++++++++++++++
 ...test_openai_responses_guardrail_handler.py | 46 +++++++++++++++++++
 4 files changed, 104 insertions(+), 3 deletions(-)

diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py
index 091ad8ecba9..fc276dd7fe6 100644
--- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py
+++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py
@@ -1020,7 +1020,8 @@ class AnthropicMessagesHandler(BaseTranslation):
 
         Get the string so far, check the apply guardrail to the string so far, and return the list of responses so far.
         With ``deliver_ended_stream_rewrites``, an ended stream whose guardrail rewrote the text gets the rewrite
-        written back across the buffered chunks (full rewritten text in the first ``text_delta``, the rest blanked).
+        written back across the buffered chunks (full rewritten text in the first ``text_delta``, the rest blanked);
+        a rewrite on a stream that never reported a ``stop_reason`` has no write-back and fails closed instead.
         """
         from litellm.integrations.custom_guardrail import ModifyResponseException
 
@@ -1098,6 +1099,11 @@ class AnthropicMessagesHandler(BaseTranslation):
             if e.original_response is None:
                 e.original_response = self._build_streaming_usage_response(responses_so_far, request_data)
             raise
+        unended_texts: Final = _guardrailed_inputs.get("texts")
+        if deliver_ended_stream_rewrites and unended_texts and tuple(unended_texts) != (string_so_far,):
+            from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
+
+            raise UndeliverableStreamRewrite(guardrail_to_apply.guardrail_name or "unknown")
         return responses_so_far
 
     def _prepare_request_data(
diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py
index 8b808348c79..724a0a1d2f0 100644
--- a/litellm/llms/openai/responses/guardrail_translation/handler.py
+++ b/litellm/llms/openai/responses/guardrail_translation/handler.py
@@ -638,7 +638,9 @@ class OpenAIResponsesHandler(BaseTranslation):
                 return responses_so_far
 
         # ------------------------------------------------------------------ #
-        # Case 2: response.output_item.done — extract tool calls only.        #
+        # Case 2: response.output_item.done — extract tool calls only, then  #
+        # fall through to the text fallback when a caller expects rewrites   #
+        # delivered, so a buffer truncated here still fails closed on text.  #
         # ------------------------------------------------------------------ #
         if final_chunk.get("type") == "response.output_item.done":
             model_response_stream: Final = (
@@ -656,7 +658,8 @@ class OpenAIResponsesHandler(BaseTranslation):
                     input_type="response",
                     logging_obj=litellm_logging_obj,
                 )
-            return responses_so_far
+            if not deliver_ended_stream_rewrites:
+                return responses_so_far
 
         # ------------------------------------------------------------------ #
         # Fallback: apply guardrail to the accumulated text string.           #
diff --git a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py
index f60655cd7ec..274c351ebc7 100644
--- a/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py
+++ b/tests/test_litellm/llms/anthropic/chat/guardrail_translation/test_anthropic_guardrail_handler.py
@@ -328,6 +328,52 @@ class TestAnthropicMessagesHandlerStreamingOutputProcessing:
 
         assert chunks == original
 
+    @pytest.mark.asyncio
+    async def test_unended_stream_rewrite_with_delivery_expected_fails_closed(self):
+        from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
+
+        handler = AnthropicMessagesHandler()
+        chunks = self._ended_sse_chunks()[:-2]
+
+        with pytest.raises(UndeliverableStreamRewrite):
+            await handler.process_output_streaming_response(
+                responses_so_far=chunks,
+                guardrail_to_apply=self._masking_guardrail(),
+                litellm_logging_obj=MagicMock(),
+                deliver_ended_stream_rewrites=True,
+            )
+
+    @pytest.mark.asyncio
+    async def test_unended_stream_without_rewrite_is_released_with_delivery_expected(self):
+        handler = AnthropicMessagesHandler()
+        chunks = self._ended_sse_chunks()[:-2]
+        original = [bytes(chunk) for chunk in chunks]
+
+        result = await handler.process_output_streaming_response(
+            responses_so_far=chunks,
+            guardrail_to_apply=MockPassThroughGuardrail(guardrail_name="test"),
+            litellm_logging_obj=MagicMock(),
+            deliver_ended_stream_rewrites=True,
+        )
+
+        assert result is chunks
+        assert chunks == original
+
+    @pytest.mark.asyncio
+    async def test_unended_stream_rewrite_without_delivery_expected_does_not_raise(self):
+        handler = AnthropicMessagesHandler()
+        chunks = self._ended_sse_chunks()[:-2]
+        original = [bytes(chunk) for chunk in chunks]
+
+        result = await handler.process_output_streaming_response(
+            responses_so_far=chunks,
+            guardrail_to_apply=self._masking_guardrail(),
+            litellm_logging_obj=MagicMock(),
+        )
+
+        assert result is chunks
+        assert chunks == original
+
 
 class TestAnthropicMessagesHandlerInputProcessing:
     """Test input processing preserves litellm_metadata for dynamic guardrails."""
diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py
index 155b9ef9810..bd07e924d12 100644
--- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py
+++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py
@@ -1248,6 +1248,52 @@ class TestOpenAIResponsesHandlerStreamingOutputProcessing:
                 deliver_ended_stream_rewrites=True,
             )
 
+    @pytest.mark.asyncio
+    async def test_output_item_done_last_rewrite_with_delivery_expected_fails_closed(self):
+        from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
+
+        handler = OpenAIResponsesHandler()
+        events = self._ended_stream_events()[:-1]
+
+        with pytest.raises(UndeliverableStreamRewrite):
+            await handler.process_output_streaming_response(
+                responses_so_far=events,
+                guardrail_to_apply=self._masking_guardrail(),
+                litellm_logging_obj=None,
+                deliver_ended_stream_rewrites=True,
+            )
+
+    @pytest.mark.asyncio
+    async def test_output_item_done_last_scans_text_with_delivery_expected(self):
+        handler = OpenAIResponsesHandler()
+        events = self._ended_stream_events()[:-1]
+        guardrail = MockRecordingGuardrail(guardrail_name="test")
+
+        result = await handler.process_output_streaming_response(
+            responses_so_far=events,
+            guardrail_to_apply=guardrail,
+            litellm_logging_obj=None,
+            deliver_ended_stream_rewrites=True,
+        )
+
+        assert result is events
+        assert [inputs.get("texts") for inputs in guardrail.seen_inputs] == [["hello world"]]
+
+    @pytest.mark.asyncio
+    async def test_output_item_done_last_without_delivery_expected_skips_text(self):
+        handler = OpenAIResponsesHandler()
+        events = self._ended_stream_events()[:-1]
+        guardrail = MockRecordingGuardrail(guardrail_name="test")
+
+        result = await handler.process_output_streaming_response(
+            responses_so_far=events,
+            guardrail_to_apply=guardrail,
+            litellm_logging_obj=None,
+        )
+
+        assert result is events
+        assert guardrail.seen_inputs == []
+
     @pytest.mark.asyncio
     async def test_fallback_rewrite_without_delivery_expected_does_not_raise(self):
         handler = OpenAIResponsesHandler()

From 55c7872496b5568bc259fc88c0645f702af1576e Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Wed, 2 Sep 2026 12:57:06 -0700
Subject: [PATCH 21/85] fix(proxy-extras): rebuild indexes left INVALID by a
 migration deadlock

Two replicas racing prisma migrate deploy can deadlock, and the loser dies
mid CREATE INDEX CONCURRENTLY, leaving the index INVALID. The retried
migration's IF NOT EXISTS then skips it, so the planner never uses it.

After migrations succeed, look for INVALID indexes on LiteLLM tables and
have one replica (advisory try-lock) REINDEX INDEX CONCURRENTLY each of
them, dropping _ccnew/_ccold leftovers of an interrupted rebuild instead.
The repair never blocks startup: a failed rebuild is logged and retried
on the next boot. Also encode DATABASE_URL query values with quote instead
of quote_plus so options=-c%20... reaches psycopg intact.
---
 .../litellm_proxy_extras/utils.py             | 128 +++++++++-
 .../test_invalid_index_repair.py              | 238 ++++++++++++++++++
 2 files changed, 359 insertions(+), 7 deletions(-)
 create mode 100644 tests/proxy_migration_tests/test_invalid_index_repair.py

diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py
index b8032dd0d28..df019d78078 100644
--- a/litellm-proxy-extras/litellm_proxy_extras/utils.py
+++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py
@@ -6,18 +6,23 @@ import shutil
 import subprocess
 import tempfile
 import time
+from dataclasses import dataclass
 from pathlib import Path
-from typing import Optional
+from typing import TYPE_CHECKING, Final, Optional
 
 from litellm_proxy_extras._logging import logger
-from litellm_proxy_extras.replica_identity import (
-    REPLICA_IDENTITY_FULL_ENV_VAR,
-    apply_replica_identity_full,
-)
 from litellm_proxy_extras.prisma_toolchain import (
     ensure_prisma_toolchain,
     prisma_command_timeout,
 )
+from litellm_proxy_extras.replica_identity import (
+    REPLICA_IDENTITY_FULL_ENV_VAR,
+    apply_replica_identity_full,
+)
+
+if TYPE_CHECKING:
+    import psycopg
+    import psycopg.sql
 
 
 def str_to_bool(value: Optional[str]) -> bool:
@@ -40,6 +45,29 @@ def _get_prisma_env() -> dict:
 
 _MIGRATION_TS_RE = re.compile(r"^(\d{14})_")
 
+INDEX_REPAIR_ADVISORY_LOCK_KEY: Final = int.from_bytes(b"litellm", "big")
+_TRANSIENT_INDEX_SUFFIX_RE: Final = re.compile(r"_cc(?:new|old)\d*$")
+_INVALID_LITELLM_INDEXES_SQL: Final = (
+    "SELECT n.nspname, c.relname, pg_size_pretty(pg_table_size(t.oid)) "
+    "FROM pg_index i "
+    "JOIN pg_class c ON c.oid = i.indexrelid "
+    "JOIN pg_class t ON t.oid = i.indrelid "
+    "JOIN pg_namespace n ON n.oid = t.relnamespace "
+    "WHERE NOT i.indisvalid "
+    "  AND c.relkind = 'i' "
+    "  AND n.nspname = %s "
+    "  AND t.relname LIKE %s "
+    "  AND NOT EXISTS (SELECT 1 FROM pg_constraint k WHERE k.conindid = i.indexrelid) "
+    "ORDER BY c.relname"
+)
+
+
+@dataclass(frozen=True, slots=True)
+class _InvalidIndex:
+    schema: str
+    name: str
+    table_size: str
+
 _SPEND_LOGS_ALTER_RE = re.compile(r'^ALTER\s+TABLE\s+"LiteLLM_SpendLogs"\s', re.IGNORECASE)
 _SPEND_LOGS_ARTIFACT_DROP_RE = re.compile(
     r'^DROP\s+TABLE\s+"LiteLLM_SpendLogs_[^"]*"', re.IGNORECASE
@@ -557,7 +585,7 @@ class ProxyExtrasDBManager:
     def _strip_prisma_query_params(url: str) -> str:
         """Remove Prisma-specific query params (connection_limit, pool_timeout,
         schema, etc.) from DATABASE_URL so psycopg can parse it."""
-        from urllib.parse import urlparse, urlunparse, parse_qsl, urlencode
+        from urllib.parse import parse_qsl, quote, urlencode, urlparse, urlunparse
 
         parsed = urlparse(url)
         if not parsed.query:
@@ -578,7 +606,7 @@ class ProxyExtrasDBManager:
             "target_session_attrs",
         }
         kept = [(k, v) for k, v in parse_qsl(parsed.query) if k in libpq_params]
-        return urlunparse(parsed._replace(query=urlencode(kept)))
+        return urlunparse(parsed._replace(query=urlencode(kept, quote_via=quote)))
 
     @staticmethod
     def _warn_if_db_ahead_of_head(migrations_dir: str) -> None:
@@ -652,6 +680,91 @@ class ProxyExtrasDBManager:
             ", ".join(sorted_hostile[:5]) + (" ..." if len(sorted_hostile) > 5 else ""),
         )
 
+    @staticmethod
+    def _invalid_litellm_indexes(
+        conn: "psycopg.Connection[tuple[str, str, str]]", schema: str
+    ) -> tuple[_InvalidIndex, ...]:
+        rows: Final = conn.execute(_INVALID_LITELLM_INDEXES_SQL, (schema, "LiteLLM\\_%")).fetchall()
+        return tuple(_InvalidIndex(*row) for row in rows)
+
+    @staticmethod
+    def _index_repair(index: _InvalidIndex) -> tuple["psycopg.sql.Composed", str]:
+        from psycopg import sql
+
+        target: Final = sql.Identifier(index.schema, index.name)
+        if _TRANSIENT_INDEX_SUFFIX_RE.search(index.name):
+            return sql.SQL("DROP INDEX CONCURRENTLY IF EXISTS {}").format(target), "Dropped leftover"
+        return sql.SQL("REINDEX INDEX CONCURRENTLY {}").format(target), "Rebuilt"
+
+    @staticmethod
+    def _repair_index(conn: "psycopg.Connection[tuple[str, str, str]]", index: _InvalidIndex) -> None:
+        import psycopg
+
+        statement, action = ProxyExtrasDBManager._index_repair(index)
+        try:
+            conn.execute(statement)
+        except psycopg.Error as e:
+            logger.warning(
+                "Could not repair invalid index %s.%s, will retry on the next startup. "
+                "If this keeps happening, run `%s` by hand as the index owner. Error: %s",
+                index.schema,
+                index.name,
+                statement.as_string(conn),
+                e,
+            )
+            return
+        logger.info("%s invalid index %s.%s", action, index.schema, index.name)
+
+    @staticmethod
+    def repair_invalid_indexes(lock_timeout: str = "30s") -> bool:
+        """Rebuild LiteLLM indexes an interrupted CREATE INDEX CONCURRENTLY left
+        INVALID (a migration deadlock between replicas is the usual cause; the
+        retried migration skips them because of IF NOT EXISTS). Never raises:
+        returns True when no invalid index remains, False when the repair was
+        skipped or failed and will be retried on the next startup."""
+        database_url: Final = os.getenv("DATABASE_URL")
+        if not database_url:
+            return False
+
+        try:
+            import psycopg
+            from psycopg import sql
+        except ImportError:
+            logger.warning(
+                "psycopg is not installed; skipping the invalid index check. "
+                "Install the litellm[extra_proxy] extra, which includes psycopg."
+            )
+            return False
+
+        schema: Final = ProxyExtrasDBManager._prisma_schema_param(database_url) or "public"
+        cleaned_url: Final = ProxyExtrasDBManager._strip_prisma_query_params(database_url)
+        try:
+            with psycopg.connect(cleaned_url, connect_timeout=10, autocommit=True) as conn:
+                conn.execute("SET statement_timeout = 0")
+                conn.execute(sql.SQL("SET lock_timeout = {}").format(sql.Literal(lock_timeout)))
+                found: Final = ProxyExtrasDBManager._invalid_litellm_indexes(conn, schema)
+                if not found:
+                    return True
+                logger.warning(
+                    "Found %d invalid index(es) left by an interrupted CREATE INDEX "
+                    "CONCURRENTLY, rebuilding: %s",
+                    len(found),
+                    ", ".join(f"{index.name} (table size {index.table_size})" for index in found),
+                )
+                lock_row: Final = conn.execute(
+                    "SELECT pg_try_advisory_lock(%s)", (INDEX_REPAIR_ADVISORY_LOCK_KEY,)
+                ).fetchone()
+                if lock_row is None or not lock_row[0]:
+                    logger.info("Another replica is already rebuilding the invalid indexes, skipping")
+                    return False
+                for index in ProxyExtrasDBManager._invalid_litellm_indexes(conn, schema):
+                    ProxyExtrasDBManager._repair_index(conn, index)
+                remaining: Final = ProxyExtrasDBManager._invalid_litellm_indexes(conn, schema)
+        except psycopg.Error as e:
+            logger.warning("Could not check for invalid indexes, will retry on the next startup. Error: %s", e)
+            return False
+        return not remaining
+
     @staticmethod
     def _setup_database_v2(use_migrate: bool) -> bool:
         """
@@ -886,6 +999,7 @@ class ProxyExtrasDBManager:
             use_migrate=use_migrate, use_v2_resolver=use_v2_resolver
         )
         if migrated:
+            ProxyExtrasDBManager.repair_invalid_indexes()
             ProxyExtrasDBManager.apply_replica_identity_full_if_requested()
         return migrated
 
diff --git a/tests/proxy_migration_tests/test_invalid_index_repair.py b/tests/proxy_migration_tests/test_invalid_index_repair.py
new file mode 100644
index 00000000000..787a62b361b
--- /dev/null
+++ b/tests/proxy_migration_tests/test_invalid_index_repair.py
@@ -0,0 +1,238 @@
+import os
+import subprocess
+import threading
+import uuid
+from collections.abc import Iterator, Mapping
+from pathlib import Path
+from types import MappingProxyType
+from typing import Final
+
+import pytest
+from litellm_proxy_extras.utils import INDEX_REPAIR_ADVISORY_LOCK_KEY, ProxyExtrasDBManager
+
+psycopg = pytest.importorskip("psycopg")
+
+pytestmark = pytest.mark.timeout(120)
+
+requires_db: Final = pytest.mark.skipif(
+    "DATABASE_URL" not in os.environ,
+    reason="requires a postgres database (DATABASE_URL)",
+)
+
+HEALTH_TABLE: Final = "LiteLLM_HealthCheckTable"
+HEALTH_INDEX: Final = "LiteLLM_HealthCheckTable_model_id_model_name_checked_at_idx"
+HEALTH_INDEX_COLUMNS: Final = '"model_id", "model_name", "checked_at" DESC'
+LOOKALIKE_TABLE: Final = "LiteLLMLookalikeTable"
+LOOKALIKE_INDEX: Final = "LiteLLMLookalikeTable_id_idx"
+PARTITIONED_TABLE: Final = "LiteLLM_PartitionedTable"
+PARTITIONED_INDEX: Final = "LiteLLM_PartitionedTable_id_idx"
+
+
+def _base_url() -> str:
+    return os.environ["DATABASE_URL"].split("?")[0]
+
+
+def _index_validity(schema: str) -> Mapping[str, bool]:
+    with psycopg.connect(_base_url(), autocommit=True) as conn:
+        rows = conn.execute(
+            "SELECT c.relname, i.indisvalid FROM pg_index i "
+            "JOIN pg_class c ON c.oid = i.indexrelid "
+            "JOIN pg_namespace n ON n.oid = c.relnamespace "
+            "WHERE n.nspname = %s",
+            (schema,),
+        ).fetchall()
+    return MappingProxyType(dict(rows))
+
+
+def _interrupt_concurrent_build(schema: str, table: str, statement: str) -> None:
+    """Abort a CONCURRENTLY build while it waits on an older snapshot, the same
+    spot the deadlock loser dies at, so it leaves its index INVALID."""
+    with psycopg.connect(_base_url()) as pin:
+        pin.isolation_level = psycopg.IsolationLevel.REPEATABLE_READ
+        pin.execute(f'SELECT count(*) FROM "{schema}"."{table}"')
+        with psycopg.connect(_base_url(), autocommit=True) as builder:
+            builder.execute("SET statement_timeout = '1s'")
+            with pytest.raises(psycopg.errors.QueryCanceled):
+                builder.execute(statement)
+
+
+def _leave_invalid_index(schema: str, table: str, index: str, columns: str) -> None:
+    _interrupt_concurrent_build(
+        schema, table, f'CREATE INDEX CONCURRENTLY "{index}" ON "{schema}"."{table}" ({columns})'
+    )
+
+
+def _leave_invalid_reindex_leftover(schema: str, table: str, index: str) -> None:
+    _interrupt_concurrent_build(schema, table, f'REINDEX INDEX CONCURRENTLY "{schema}"."{index}"')
+
+
+@pytest.fixture
+def scratch_schema(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]:
+    schema: Final = f"invalid_index_{uuid.uuid4().hex[:8]}"
+    with psycopg.connect(_base_url(), autocommit=True) as conn:
+        conn.execute(f'CREATE SCHEMA "{schema}"')
+        conn.execute(
+            f'CREATE TABLE "{schema}"."{HEALTH_TABLE}" (model_id TEXT, model_name TEXT, checked_at TIMESTAMPTZ)'
+        )
+        conn.execute(f'CREATE TABLE "{schema}"."{LOOKALIKE_TABLE}" (id TEXT)')
+
+    monkeypatch.setenv("DATABASE_URL", f"{_base_url()}?schema={schema}")
+    yield schema
+
+    with psycopg.connect(_base_url(), autocommit=True) as conn:
+        conn.execute(f'DROP SCHEMA "{schema}" CASCADE')
+
+
+@requires_db
+def test_repair_rebuilds_invalid_litellm_indexes_and_leaves_lookalike_tables_alone(scratch_schema: str) -> None:
+    _leave_invalid_index(scratch_schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS)
+    _leave_invalid_index(scratch_schema, LOOKALIKE_TABLE, LOOKALIKE_INDEX, "id")
+    assert _index_validity(scratch_schema) == {HEALTH_INDEX: False, LOOKALIKE_INDEX: False}
+
+    assert ProxyExtrasDBManager.repair_invalid_indexes() is True
+
+    assert _index_validity(scratch_schema) == {HEALTH_INDEX: True, LOOKALIKE_INDEX: False}
+
+
+@requires_db
+def test_repair_drops_leftovers_of_interrupted_rebuilds(scratch_schema: str) -> None:
+    _leave_invalid_index(scratch_schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS)
+    _leave_invalid_reindex_leftover(scratch_schema, HEALTH_TABLE, HEALTH_INDEX)
+    _leave_invalid_index(scratch_schema, HEALTH_TABLE, f"{HEALTH_TABLE}_model_id_idx_ccold", '"model_id"')
+    _leave_invalid_index(scratch_schema, HEALTH_TABLE, f"{HEALTH_TABLE}_model_id_idx_ccnew1", '"model_id"')
+    before: Final = _index_validity(scratch_schema)
+    assert len(before) == 4
+    assert set(before.values()) == {False}
+
+    assert ProxyExtrasDBManager.repair_invalid_indexes() is True
+
+    assert _index_validity(scratch_schema) == {HEALTH_INDEX: True}
+
+
+@requires_db
+def test_repair_is_a_no_op_when_every_index_is_valid(scratch_schema: str) -> None:
+    with psycopg.connect(_base_url(), autocommit=True) as conn:
+        conn.execute(f'CREATE INDEX "{HEALTH_INDEX}" ON "{scratch_schema}"."{HEALTH_TABLE}" ({HEALTH_INDEX_COLUMNS})')
+
+    assert ProxyExtrasDBManager.repair_invalid_indexes() is True
+
+    assert _index_validity(scratch_schema) == {HEALTH_INDEX: True}
+
+
+@requires_db
+def test_repair_leaves_partitioned_parent_indexes_alone(scratch_schema: str) -> None:
+    with psycopg.connect(_base_url(), autocommit=True) as conn:
+        conn.execute(f'CREATE TABLE "{scratch_schema}"."{PARTITIONED_TABLE}" (id INT) PARTITION BY RANGE (id)')
+        conn.execute(
+            f'CREATE TABLE "{scratch_schema}"."{PARTITIONED_TABLE}_p0" '
+            f'PARTITION OF "{scratch_schema}"."{PARTITIONED_TABLE}" FOR VALUES FROM (0) TO (10)'
+        )
+        conn.execute(f'CREATE INDEX "{PARTITIONED_INDEX}" ON ONLY "{scratch_schema}"."{PARTITIONED_TABLE}" (id)')
+    assert _index_validity(scratch_schema) == {PARTITIONED_INDEX: False}
+
+    assert ProxyExtrasDBManager.repair_invalid_indexes() is True
+
+    assert _index_validity(scratch_schema) == {PARTITIONED_INDEX: False}
+
+
+@requires_db
+def test_repair_yields_to_the_replica_holding_the_repair_lock(scratch_schema: str) -> None:
+    _leave_invalid_index(scratch_schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS)
+
+    with psycopg.connect(_base_url(), autocommit=True) as other_replica:
+        other_replica.execute("SELECT pg_advisory_lock(%s)", (INDEX_REPAIR_ADVISORY_LOCK_KEY,))
+        assert ProxyExtrasDBManager.repair_invalid_indexes() is False
+        assert _index_validity(scratch_schema) == {HEALTH_INDEX: False}
+
+    assert ProxyExtrasDBManager.repair_invalid_indexes() is True
+    assert _index_validity(scratch_schema) == {HEALTH_INDEX: True}
+
+
+@requires_db
+def test_repair_gives_up_on_a_blocked_rebuild_and_finishes_it_on_the_next_startup(scratch_schema: str) -> None:
+    _leave_invalid_index(scratch_schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS)
+
+    with psycopg.connect(_base_url()) as pin:
+        pin.isolation_level = psycopg.IsolationLevel.REPEATABLE_READ
+        pin.execute(f'SELECT count(*) FROM "{scratch_schema}"."{HEALTH_TABLE}"')
+        assert ProxyExtrasDBManager.repair_invalid_indexes(lock_timeout="1s") is False
+        blocked: Final = _index_validity(scratch_schema)
+        assert blocked[HEALTH_INDEX] is False
+        assert [name for name in blocked if name.endswith("_ccnew")]
+
+    assert ProxyExtrasDBManager.repair_invalid_indexes() is True
+    assert _index_validity(scratch_schema) == {HEALTH_INDEX: True}
+
+
+def _hold_snapshot(schema: str, table: str, pinned: threading.Event, seconds: float) -> None:
+    with psycopg.connect(_base_url()) as pin:
+        pin.isolation_level = psycopg.IsolationLevel.REPEATABLE_READ
+        pin.execute(f'SELECT count(*) FROM "{schema}"."{table}"')
+        pinned.set()
+        pin.execute("SELECT pg_sleep(%s)", (seconds,))
+
+
+@requires_db
+def test_repair_outlives_a_statement_timeout_passed_through_database_url_options(
+    scratch_schema: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    _leave_invalid_index(scratch_schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS)
+    monkeypatch.setenv("DATABASE_URL", f"{_base_url()}?schema={scratch_schema}&options=-c%20statement_timeout%3D2000")
+    pinned: Final = threading.Event()
+    holder: Final = threading.Thread(target=_hold_snapshot, args=(scratch_schema, HEALTH_TABLE, pinned, 5.0))
+    holder.start()
+    pinned.wait()
+    try:
+        assert ProxyExtrasDBManager.repair_invalid_indexes() is True
+    finally:
+        holder.join()
+
+    assert _index_validity(scratch_schema) == {HEALTH_INDEX: True}
+
+
+@requires_db
+def test_repair_defaults_to_the_public_schema(monkeypatch: pytest.MonkeyPatch) -> None:
+    table: Final = f"LiteLLM_ScratchTable_{uuid.uuid4().hex[:8]}"
+    index: Final = f"{table}_id_idx"
+    monkeypatch.setenv("DATABASE_URL", _base_url())
+    with psycopg.connect(_base_url(), autocommit=True) as conn:
+        conn.execute(f'CREATE TABLE public."{table}" (id TEXT)')
+    try:
+        _leave_invalid_index("public", table, index, "id")
+        assert _index_validity("public")[index] is False
+
+        assert ProxyExtrasDBManager.repair_invalid_indexes() is True
+
+        assert _index_validity("public")[index] is True
+    finally:
+        with psycopg.connect(_base_url(), autocommit=True) as conn:
+            conn.execute(f'DROP TABLE public."{table}"')
+
+
+def test_repair_survives_an_unreachable_database(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@127.0.0.1:9/x?schema=whatever")
+
+    assert ProxyExtrasDBManager.repair_invalid_indexes() is False
+
+
+class _MigrateDeployApplied:
+    stdout = "Applied migration.\n"
+    stderr = ""
+    returncode = 0
+
+
+@requires_db
+@pytest.mark.parametrize("use_v2_resolver", [True, False])
+def test_setup_database_repairs_the_index_after_a_recovered_deploy(
+    scratch_schema: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, use_v2_resolver: bool
+) -> None:
+    _leave_invalid_index(scratch_schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS)
+    (tmp_path / "schema.prisma").write_text("// stub")
+    monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
+    monkeypatch.setattr(ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None)
+    monkeypatch.setattr(ProxyExtrasDBManager, "_resolve_all_migrations", lambda *_, **__: True)
+    monkeypatch.setattr(subprocess, "run", lambda *_, **__: _MigrateDeployApplied())
+
+    assert ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=use_v2_resolver) is True
+
+    assert _index_validity(scratch_schema) == {HEALTH_INDEX: True}

From 2d64020d432f1c6112b836a752720a25a5ab93e9 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Wed, 2 Sep 2026 15:24:42 -0700
Subject: [PATCH 22/85] fix(proxy-extras): run the index repair over DIRECT_URL
 and exercise it through a real migrate deploy

---
 .../litellm_proxy_extras/utils.py             |  7 ++-
 .../test_invalid_index_repair.py              | 63 ++++++++++++-------
 2 files changed, 46 insertions(+), 24 deletions(-)

diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py
index df019d78078..75fda59f28e 100644
--- a/litellm-proxy-extras/litellm_proxy_extras/utils.py
+++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py
@@ -721,8 +721,11 @@ class ProxyExtrasDBManager:
         INVALID (a migration deadlock between replicas is the usual cause; the
         retried migration skips them because of IF NOT EXISTS). Never raises:
         returns True when no invalid index remains, False when the repair was
-        skipped or failed and will be retried on the next startup."""
-        database_url: Final = os.getenv("DATABASE_URL")
+        skipped or failed and will be retried on the next startup. Runs over
+        DIRECT_URL when set: the session settings, the advisory lock and REINDEX
+        CONCURRENTLY all need one server session, which a transaction pooler
+        does not give."""
+        database_url: Final = os.getenv("DIRECT_URL") or os.getenv("DATABASE_URL")
         if not database_url:
             return False
 
diff --git a/tests/proxy_migration_tests/test_invalid_index_repair.py b/tests/proxy_migration_tests/test_invalid_index_repair.py
index 787a62b361b..2e46775cd2b 100644
--- a/tests/proxy_migration_tests/test_invalid_index_repair.py
+++ b/tests/proxy_migration_tests/test_invalid_index_repair.py
@@ -1,9 +1,7 @@
 import os
-import subprocess
 import threading
 import uuid
 from collections.abc import Iterator, Mapping
-from pathlib import Path
 from types import MappingProxyType
 from typing import Final
 
@@ -66,16 +64,14 @@ def _leave_invalid_reindex_leftover(schema: str, table: str, index: str) -> None
     _interrupt_concurrent_build(schema, table, f'REINDEX INDEX CONCURRENTLY "{schema}"."{index}"')
 
 
-@pytest.fixture
-def scratch_schema(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]:
+def _scratch_schema(monkeypatch: pytest.MonkeyPatch, *table_definitions: str) -> Iterator[str]:
     schema: Final = f"invalid_index_{uuid.uuid4().hex[:8]}"
     with psycopg.connect(_base_url(), autocommit=True) as conn:
         conn.execute(f'CREATE SCHEMA "{schema}"')
-        conn.execute(
-            f'CREATE TABLE "{schema}"."{HEALTH_TABLE}" (model_id TEXT, model_name TEXT, checked_at TIMESTAMPTZ)'
-        )
-        conn.execute(f'CREATE TABLE "{schema}"."{LOOKALIKE_TABLE}" (id TEXT)')
+        for definition in table_definitions:
+            conn.execute(f'CREATE TABLE "{schema}".{definition}')
 
+    monkeypatch.delenv("DIRECT_URL", raising=False)
     monkeypatch.setenv("DATABASE_URL", f"{_base_url()}?schema={schema}")
     yield schema
 
@@ -83,6 +79,20 @@ def scratch_schema(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]:
         conn.execute(f'DROP SCHEMA "{schema}" CASCADE')
 
 
+@pytest.fixture
+def scratch_schema(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]:
+    yield from _scratch_schema(
+        monkeypatch,
+        f'"{HEALTH_TABLE}" (model_id TEXT, model_name TEXT, checked_at TIMESTAMPTZ)',
+        f'"{LOOKALIKE_TABLE}" (id TEXT)',
+    )
+
+
+@pytest.fixture
+def empty_schema(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]:
+    yield from _scratch_schema(monkeypatch)
+
+
 @requires_db
 def test_repair_rebuilds_invalid_litellm_indexes_and_leaves_lookalike_tables_alone(scratch_schema: str) -> None:
     _leave_invalid_index(scratch_schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS)
@@ -210,29 +220,38 @@ def test_repair_defaults_to_the_public_schema(monkeypatch: pytest.MonkeyPatch) -
 
 
 def test_repair_survives_an_unreachable_database(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.delenv("DIRECT_URL", raising=False)
     monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@127.0.0.1:9/x?schema=whatever")
 
     assert ProxyExtrasDBManager.repair_invalid_indexes() is False
 
 
-class _MigrateDeployApplied:
-    stdout = "Applied migration.\n"
-    stderr = ""
-    returncode = 0
+@requires_db
+def test_repair_runs_over_direct_url_when_set(scratch_schema: str) -> None:
+    _leave_invalid_index(scratch_schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS)
+    direct_url: Final = os.environ["DATABASE_URL"]
+    with pytest.MonkeyPatch.context() as env:
+        env.setenv("DIRECT_URL", direct_url)
+        env.setenv("DATABASE_URL", "postgresql://u:p@127.0.0.1:9/x?schema=whatever")
+        assert ProxyExtrasDBManager.repair_invalid_indexes() is True
+
+    assert _index_validity(scratch_schema) == {HEALTH_INDEX: True}
+
+
+def _invalidate_deployed_index(schema: str) -> None:
+    with psycopg.connect(_base_url(), autocommit=True) as conn:
+        conn.execute(f'DROP INDEX "{schema}"."{HEALTH_INDEX}"')
+    _leave_invalid_index(schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS)
 
 
 @requires_db
+@pytest.mark.timeout(300)
 @pytest.mark.parametrize("use_v2_resolver", [True, False])
-def test_setup_database_repairs_the_index_after_a_recovered_deploy(
-    scratch_schema: str, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, use_v2_resolver: bool
-) -> None:
-    _leave_invalid_index(scratch_schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS)
-    (tmp_path / "schema.prisma").write_text("// stub")
-    monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
-    monkeypatch.setattr(ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None)
-    monkeypatch.setattr(ProxyExtrasDBManager, "_resolve_all_migrations", lambda *_, **__: True)
-    monkeypatch.setattr(subprocess, "run", lambda *_, **__: _MigrateDeployApplied())
+def test_setup_database_repairs_the_index_after_a_recovered_deploy(empty_schema: str, use_v2_resolver: bool) -> None:
+    assert ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=use_v2_resolver) is True
+    _invalidate_deployed_index(empty_schema)
+    assert _index_validity(empty_schema)[HEALTH_INDEX] is False
 
     assert ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=use_v2_resolver) is True
 
-    assert _index_validity(scratch_schema) == {HEALTH_INDEX: True}
+    assert _index_validity(empty_schema)[HEALTH_INDEX] is True

From bfdaedf51b13f4bc8872d51a99d7f288fc0cc56f Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Wed, 2 Sep 2026 16:23:50 -0700
Subject: [PATCH 23/85] fix(proxy-extras): take the repair schema from
 DATABASE_URL and deploy the real-migration test on a fresh database

---
 .../litellm_proxy_extras/utils.py             | 17 +++----
 .../test_invalid_index_repair.py              | 47 +++++++++++--------
 2 files changed, 36 insertions(+), 28 deletions(-)

diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py
index 75fda59f28e..40bc4cd1dfd 100644
--- a/litellm-proxy-extras/litellm_proxy_extras/utils.py
+++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py
@@ -721,12 +721,13 @@ class ProxyExtrasDBManager:
         INVALID (a migration deadlock between replicas is the usual cause; the
         retried migration skips them because of IF NOT EXISTS). Never raises:
         returns True when no invalid index remains, False when the repair was
-        skipped or failed and will be retried on the next startup. Runs over
-        DIRECT_URL when set: the session settings, the advisory lock and REINDEX
-        CONCURRENTLY all need one server session, which a transaction pooler
-        does not give."""
-        database_url: Final = os.getenv("DIRECT_URL") or os.getenv("DATABASE_URL")
-        if not database_url:
+        skipped or failed and will be retried on the next startup. Looks in the
+        schema DATABASE_URL names, the only URL Prisma migrates through, but
+        connects over DIRECT_URL when set: the session settings, the advisory
+        lock and REINDEX CONCURRENTLY all need one server session, which a
+        transaction pooler does not give."""
+        prisma_url: Final = os.getenv("DATABASE_URL")
+        if not prisma_url:
             return False
 
         try:
@@ -739,8 +740,8 @@ class ProxyExtrasDBManager:
             )
             return False
 
-        schema: Final = ProxyExtrasDBManager._prisma_schema_param(database_url) or "public"
-        cleaned_url: Final = ProxyExtrasDBManager._strip_prisma_query_params(database_url)
+        schema: Final = ProxyExtrasDBManager._prisma_schema_param(prisma_url) or "public"
+        cleaned_url: Final = ProxyExtrasDBManager._strip_prisma_query_params(os.getenv("DIRECT_URL") or prisma_url)
         try:
             with psycopg.connect(cleaned_url, connect_timeout=10, autocommit=True) as conn:
                 conn.execute("SET statement_timeout = 0")
diff --git a/tests/proxy_migration_tests/test_invalid_index_repair.py b/tests/proxy_migration_tests/test_invalid_index_repair.py
index 2e46775cd2b..741fa7386df 100644
--- a/tests/proxy_migration_tests/test_invalid_index_repair.py
+++ b/tests/proxy_migration_tests/test_invalid_index_repair.py
@@ -64,12 +64,15 @@ def _leave_invalid_reindex_leftover(schema: str, table: str, index: str) -> None
     _interrupt_concurrent_build(schema, table, f'REINDEX INDEX CONCURRENTLY "{schema}"."{index}"')
 
 
-def _scratch_schema(monkeypatch: pytest.MonkeyPatch, *table_definitions: str) -> Iterator[str]:
+@pytest.fixture
+def scratch_schema(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]:
     schema: Final = f"invalid_index_{uuid.uuid4().hex[:8]}"
     with psycopg.connect(_base_url(), autocommit=True) as conn:
         conn.execute(f'CREATE SCHEMA "{schema}"')
-        for definition in table_definitions:
-            conn.execute(f'CREATE TABLE "{schema}".{definition}')
+        conn.execute(
+            f'CREATE TABLE "{schema}"."{HEALTH_TABLE}" (model_id TEXT, model_name TEXT, checked_at TIMESTAMPTZ)'
+        )
+        conn.execute(f'CREATE TABLE "{schema}"."{LOOKALIKE_TABLE}" (id TEXT)')
 
     monkeypatch.delenv("DIRECT_URL", raising=False)
     monkeypatch.setenv("DATABASE_URL", f"{_base_url()}?schema={schema}")
@@ -80,17 +83,22 @@ def _scratch_schema(monkeypatch: pytest.MonkeyPatch, *table_definitions: str) ->
 
 
 @pytest.fixture
-def scratch_schema(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]:
-    yield from _scratch_schema(
-        monkeypatch,
-        f'"{HEALTH_TABLE}" (model_id TEXT, model_name TEXT, checked_at TIMESTAMPTZ)',
-        f'"{LOOKALIKE_TABLE}" (id TEXT)',
-    )
+def fresh_database(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]:
+    """A brand-new database, what a first deploy sees. A scratch schema would
+    not do: the migrations guard on pg_constraint by name across every schema,
+    so a LiteLLM schema already pushed into public makes them skip and then
+    fail, which is exactly what CI's database looks like."""
+    admin_url: Final = _base_url()
+    name: Final = f"invalid_index_{uuid.uuid4().hex[:8]}"
+    with psycopg.connect(admin_url, autocommit=True) as conn:
+        conn.execute(f'CREATE DATABASE "{name}"')
 
+    monkeypatch.delenv("DIRECT_URL", raising=False)
+    monkeypatch.setenv("DATABASE_URL", f"{admin_url.rsplit('/', 1)[0]}/{name}")
+    yield "public"
 
-@pytest.fixture
-def empty_schema(monkeypatch: pytest.MonkeyPatch) -> Iterator[str]:
-    yield from _scratch_schema(monkeypatch)
+    with psycopg.connect(admin_url, autocommit=True) as conn:
+        conn.execute(f'DROP DATABASE "{name}" WITH (FORCE)')
 
 
 @requires_db
@@ -227,12 +235,11 @@ def test_repair_survives_an_unreachable_database(monkeypatch: pytest.MonkeyPatch
 
 
 @requires_db
-def test_repair_runs_over_direct_url_when_set(scratch_schema: str) -> None:
+def test_repair_connects_over_direct_url_but_looks_in_the_schema_database_url_names(scratch_schema: str) -> None:
     _leave_invalid_index(scratch_schema, HEALTH_TABLE, HEALTH_INDEX, HEALTH_INDEX_COLUMNS)
-    direct_url: Final = os.environ["DATABASE_URL"]
     with pytest.MonkeyPatch.context() as env:
-        env.setenv("DIRECT_URL", direct_url)
-        env.setenv("DATABASE_URL", "postgresql://u:p@127.0.0.1:9/x?schema=whatever")
+        env.setenv("DIRECT_URL", f"{_base_url()}?schema=public")
+        env.setenv("DATABASE_URL", f"postgresql://u:p@127.0.0.1:9/x?schema={scratch_schema}")
         assert ProxyExtrasDBManager.repair_invalid_indexes() is True
 
     assert _index_validity(scratch_schema) == {HEALTH_INDEX: True}
@@ -247,11 +254,11 @@ def _invalidate_deployed_index(schema: str) -> None:
 @requires_db
 @pytest.mark.timeout(300)
 @pytest.mark.parametrize("use_v2_resolver", [True, False])
-def test_setup_database_repairs_the_index_after_a_recovered_deploy(empty_schema: str, use_v2_resolver: bool) -> None:
+def test_setup_database_repairs_the_index_after_a_recovered_deploy(fresh_database: str, use_v2_resolver: bool) -> None:
     assert ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=use_v2_resolver) is True
-    _invalidate_deployed_index(empty_schema)
-    assert _index_validity(empty_schema)[HEALTH_INDEX] is False
+    _invalidate_deployed_index(fresh_database)
+    assert _index_validity(fresh_database)[HEALTH_INDEX] is False
 
     assert ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=use_v2_resolver) is True
 
-    assert _index_validity(empty_schema)[HEALTH_INDEX] is True
+    assert _index_validity(fresh_database)[HEALTH_INDEX] is True

From cedf35992bdec398e7623d28e45b5218e4eff9bf Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Thu, 3 Sep 2026 04:02:55 -0700
Subject: [PATCH 24/85] fix(proxy): give each spend-log queue monitor its own
 flush event

`PrismaClient.spend_log_flush_requested` was an `asyncio.Event` built at
import time, so it bound to whichever event loop first awaited it and every
later loop got `RuntimeError: ... is bound to a different event loop` out of
`_wait_for_spend_log_flush_request`. The queue monitor's blanket `except
Exception` swallowed that into its error logger, so the flush silently never
happened and the row sat in the worker's queue until the next poll.

The monitor now creates its own Event inside the loop that awaits it and
hands it to the client, and `request_spend_log_flush` signals through the
client instead of the class. A request that arrives before the monitor is
running is dropped and loses nothing, because the monitor reads the queue on
its first pass before it ever waits.

In CI this showed up as the proxy-endpoints shard flaking on
test_monitor_spend_logs_queue_flushes_as_soon_as_one_is_requested whenever
--dist=loadscope put the health-endpoint tests, which boot a proxy TestClient
and start a monitor, on the same worker ahead of the spend-log tests.
---
 litellm/proxy/db/db_spend_update_writer.py    |  2 +-
 litellm/proxy/utils.py                        | 22 ++++---
 .../proxy/db/test_db_spend_update_writer.py   |  7 +-
 .../prisma_and_spend/test_spend_functions.py  | 66 +++++++++++++++++--
 4 files changed, 79 insertions(+), 18 deletions(-)

diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py
index e6880d521f1..18058f31385 100644
--- a/litellm/proxy/db/db_spend_update_writer.py
+++ b/litellm/proxy/db/db_spend_update_writer.py
@@ -940,7 +940,7 @@ class DBSpendUpdateWriter:
 
             await enqueue_spend_logs(prisma_client, (payload,))
             if payload.get("call_type") in RESPONSES_SESSION_CALL_TYPES:
-                request_spend_log_flush()
+                request_spend_log_flush(prisma_client)
         else:
             verbose_proxy_logger.debug("prisma_client is None. Skipping writing spend logs to db.")
 
diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index cab2bd6d9db..aa3076dbd19 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -3519,7 +3519,7 @@ class _StaleReadEngine:
 class PrismaClient:
     spend_log_transactions: list = []
     _spend_log_transactions_lock = asyncio.Lock()
-    spend_log_flush_requested: ClassVar[asyncio.Event] = asyncio.Event()
+    spend_log_flush_requested: "asyncio.Event | None" = None
     spend_log_queue_bytes: ClassVar[int] = 0
     spend_logs_queue_monitor_task: "asyncio.Task[None] | None" = None
     tool_usage_transactions: list["ToolUsageTransaction"] = []
@@ -6245,23 +6245,27 @@ async def enqueue_spend_logs(
         )
 
 
-def request_spend_log_flush() -> None:
-    """Wake the queue monitor now rather than leaving the rows for its next poll.
+def request_spend_log_flush(prisma_client: PrismaClient) -> None:
+    """Wake this client's queue monitor now rather than leaving the rows for its next poll.
 
     The Responses API hands the client an id it can chain from straight away, and that
     lookup reads the DB, so the row cannot sit in this worker's queue for a poll interval.
     Repeated requests coalesce into the monitor's next pass, so the batching holds.
+    A request made before the monitor is running is dropped, and loses nothing: the
+    monitor reads the queue on its first pass, before it ever waits on a request.
     """
-    PrismaClient.spend_log_flush_requested.set()
+    flush_requested: Final = prisma_client.spend_log_flush_requested
+    if flush_requested is not None:
+        flush_requested.set()
 
 
-async def _wait_for_spend_log_flush_request(interval: float) -> bool:
+async def _wait_for_spend_log_flush_request(flush_requested: asyncio.Event, interval: float) -> bool:
     """Wait out ``interval``, returning early and True when a flush was requested."""
     try:
-        await asyncio.wait_for(PrismaClient.spend_log_flush_requested.wait(), timeout=interval)
+        await asyncio.wait_for(flush_requested.wait(), timeout=interval)
     except asyncio.TimeoutError:
         return False
-    PrismaClient.spend_log_flush_requested.clear()
+    flush_requested.clear()
     return True
 
 
@@ -6681,6 +6685,8 @@ async def _monitor_spend_logs_queue(
     max_backoff: Final = 30.0  # Maximum backoff interval in seconds
     backoff_multiplier: Final = 1.5  # Exponential backoff multiplier
     current_interval = base_interval
+    flush_requested: Final = asyncio.Event()
+    prisma_client.spend_log_flush_requested = flush_requested  # rebind-ok: the client owns its monitor's flush signal
 
     verbose_proxy_logger.info(
         "Starting spend logs queue monitor (threshold: %s, poll_interval: %ss)", threshold, base_interval
@@ -6719,7 +6725,7 @@ async def _monitor_spend_logs_queue(
                 # Exponential backoff when no logs to process
                 current_interval = min(current_interval * backoff_multiplier, max_backoff)
 
-            if await _wait_for_spend_log_flush_request(current_interval):
+            if await _wait_for_spend_log_flush_request(flush_requested, current_interval):
                 current_interval = base_interval
         except Exception as e:
             spend_log_error("Error in spend logs queue monitor: %s", str(e), exc=e)
diff --git a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py
index 11ef911de3e..104dcf55e66 100644
--- a/tests/test_litellm/proxy/db/test_db_spend_update_writer.py
+++ b/tests/test_litellm/proxy/db/test_db_spend_update_writer.py
@@ -2941,11 +2941,9 @@ async def test_insert_spend_log_asks_for_an_immediate_flush_on_responses_calls(c
     A `previous_response_id` chained straight off the previous turn reads the DB, so a
     Responses row cannot sit in this worker's queue until the monitor's next poll.
     """
-    from litellm.proxy.utils import PrismaClient
-
     db_writer = DBSpendUpdateWriter()
     prisma = _tool_usage_prisma()
-    PrismaClient.spend_log_flush_requested.clear()
+    prisma.spend_log_flush_requested = asyncio.Event()
 
     await db_writer._insert_spend_log_to_db(
         payload={"request_id": "req-1", "call_type": call_type},
@@ -2953,8 +2951,7 @@ async def test_insert_spend_log_asks_for_an_immediate_flush_on_responses_calls(c
     )
 
     assert prisma.spend_log_transactions == [{"request_id": "req-1", "call_type": call_type}]
-    assert PrismaClient.spend_log_flush_requested.is_set() is expects_flush
-    PrismaClient.spend_log_flush_requested.clear()
+    assert prisma.spend_log_flush_requested.is_set() is expects_flush
 
 
 @pytest.mark.asyncio
diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/test_spend_functions.py b/tests/test_litellm/proxy/utils/prisma_and_spend/test_spend_functions.py
index a1eb88a7834..ed854d2c95c 100644
--- a/tests/test_litellm/proxy/utils/prisma_and_spend/test_spend_functions.py
+++ b/tests/test_litellm/proxy/utils/prisma_and_spend/test_spend_functions.py
@@ -538,10 +538,9 @@ async def test_monitor_spend_logs_queue_flushes_as_soon_as_one_is_requested(
     """
     import litellm.constants as constants_mod
     import litellm.proxy.utils as utils_mod
-    from litellm.proxy.utils import PrismaClient, request_spend_log_flush
+    from litellm.proxy.utils import request_spend_log_flush
 
     monkeypatch.setattr(constants_mod, "SPEND_LOG_QUEUE_POLL_INTERVAL", 30.0, raising=False)
-    PrismaClient.spend_log_flush_requested.clear()
     mock_prisma_client.spend_log_transactions = []
     mock_prisma_client.tool_usage_transactions = []
 
@@ -562,16 +561,75 @@ async def test_monitor_spend_logs_queue_flushes_as_soon_as_one_is_requested(
     try:
         await asyncio.sleep(0.05)
         assert not flushed.is_set()
+        assert isinstance(mock_prisma_client.spend_log_flush_requested, asyncio.Event)
 
         mock_prisma_client.spend_log_transactions.append(make_spend_log_row(request_id="r1"))
-        request_spend_log_flush()
+        request_spend_log_flush(mock_prisma_client)
 
         await asyncio.wait_for(flushed.wait(), timeout=5.0)
     finally:
         monitor.cancel()
         with suppress(asyncio.CancelledError):
             await monitor
-        PrismaClient.spend_log_flush_requested.clear()
+
+
+def test_monitor_spend_logs_queue_flush_survives_an_earlier_event_loop(
+    mock_prisma_client: Any,
+    make_spend_log_row: Any,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """A second monitor, started in a fresh event loop, is still woken by a flush request,
+    so a worker whose first loop is gone keeps flushing Responses rows instead of stalling.
+    """
+    import litellm.constants as constants_mod
+    import litellm.proxy.utils as utils_mod
+    from litellm.proxy.utils import request_spend_log_flush
+
+    monkeypatch.setattr(constants_mod, "SPEND_LOG_QUEUE_POLL_INTERVAL", 30.0, raising=False)
+    mock_prisma_client.tool_usage_transactions = []
+
+    async def _flush_once_under_a_monitor() -> None:
+        flushed: Final = asyncio.Event()
+
+        async def _fake_job(*args: Any, **kwargs: Any) -> None:
+            flushed.set()
+
+        monkeypatch.setattr(utils_mod, "update_spend_logs_job", _fake_job)
+        mock_prisma_client.spend_log_transactions = []
+
+        monitor: Final = asyncio.create_task(
+            _monitor_spend_logs_queue(
+                prisma_client=mock_prisma_client,
+                db_writer_client=None,
+                proxy_logging_obj=MagicMock(),
+            )
+        )
+        try:
+            await asyncio.sleep(0.05)
+            assert not flushed.is_set()
+
+            mock_prisma_client.spend_log_transactions.append(make_spend_log_row(request_id="r1"))
+            request_spend_log_flush(mock_prisma_client)
+
+            await asyncio.wait_for(flushed.wait(), timeout=5.0)
+        finally:
+            monitor.cancel()
+            with suppress(asyncio.CancelledError):
+                await monitor
+
+    asyncio.run(_flush_once_under_a_monitor())
+    asyncio.run(_flush_once_under_a_monitor())
+
+
+def test_request_spend_log_flush_is_a_no_op_before_the_monitor_starts(mock_prisma_client: Any) -> None:
+    """A Responses row enqueued before the monitor's first pass must not fail the request."""
+    from litellm.proxy.utils import request_spend_log_flush
+
+    mock_prisma_client.spend_log_flush_requested = None
+
+    request_spend_log_flush(mock_prisma_client)
+
+    assert mock_prisma_client.spend_log_flush_requested is None
 
 
 def test_raise_failed_update_spend_exception_emits_failure_handler() -> None:

From 29bcb0eeb9b085492764cdd970bc4b28340f554b Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Thu, 3 Sep 2026 04:55:57 -0700
Subject: [PATCH 25/85] test(proxy): pin that a flush requested before the
 monitor starts costs the row nothing

The monitor reads the queue on its first pass, before it ever waits on a
request, so dropping a request made while spend_log_flush_requested is still
None delays nothing. Reordering the loop to wait first would turn that drop
into a real delay for the Responses chaining flow, and now fails this test.
---
 .../prisma_and_spend/test_spend_functions.py  | 38 +++++++++++++++++--
 1 file changed, 35 insertions(+), 3 deletions(-)

diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/test_spend_functions.py b/tests/test_litellm/proxy/utils/prisma_and_spend/test_spend_functions.py
index ed854d2c95c..c8b87bd671e 100644
--- a/tests/test_litellm/proxy/utils/prisma_and_spend/test_spend_functions.py
+++ b/tests/test_litellm/proxy/utils/prisma_and_spend/test_spend_functions.py
@@ -621,16 +621,48 @@ def test_monitor_spend_logs_queue_flush_survives_an_earlier_event_loop(
     asyncio.run(_flush_once_under_a_monitor())
 
 
-def test_request_spend_log_flush_is_a_no_op_before_the_monitor_starts(mock_prisma_client: Any) -> None:
-    """A Responses row enqueued before the monitor's first pass must not fail the request."""
+@pytest.mark.asyncio
+async def test_flush_requested_before_the_monitor_starts_costs_the_row_nothing(
+    mock_prisma_client: Any,
+    make_spend_log_row: Any,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """A Responses row enqueued before the monitor exists still reaches the DB on its first
+    pass, so dropping that early request delays nothing.
+    """
+    import litellm.constants as constants_mod
+    import litellm.proxy.utils as utils_mod
     from litellm.proxy.utils import request_spend_log_flush
 
+    monkeypatch.setattr(constants_mod, "SPEND_LOG_QUEUE_POLL_INTERVAL", 30.0, raising=False)
     mock_prisma_client.spend_log_flush_requested = None
+    mock_prisma_client.tool_usage_transactions = []
+    mock_prisma_client.spend_log_transactions = [make_spend_log_row(request_id="r1")]
+
+    flushed: Final = asyncio.Event()
+
+    async def _fake_job(*args: Any, **kwargs: Any) -> None:
+        flushed.set()
+
+    monkeypatch.setattr(utils_mod, "update_spend_logs_job", _fake_job)
 
     request_spend_log_flush(mock_prisma_client)
-
     assert mock_prisma_client.spend_log_flush_requested is None
 
+    monitor: Final = asyncio.create_task(
+        _monitor_spend_logs_queue(
+            prisma_client=mock_prisma_client,
+            db_writer_client=None,
+            proxy_logging_obj=MagicMock(),
+        )
+    )
+    try:
+        await asyncio.wait_for(flushed.wait(), timeout=5.0)
+    finally:
+        monitor.cancel()
+        with suppress(asyncio.CancelledError):
+            await monitor
+
 
 def test_raise_failed_update_spend_exception_emits_failure_handler() -> None:
     proxy_logging = MagicMock()

From 2a2c49ad4cd711aded30da0384c1a200ddfca5d7 Mon Sep 17 00:00:00 2001
From: mubashir1osmani <mubashir.osmani777@gmail.com>
Date: Thu, 3 Sep 2026 16:09:44 -0400
Subject: [PATCH 26/85] feat(ui): surface batch results on the logs page

---
 .../LogDetailContent.test.tsx                 | 78 ++++++++++++++++
 .../LogDetailsDrawer/LogDetailContent.tsx     | 61 +++++++++++++
 .../RequestLogsTableColumns.test.tsx          | 53 +++++++++++
 .../view_logs/RequestLogsTableColumns.tsx     | 29 +++++-
 .../components/view_logs/TypeBadges.test.tsx  | 14 ++-
 .../src/components/view_logs/TypeBadges.tsx   | 26 ++++++
 .../view_logs/batchLogUtils.test.ts           | 90 +++++++++++++++++++
 .../src/components/view_logs/batchLogUtils.ts | 67 ++++++++++++++
 .../src/components/view_logs/constants.ts     |  3 +
 9 files changed, 418 insertions(+), 3 deletions(-)
 create mode 100644 ui/litellm-dashboard/src/components/view_logs/batchLogUtils.test.ts
 create mode 100644 ui/litellm-dashboard/src/components/view_logs/batchLogUtils.ts

diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx
index a679dc49427..91778d2a98a 100644
--- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx
@@ -113,6 +113,84 @@ describe("LogDetailContent", () => {
     expect(screen.getAllByText("$0.00200000").length).toBeGreaterThanOrEqual(1);
   });
 
+  it("shows reasoning tokens in Metrics when the usage breakout carries them", () => {
+    render(
+      <LogDetailContent
+        logEntry={createLogEntry({
+          metadata: {
+            status: "success",
+            usage_object: { completion_tokens_details: { text_tokens: 32, reasoning_tokens: 224 } },
+          },
+        })}
+      />,
+    );
+
+    expect(screen.getByText("Reasoning Tokens")).toBeInTheDocument();
+    expect(screen.getByText("224")).toBeInTheDocument();
+  });
+
+  it("hides the reasoning metric when the breakout is absent or zero", () => {
+    render(
+      <LogDetailContent
+        logEntry={createLogEntry({
+          metadata: {
+            status: "success",
+            usage_object: { completion_tokens_details: { text_tokens: 32, reasoning_tokens: 0 } },
+          },
+        })}
+      />,
+    );
+
+    expect(screen.queryByText("Reasoning Tokens")).not.toBeInTheDocument();
+  });
+
+  describe("Batch Results section", () => {
+    const batchCostEntry = (metadata: Record<string, unknown>) =>
+      createLogEntry({
+        request_id: "batch_abc123_batch_cost",
+        call_type: "aretrieve_batch",
+        metadata: { status: "success", ...metadata },
+      });
+
+    it("renders batch id, per-request outcome counts, and batch models for a batch cost row", () => {
+      render(
+        <LogDetailContent
+          logEntry={batchCostEntry({
+            batch_models: ["gemini-2.5-flash"],
+            batch_successful_requests: 2,
+            batch_failed_requests: 1,
+          })}
+        />,
+      );
+
+      const section = screen.getByText("Batch Results").closest('[data-slot="card"]') as HTMLElement;
+      expect(within(section).getByText("batch_abc123")).toBeInTheDocument();
+      expect(within(section).getByText("2")).toBeInTheDocument();
+      expect(within(section).getByText("1")).toBeInTheDocument();
+      expect(within(section).getByText("gemini-2.5-flash")).toBeInTheDocument();
+    });
+
+    it("still renders the batch id when a legacy row carries no counts", () => {
+      render(<LogDetailContent logEntry={batchCostEntry({})} />);
+
+      expect(screen.getByText("Batch Results")).toBeInTheDocument();
+      expect(screen.getByText("batch_abc123")).toBeInTheDocument();
+      expect(screen.queryByText("Successful Requests")).not.toBeInTheDocument();
+    });
+
+    it("never renders for a non-batch call type", () => {
+      render(
+        <LogDetailContent
+          logEntry={createLogEntry({
+            metadata: { status: "success", batch_successful_requests: 2, batch_failed_requests: 1 },
+          })}
+        />,
+      );
+
+      expect(screen.queryByText("Batch Results")).not.toBeInTheDocument();
+    });
+  });
+
   it("should show Input Tokens and Output Tokens for anthropic_messages when uncached text_tokens exist", () => {
     render(
       <LogDetailContent
diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx
index 4c5c7b7b43f..8e0f041b88d 100644
--- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx
@@ -13,6 +13,13 @@ import { formatNumberWithCommas } from "@/utils/dataUtils";
 import { PROMPT_CACHE_CREATION_TOOLTIP, PROMPT_CACHE_READ_TOOLTIP } from "@/utils/promptCacheUsage";
 import GuardrailViewer from "../GuardrailViewer/GuardrailViewer";
 import EvalViewer from "../EvalViewer/EvalViewer";
+import {
+  getBatchIdFromRequestId,
+  getBatchModels,
+  getBatchRequestCounts,
+  getReasoningTokens,
+  isBatchCallType,
+} from "../batchLogUtils";
 import { CostBreakdownViewer } from "../CostBreakdownViewer";
 import { ConfigInfoMessage } from "../ConfigInfoMessage";
 import { VectorStoreViewer } from "../VectorStoreViewer";
@@ -150,6 +157,9 @@ export function LogDetailContent({ logEntry, isLoadingDetails = false, accessTok
         </Card>
       </div>
 
+      {/* Batch Results */}
+      {isBatchCallType(logEntry.call_type) && <BatchResultsSection logEntry={logEntry} metadata={metadata} />}
+
       {/* Routing */}
       <RoutingDecisionCard decision={metadata?.routing_decision as RoutingDecision | undefined} />
 
@@ -374,6 +384,53 @@ function MetricLabel({ label, tooltip, docsUrl }: { label: string; tooltip: stri
   );
 }
 
+/**
+ * Aggregate per-request outcomes for a batch cost row: batch id, success/failure counts
+ * from the parsed output and error files, and the models the batch actually ran on.
+ */
+function BatchResultsSection({ logEntry, metadata }: { logEntry: LogEntry; metadata: Record<string, any> }) {
+  const counts = getBatchRequestCounts(metadata);
+  const batchId = getBatchIdFromRequestId(logEntry.request_id);
+  const batchModels = getBatchModels(metadata);
+  if (!counts && !batchId && !batchModels) return null;
+
+  return (
+    <div className="bg-card rounded-lg shadow-sm w-full max-w-full overflow-hidden mb-6">
+      <Card size="sm" style={{ marginBottom: 0 }}>
+        <CardHeader>
+          <CardTitle>Batch Results</CardTitle>
+        </CardHeader>
+        <CardContent>
+          <DescriptionList>
+            {batchId && (
+              <DescriptionItem label="Batch ID">
+                <TruncatedValue value={batchId} />
+              </DescriptionItem>
+            )}
+            {counts && (
+              <>
+                <DescriptionItem label="Successful Requests">
+                  {formatNumberWithCommas(counts.successful)}
+                </DescriptionItem>
+                <DescriptionItem label="Failed Requests">
+                  {counts.failed > 0 ? (
+                    <Badge variant="secondary" className="bg-destructive/15 text-destructive">
+                      {formatNumberWithCommas(counts.failed)}
+                    </Badge>
+                  ) : (
+                    formatNumberWithCommas(counts.failed)
+                  )}
+                </DescriptionItem>
+              </>
+            )}
+            {batchModels && <DescriptionItem label="Models">{batchModels.join(", ")}</DescriptionItem>}
+          </DescriptionList>
+        </CardContent>
+      </Card>
+    </div>
+  );
+}
+
 function MetricsSection({ logEntry, metadata }: { logEntry: LogEntry; metadata: Record<string, any> }) {
   const completionStartTime = logEntry.completionStartTime;
   const ttftMs =
@@ -391,6 +448,7 @@ function MetricsSection({ logEntry, metadata }: { logEntry: LogEntry; metadata:
   const uncachedInputTokens = getUncachedInputTextTokens(metadata);
   const showAnthropicMessagesInputOutput =
     logEntry.call_type === "anthropic_messages" && uncachedInputTokens !== undefined;
+  const reasoningTokens = getReasoningTokens(metadata);
 
   return (
     <div className="bg-card rounded-lg shadow-sm w-full max-w-full overflow-hidden mb-6">
@@ -416,6 +474,9 @@ function MetricsSection({ logEntry, metadata }: { logEntry: LogEntry; metadata:
                 />
               </DescriptionItem>
             )}
+            {reasoningTokens !== undefined && reasoningTokens > 0 && (
+              <DescriptionItem label="Reasoning Tokens">{formatNumberWithCommas(reasoningTokens)}</DescriptionItem>
+            )}
             <DescriptionItem label="Cost">${formatNumberWithCommas(logEntry.spend || 0, 8)}</DescriptionItem>
             <DescriptionItem label="Duration">
               {logEntry.request_duration_ms != null ? (logEntry.request_duration_ms / 1000).toFixed(3) : "-"} s
diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx
index 3c9e6543c1c..3e8e522afe1 100644
--- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx
@@ -134,6 +134,59 @@ describe("Type column", () => {
 
     expect(screen.getByText("MCP")).toBeInTheDocument();
   });
+
+  it("marks a batch cost row with the Batch badge instead of LLM", () => {
+    renderRows([logEntry({ request_id: "batch_1_batch_cost", call_type: "aretrieve_batch" })]);
+
+    expect(screen.getByText("Batch")).toBeInTheDocument();
+    expect(screen.queryByText("LLM")).not.toBeInTheDocument();
+  });
+});
+
+describe("batch rows", () => {
+  const batchRow = (overrides: Partial<LogEntry>): LogEntry =>
+    logEntry({
+      request_id: "batch_abc123_batch_cost",
+      call_type: "aretrieve_batch",
+      ...overrides,
+    });
+
+  it("rolls partial failures into the status badge instead of reporting blanket Success", async () => {
+    const user = userEvent.setup();
+    renderRows([batchRow({ metadata: { batch_successful_requests: 2, batch_failed_requests: 1 } })]);
+
+    expect(screen.queryByText("Success")).not.toBeInTheDocument();
+    await user.hover(screen.getByText("2/3 succeeded"));
+    expect(await screen.findByText("1 of 3 batch requests failed")).toBeInTheDocument();
+  });
+
+  it("keeps the Success badge when every batch request succeeded", () => {
+    renderRows([batchRow({ metadata: { batch_successful_requests: 3, batch_failed_requests: 0 } })]);
+
+    expect(screen.getByText("Success")).toBeInTheDocument();
+  });
+
+  it("keeps the Failure badge when the batch row itself failed, whatever the counts say", () => {
+    renderRows([batchRow({ metadata: { status: "failure", batch_successful_requests: 2, batch_failed_requests: 1 } })]);
+
+    expect(screen.getByText("Failure")).toBeInTheDocument();
+    expect(screen.queryByText("2/3 succeeded")).not.toBeInTheDocument();
+  });
+
+  it("shows the provider batch id, not the synthetic _batch_cost request id", () => {
+    renderRows([batchRow({ metadata: { batch_successful_requests: 1, batch_failed_requests: 0 } })]);
+
+    expect(screen.getByText("batch_abc123")).toBeInTheDocument();
+    expect(screen.queryByText("batch_abc123_batch_cost")).not.toBeInTheDocument();
+    expect(screen.getByText("batch cost")).toBeInTheDocument();
+  });
+
+  it("leaves ordinary request ids untouched", () => {
+    renderRows([logEntry({ request_id: "chatcmpl-42" })]);
+
+    expect(screen.getByText("chatcmpl-42")).toBeInTheDocument();
+    expect(screen.queryByText("batch cost")).not.toBeInTheDocument();
+  });
 });
 
 describe("Model column", () => {
diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx
index 9d4dc4f7898..b4296abe266 100644
--- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx
@@ -7,9 +7,10 @@ import { CellTooltip, DateCell, IdCell, MoneyCell, StatusBadge } from "@/compone
 import { getSpendString } from "@/utils/dataUtils";
 
 import { getProviderLogoAndName } from "../provider_info_helpers";
+import { getBatchIdFromRequestId, getBatchRequestCounts, isBatchCallType } from "./batchLogUtils";
 import type { LogEntry } from "./columns";
 import { AGENT_CALL_TYPES, MCP_CALL_TYPES } from "./constants";
-import { AgentBadge, AgentIcon, LlmBadge, McpBadge, SparkleIcon, WrenchIcon } from "./TypeBadges";
+import { AgentBadge, AgentIcon, BatchBadge, LlmBadge, McpBadge, SparkleIcon, WrenchIcon } from "./TypeBadges";
 
 export interface RequestLogsTableColumnsDeps {
   onKeyHashClick: (keyHash: string) => void;
@@ -66,6 +67,7 @@ export const getRequestLogsTableColumns = ({
       if (sessionCount <= 1) {
         if (isMcp) return <McpBadge />;
         if (isAgent) return <AgentBadge />;
+        if (isBatchCallType(log.call_type)) return <BatchBadge />;
         return <LlmBadge />;
       }
 
@@ -106,6 +108,17 @@ export const getRequestLogsTableColumns = ({
     cell: ({ row }) => {
       const status = readMetaString(row.original.metadata, "status") ?? "Success";
       const isSuccess = status.toLowerCase() !== "failure";
+      const batchCounts = isSuccess ? getBatchRequestCounts(row.original.metadata) : undefined;
+      if (batchCounts && batchCounts.failed > 0) {
+        const total = batchCounts.successful + batchCounts.failed;
+        return (
+          <StatusBadge
+            tone="warning"
+            label={`${batchCounts.successful}/${total} succeeded`}
+            tooltip={`${batchCounts.failed} of ${total} batch requests failed`}
+          />
+        );
+      }
       return <StatusBadge tone={isSuccess ? "success" : "error"} label={isSuccess ? "Success" : "Failure"} />;
     },
   },
@@ -122,7 +135,19 @@ export const getRequestLogsTableColumns = ({
     accessorKey: "request_id",
     header: "Request ID",
     enableSorting: false,
-    cell: ({ row }) => <IdCell value={row.original.request_id} variant="plain" />,
+    cell: ({ row }) => {
+      const log = row.original;
+      const batchId = isBatchCallType(log.call_type) ? getBatchIdFromRequestId(log.request_id) : undefined;
+      if (batchId) {
+        return (
+          <div className="flex flex-col">
+            <IdCell value={batchId} variant="plain" copyable tooltip={`Batch ${batchId} (row: ${log.request_id})`} />
+            <span className="text-[10px] text-muted-foreground">batch cost</span>
+          </div>
+        );
+      }
+      return <IdCell value={log.request_id} variant="plain" />;
+    },
   },
   {
     id: "spend",
diff --git a/ui/litellm-dashboard/src/components/view_logs/TypeBadges.test.tsx b/ui/litellm-dashboard/src/components/view_logs/TypeBadges.test.tsx
index e3467310265..8a964ea2124 100644
--- a/ui/litellm-dashboard/src/components/view_logs/TypeBadges.test.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/TypeBadges.test.tsx
@@ -1,6 +1,6 @@
 import { render, screen } from "@testing-library/react";
 import { describe, expect, it } from "vitest";
-import { LlmBadge, McpBadge, AgentBadge } from "./TypeBadges";
+import { LlmBadge, McpBadge, AgentBadge, BatchBadge } from "./TypeBadges";
 
 describe("TypeBadges", () => {
   describe("LlmBadge", () => {
@@ -43,4 +43,16 @@ describe("TypeBadges", () => {
       expect(screen.getByText("12")).toBeInTheDocument();
     });
   });
+
+  describe("BatchBadge", () => {
+    it("should render with default 'Batch' text when no count is provided", () => {
+      render(<BatchBadge />);
+      expect(screen.getByText("Batch")).toBeInTheDocument();
+    });
+
+    it("should render the count when provided", () => {
+      render(<BatchBadge count={4} />);
+      expect(screen.getByText("4")).toBeInTheDocument();
+    });
+  });
 });
diff --git a/ui/litellm-dashboard/src/components/view_logs/TypeBadges.tsx b/ui/litellm-dashboard/src/components/view_logs/TypeBadges.tsx
index 1ba4365f66e..84079848487 100644
--- a/ui/litellm-dashboard/src/components/view_logs/TypeBadges.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/TypeBadges.tsx
@@ -57,6 +57,25 @@ export const AgentIcon = ({ size = 12 }: { size?: number }) => (
   </svg>
 );
 
+/** Stacked-layers icon for Batch API call types (Lucide Layers-style). */
+export const LayersIcon = ({ size = 12 }: { size?: number }) => (
+  <svg
+    width={size}
+    height={size}
+    viewBox="0 0 24 24"
+    fill="none"
+    stroke="currentColor"
+    strokeWidth="2"
+    strokeLinecap="round"
+    strokeLinejoin="round"
+    className="shrink-0"
+  >
+    <path d="M12 2 2 7l10 5 10-5-10-5z" />
+    <path d="m2 17 10 5 10-5" />
+    <path d="m2 12 10 5 10-5" />
+  </svg>
+);
+
 export const LlmBadge = ({ count }: { count?: number }) => (
   <span className="inline-flex items-center gap-1 px-2 py-0.5 bg-info/10 text-info border border-info/20 rounded-full text-[11px] font-medium whitespace-nowrap">
     <SparkleIcon />
@@ -77,3 +96,10 @@ export const AgentBadge = ({ count }: { count?: number }) => (
     {count != null ? count : "Agent"}
   </span>
 );
+
+export const BatchBadge = ({ count }: { count?: number }) => (
+  <span className="inline-flex items-center gap-1 px-2 py-0.5 bg-teal-50 text-teal-700 border border-teal-200 rounded-full text-[11px] font-medium whitespace-nowrap dark:bg-teal-950 dark:text-teal-300 dark:border-teal-800">
+    <LayersIcon />
+    {count != null ? count : "Batch"}
+  </span>
+);
diff --git a/ui/litellm-dashboard/src/components/view_logs/batchLogUtils.test.ts b/ui/litellm-dashboard/src/components/view_logs/batchLogUtils.test.ts
new file mode 100644
index 00000000000..4da1e389646
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/batchLogUtils.test.ts
@@ -0,0 +1,90 @@
+import { describe, expect, it } from "vitest";
+
+import {
+  getBatchIdFromRequestId,
+  getBatchModels,
+  getBatchRequestCounts,
+  getReasoningTokens,
+  isBatchCallType,
+} from "./batchLogUtils";
+
+/** Metadata shape the batch cost poller writes on an aretrieve_batch spend row. */
+const batchCostMetadata = {
+  batch_models: ["gemini-2.5-flash"],
+  batch_successful_requests: 2,
+  batch_failed_requests: 1,
+  usage_object: {
+    total_tokens: 270,
+    prompt_tokens: 14,
+    completion_tokens: 256,
+    completion_tokens_details: { text_tokens: 32, reasoning_tokens: 224 },
+  },
+};
+
+describe("isBatchCallType", () => {
+  it("recognizes the poller's aretrieve_batch and the create call types", () => {
+    for (const callType of ["aretrieve_batch", "retrieve_batch", "acreate_batch", "create_batch"]) {
+      expect(isBatchCallType(callType)).toBe(true);
+    }
+    expect(isBatchCallType("acompletion")).toBe(false);
+  });
+});
+
+describe("getBatchRequestCounts", () => {
+  it("reads both counts off a batch cost row", () => {
+    expect(getBatchRequestCounts(batchCostMetadata)).toEqual({ successful: 2, failed: 1 });
+  });
+
+  it("returns undefined for a non-batch row and for null counts, so no rollup renders", () => {
+    expect(getBatchRequestCounts({ status: "success" })).toBeUndefined();
+    expect(getBatchRequestCounts({ batch_successful_requests: null, batch_failed_requests: null })).toBeUndefined();
+    expect(getBatchRequestCounts(undefined)).toBeUndefined();
+  });
+
+  it("treats a lone present count as the other being 0, for rows logged mid-rollout", () => {
+    expect(getBatchRequestCounts({ batch_successful_requests: 3 })).toEqual({ successful: 3, failed: 0 });
+  });
+});
+
+describe("getBatchIdFromRequestId", () => {
+  it("strips the poller's synthetic _batch_cost suffix down to the provider batch id", () => {
+    expect(getBatchIdFromRequestId("batch_abc123_batch_cost")).toBe("batch_abc123");
+  });
+
+  it("returns undefined for ordinary request ids and a bare suffix", () => {
+    expect(getBatchIdFromRequestId("chatcmpl-123")).toBeUndefined();
+    expect(getBatchIdFromRequestId("_batch_cost")).toBeUndefined();
+  });
+});
+
+describe("getBatchModels", () => {
+  it("returns the model list from metadata.batch_models", () => {
+    expect(getBatchModels(batchCostMetadata)).toEqual(["gemini-2.5-flash"]);
+  });
+
+  it("returns undefined when absent, null, or empty", () => {
+    expect(getBatchModels({})).toBeUndefined();
+    expect(getBatchModels({ batch_models: null })).toBeUndefined();
+    expect(getBatchModels({ batch_models: [] })).toBeUndefined();
+  });
+});
+
+describe("getReasoningTokens", () => {
+  it("reads reasoning tokens from usage_object on a batch cost row", () => {
+    expect(getReasoningTokens(batchCostMetadata)).toBe(224);
+  });
+
+  it("prefers additional_usage_values, which per-request rows carry", () => {
+    const metadata = {
+      additional_usage_values: { completion_tokens_details: { reasoning_tokens: 40 } },
+      usage_object: { completion_tokens_details: { reasoning_tokens: 999 } },
+    };
+    expect(getReasoningTokens(metadata)).toBe(40);
+  });
+
+  it("returns undefined when the breakout is null or missing", () => {
+    expect(getReasoningTokens({ usage_object: { completion_tokens_details: null } })).toBeUndefined();
+    expect(getReasoningTokens({})).toBeUndefined();
+    expect(getReasoningTokens(undefined)).toBeUndefined();
+  });
+});
diff --git a/ui/litellm-dashboard/src/components/view_logs/batchLogUtils.ts b/ui/litellm-dashboard/src/components/view_logs/batchLogUtils.ts
new file mode 100644
index 00000000000..ce7793065d2
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/view_logs/batchLogUtils.ts
@@ -0,0 +1,67 @@
+/**
+ * Helpers for reading batch-specific fields off a spend log row.
+ *
+ * The proxy's batch cost poller (CheckBatchCost) writes one spend log per completed batch
+ * with request_id "<batch_id>_batch_cost" and call_type "aretrieve_batch", carrying
+ * batch_models / batch_successful_requests / batch_failed_requests in metadata
+ * (see litellm/proxy/spend_tracking/spend_tracking_utils.py).
+ */
+
+import { BATCH_CALL_TYPES } from "./constants";
+
+export const BATCH_COST_REQUEST_ID_SUFFIX = "_batch_cost";
+
+export interface BatchRequestCounts {
+  successful: number;
+  failed: number;
+}
+
+export const isBatchCallType = (callType: string): boolean => BATCH_CALL_TYPES.includes(callType);
+
+const readMetaNumber = (metadata: Record<string, unknown> | undefined, key: string): number | undefined => {
+  const value = metadata?.[key];
+  return typeof value === "number" && Number.isFinite(value) ? value : undefined;
+};
+
+/**
+ * Per-request outcome counts of a batch cost row. Undefined when the row carries neither
+ * count (a non-batch row, or a batch logged before counts were tracked).
+ */
+export const getBatchRequestCounts = (
+  metadata: Record<string, unknown> | undefined,
+): BatchRequestCounts | undefined => {
+  const successful = readMetaNumber(metadata, "batch_successful_requests");
+  const failed = readMetaNumber(metadata, "batch_failed_requests");
+  if (successful === undefined && failed === undefined) return undefined;
+  return { successful: successful ?? 0, failed: failed ?? 0 };
+};
+
+/** The provider batch id behind a poller-written "<batch_id>_batch_cost" spend row. */
+export const getBatchIdFromRequestId = (requestId: string): string | undefined =>
+  requestId.endsWith(BATCH_COST_REQUEST_ID_SUFFIX) && requestId.length > BATCH_COST_REQUEST_ID_SUFFIX.length
+    ? requestId.slice(0, -BATCH_COST_REQUEST_ID_SUFFIX.length)
+    : undefined;
+
+/** The models the batch's requests actually ran on, from metadata.batch_models. */
+export const getBatchModels = (metadata: Record<string, unknown> | undefined): string[] | undefined => {
+  const models = metadata?.["batch_models"];
+  if (!Array.isArray(models)) return undefined;
+  const names = models.filter((model): model is string => typeof model === "string" && model !== "");
+  return names.length > 0 ? names : undefined;
+};
+
+/**
+ * Reasoning tokens aggregated across the row's completion usage. Read from the same two
+ * metadata containers the drawer already uses for prompt-token details: per-request rows
+ * carry additional_usage_values, batch cost rows carry usage_object.
+ */
+export const getReasoningTokens = (metadata: Record<string, unknown> | undefined): number | undefined => {
+  const readDetails = (container: unknown): number | undefined => {
+    if (typeof container !== "object" || container === null) return undefined;
+    const details = (container as Record<string, unknown>)["completion_tokens_details"];
+    if (typeof details !== "object" || details === null) return undefined;
+    const reasoning = (details as Record<string, unknown>)["reasoning_tokens"];
+    return typeof reasoning === "number" && Number.isFinite(reasoning) ? reasoning : undefined;
+  };
+  return readDetails(metadata?.["additional_usage_values"]) ?? readDetails(metadata?.["usage_object"]);
+};
diff --git a/ui/litellm-dashboard/src/components/view_logs/constants.ts b/ui/litellm-dashboard/src/components/view_logs/constants.ts
index 1c17f398e35..51889ae7d04 100644
--- a/ui/litellm-dashboard/src/components/view_logs/constants.ts
+++ b/ui/litellm-dashboard/src/components/view_logs/constants.ts
@@ -21,6 +21,9 @@ export const MCP_CALL_TYPES = ["call_mcp_tool", "list_mcp_tools"];
 /** Call types that represent agent/A2A requests (e.g. asend_message). */
 export const AGENT_CALL_TYPES = ["asend_message"];
 
+/** Call types that represent Batch API operations (creation and retrieval, sync and async). */
+export const BATCH_CALL_TYPES = ["acreate_batch", "create_batch", "aretrieve_batch", "retrieve_batch"];
+
 export const QUICK_SELECT_OPTIONS: { label: string; value: number; unit: string }[] = [
   { label: "Last Minute", value: 1, unit: "minutes" },
   { label: "Last 15 Minutes", value: 15, unit: "minutes" },

From c276813cb46f57b1934d5e18bedfc53a09e74df0 Mon Sep 17 00:00:00 2001
From: mubashir1osmani <mubashir.osmani777@gmail.com>
Date: Thu, 3 Sep 2026 17:23:53 -0400
Subject: [PATCH 27/85] feat(batches): enrich batch cost rows with breakdown,
 identity, session, and org spend

---
 .../proxy/common_utils/check_batch_cost.py    | 48 ++++++++++----
 litellm/batches/batch_utils.py                | 52 +++++++++-------
 litellm/litellm_core_utils/litellm_logging.py | 15 +++++
 .../spend_tracking/spend_tracking_utils.py    | 27 +++++++-
 .../test_batches_logging_unit_tests.py        | 60 ++++++++++++++++--
 .../proxy_unit_tests/test_check_batch_cost.py | 45 ++++++++++++++
 .../test_litellm/batches/test_batch_utils.py  | 34 +++++++---
 .../test_spend_tracking_utils.py              | 62 +++++++++++++++++++
 .../view_logs/RequestLogsTableColumns.tsx     |  4 +-
 9 files changed, 295 insertions(+), 52 deletions(-)

diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
index 354a6ed2fd0..be6e52681fb 100644
--- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
+++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
@@ -112,35 +112,39 @@ class CheckBatchCost:
             verbose_proxy_logger.error(f"CheckBatchCost: could not look up user {user_id} for batch {batch_id}: {e}")
             return {}
 
-    async def _get_key_alias(self, batch_id: str, api_key: str | None) -> str | None:
-        """Resolve the creating virtual key's alias from its hashed token."""
+    async def _get_key_attribution(self, batch_id: str, api_key: str | None) -> tuple[str | None, str | None]:
+        """Resolve the creating virtual key's (alias, org_id) from its hashed token."""
         if not api_key:
-            return None
+            return None, None
         try:
             key_row: prisma_models.LiteLLM_VerificationToken | None = (
                 await self.prisma_client.db.litellm_verificationtoken.find_unique(
                     where={"token": api_key}
                 )
             )
-            return getattr(key_row, "key_alias", None) if key_row is not None else None
+            if key_row is None:
+                return None, None
+            return getattr(key_row, "key_alias", None), getattr(key_row, "org_id", None)
         except Exception as e:
             verbose_proxy_logger.error(f"CheckBatchCost: could not look up key alias for batch {batch_id}: {e}")
-            return None
+            return None, None
 
-    async def _get_team_alias(self, team_id: str | None) -> str | None:
-        """Resolve a team's alias from its id."""
+    async def _get_team_attribution(self, team_id: str | None) -> tuple[str | None, str | None]:
+        """Resolve a team's (alias, organization_id) from its id."""
         if not team_id:
-            return None
+            return None, None
         try:
             team_row: prisma_models.LiteLLM_TeamTable | None = (
                 await self.prisma_client.db.litellm_teamtable.find_unique(
                     where={"team_id": team_id}
                 )
             )
-            return getattr(team_row, "team_alias", None) if team_row is not None else None
+            if team_row is None:
+                return None, None
+            return getattr(team_row, "team_alias", None), getattr(team_row, "organization_id", None)
         except Exception as e:
             verbose_proxy_logger.error(f"CheckBatchCost: could not look up team alias for team {team_id}: {e}")
-            return None
+            return None, None
 
     async def _build_creator_attribution_metadata(
         self, job: "LiteLLM_ManagedObjectTable", batch_id: str
@@ -153,6 +157,10 @@ class CheckBatchCost:
         user_api_key_alias; when it has no alias, or the key has since been rotated or
         deleted, the field keeps the creating user's alias that _get_user_info filled in,
         because a resolvable name is more useful on the spend row than a null.
+
+        user_api_key_org_id must be resolved here too: the spend update writer reads it
+        off this metadata to increment organization spend, so leaving it out silently
+        drops batch cost from org accounting for keys and teams that belong to one.
         """
         api_key = getattr(job, "api_key", None)
         team_id = getattr(job, "team_id", None)
@@ -165,12 +173,15 @@ class CheckBatchCost:
             **(await self._get_user_info(batch_id, job.created_by)),
         }
 
-        key_alias = await self._get_key_alias(batch_id, api_key)
+        key_alias, key_org_id = await self._get_key_attribution(batch_id, api_key)
         if key_alias is not None:
             metadata["user_api_key_alias"] = key_alias
-        team_alias = await self._get_team_alias(team_id)
+        team_alias, team_org_id = await self._get_team_attribution(team_id)
         if team_alias is not None:
             metadata["user_api_key_team_alias"] = team_alias
+        org_id: Final = key_org_id or team_org_id
+        if org_id is not None:
+            metadata["user_api_key_org_id"] = org_id
         if isinstance(request_tags, list) and request_tags:
             metadata["tags"] = [tag for tag in request_tags if isinstance(tag, str)]
 
@@ -804,6 +815,7 @@ class CheckBatchCost:
             function_id=str(uuid.uuid4()),
         )
 
+        deployment_api_base: Final = deployment_info.litellm_params.api_base
         logging_obj.update_environment_variables(
             litellm_params={
                 # set the user-agent header so that S3 callback consumers can easily identify CheckBatchCost callbacks
@@ -812,9 +824,17 @@ class CheckBatchCost:
                         "user-agent": CHECK_BATCH_COST_USER_AGENT,
                     }
                 },
-                "metadata": await self._build_creator_attribution_metadata(job, batch_id),
+                **({"api_base": deployment_api_base} if deployment_api_base else {}),
+                "metadata": {
+                    **(await self._build_creator_attribution_metadata(job, batch_id)),
+                    # spend logs read the deployment identity off these metadata keys, so
+                    # without them the batch cost row carries no model_id or model_group
+                    "model_info": {"id": model_id},
+                    "model_group": deployment_info.model_name,
+                },
             },
             optional_params={},
+            custom_llm_provider=str(llm_provider) if llm_provider else None,
         )
 
         if not await self._claim_job_for_costing(job):
@@ -832,6 +852,8 @@ class CheckBatchCost:
                 batch_models=batch_result.models,
                 batch_successful_requests=batch_result.successful_requests,
                 batch_failed_requests=batch_result.failed_requests,
+                batch_prompt_cost=batch_result.prompt_cost,
+                batch_completion_cost=batch_result.completion_cost,
             )
         except Exception:
             await self._release_job_claim(job)
diff --git a/litellm/batches/batch_utils.py b/litellm/batches/batch_utils.py
index 3831f57a10d..a688e9f69fb 100644
--- a/litellm/batches/batch_utils.py
+++ b/litellm/batches/batch_utils.py
@@ -10,7 +10,7 @@ from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS
 from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details
 from litellm.types.llms.openai import Batch
-from litellm.types.utils import CallTypes, ModelInfo, Usage
+from litellm.types.utils import ModelInfo, Usage
 from litellm.utils import token_counter
 
 
@@ -23,6 +23,8 @@ class BatchCostUsageResult:
     models: list[str]
     successful_requests: int
     failed_requests: int
+    prompt_cost: float = 0.0
+    completion_cost: float = 0.0
 
 
 async def calculate_batch_cost_and_usage(
@@ -130,7 +132,8 @@ class _LineOutcome(Enum):
 
 @dataclass(frozen=True, slots=True)
 class _BatchOutputLineStats:
-    cost: float
+    prompt_cost: float
+    completion_cost: float
     prompt_tokens: int
     completion_tokens: int
     total_tokens: int
@@ -193,15 +196,16 @@ def _compute_output_line_stats(
     raw_model: Final = response_body.get("model")
     response_model: Final = raw_model if isinstance(raw_model, str) and raw_model else None
     completion_details: Final = usage.completion_tokens_details
+    line_prompt_cost, line_completion_cost = _output_line_cost(
+        usage=usage,
+        custom_llm_provider=custom_llm_provider,
+        model_name=model_name,
+        response_model=response_model,
+        model_info=model_info,
+    )
     return _BatchOutputLineStats(
-        cost=_output_line_cost(
-            response_body=response_body,
-            usage=usage,
-            custom_llm_provider=custom_llm_provider,
-            model_name=model_name,
-            response_model=response_model,
-            model_info=model_info,
-        ),
+        prompt_cost=line_prompt_cost,
+        completion_cost=line_completion_cost,
         prompt_tokens=usage.prompt_tokens,
         completion_tokens=usage.completion_tokens,
         total_tokens=usage.total_tokens,
@@ -213,31 +217,24 @@ def _compute_output_line_stats(
 
 
 def _output_line_cost(
-    response_body: Mapping[str, Any],
     usage: Usage,
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"],
     model_name: str | None,
     response_model: str | None,
     model_info: ModelInfo | None,
-) -> float:
+) -> tuple[float, float]:
+    """(prompt_cost, completion_cost) for one output line, priced at batch rates."""
     from litellm.cost_calculator import batch_cost_calculator
 
-    if model_info is None and custom_llm_provider not in ("anthropic", "bedrock"):
-        return litellm.completion_cost(
-            completion_response=response_body,
-            custom_llm_provider=custom_llm_provider,
-            call_type=CallTypes.aretrieve_batch.value,
-        )
     cost_model: Final = (
         model_name if custom_llm_provider == "bedrock" and model_name else response_model or model_name or ""
     )
-    prompt_cost, completion_cost = batch_cost_calculator(
+    return batch_cost_calculator(
         usage=usage,
         model=cost_model,
         custom_llm_provider=custom_llm_provider,
         model_info=model_info,
     )
-    return prompt_cost + completion_cost
 
 
 def _aggregate_batch_cost_usage_models(
@@ -270,7 +267,9 @@ def _aggregate_batch_cost_usage_models(
         **cache_token_params,
     )
     batch_models: Final = [model_name] if model_name else [stats.model for stats in line_stats if stats.model]
-    total_cost: Final = sum((stats.cost for stats in line_stats), 0.0)
+    total_prompt_cost: Final = sum((stats.prompt_cost for stats in line_stats), 0.0)
+    total_completion_cost: Final = sum((stats.completion_cost for stats in line_stats), 0.0)
+    total_cost: Final = total_prompt_cost + total_completion_cost
     verbose_logger.debug(
         "batch output aggregate: cost=%s usage=%s models=%s successful=%d failed=%d",
         total_cost,
@@ -285,6 +284,8 @@ def _aggregate_batch_cost_usage_models(
         models=batch_models,
         successful_requests=successful_requests,
         failed_requests=failed_requests,
+        prompt_cost=total_prompt_cost,
+        completion_cost=total_completion_cost,
     )
 
 
@@ -309,7 +310,8 @@ def calculate_vertex_ai_batch_cost_and_usage(
     """
     from litellm.cost_calculator import batch_cost_calculator
 
-    total_cost = 0.0
+    total_prompt_cost = 0.0  # rebind-ok: loop accumulator, matches total_tokens below
+    total_completion_cost = 0.0  # rebind-ok: loop accumulator, matches total_tokens below
     total_tokens = 0
     prompt_tokens = 0
     completion_tokens = 0
@@ -341,7 +343,8 @@ def calculate_vertex_ai_batch_cost_and_usage(
                 model=actual_model_name,
                 custom_llm_provider="vertex_ai",
             )
-            total_cost += p_cost + c_cost
+            total_prompt_cost += p_cost
+            total_completion_cost += c_cost
         except Exception as e:
             verbose_logger.debug("vertex_ai batch cost calculation error for line: %s", str(e))
 
@@ -349,6 +352,7 @@ def calculate_vertex_ai_batch_cost_and_usage(
         completion_tokens += _completion
         total_tokens += _total
 
+    total_cost: Final = total_prompt_cost + total_completion_cost
     verbose_logger.info(
         "vertex_ai batch cost: cost=%s, prompt=%d, completion=%d, total=%d, successful=%d, failed=%d",
         total_cost,
@@ -369,6 +373,8 @@ def calculate_vertex_ai_batch_cost_and_usage(
         models=[actual_model_name],
         successful_requests=successful_requests,
         failed_requests=failed_requests,
+        prompt_cost=total_prompt_cost,
+        completion_cost=total_completion_cost,
     )
 
 
diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py
index f54eeca5178..3a616e7961d 100644
--- a/litellm/litellm_core_utils/litellm_logging.py
+++ b/litellm/litellm_core_utils/litellm_logging.py
@@ -2904,6 +2904,15 @@ class Logging(LiteLLMLoggingBaseClass):
                 result._hidden_params["batch_successful_requests"] = batch_successful_requests  # pyright: ignore[reportPrivateUsage]  # rebind-ok: same result._hidden_params pattern as response_cost/batch_models above
                 result._hidden_params["batch_failed_requests"] = batch_failed_requests  # pyright: ignore[reportPrivateUsage]  # rebind-ok: same pattern as above
                 result.usage = batch_usage
+                batch_prompt_cost: Final = kwargs.get("batch_prompt_cost", None)
+                batch_completion_cost: Final = kwargs.get("batch_completion_cost", None)
+                if batch_prompt_cost is not None and batch_completion_cost is not None:
+                    self.set_cost_breakdown(
+                        input_cost=batch_prompt_cost,
+                        output_cost=batch_completion_cost,
+                        total_cost=batch_cost,
+                        cost_for_built_in_tools_cost_usd_dollar=0.0,
+                    )
 
             elif should_compute_batch_data:
                 batch_result: Final = await _handle_completed_batch(
@@ -2919,6 +2928,12 @@ class Logging(LiteLLMLoggingBaseClass):
                 result._hidden_params["batch_successful_requests"] = batch_result.successful_requests  # pyright: ignore[reportPrivateUsage]  # rebind-ok: same pattern as above
                 result._hidden_params["batch_failed_requests"] = batch_result.failed_requests  # pyright: ignore[reportPrivateUsage]  # rebind-ok: same pattern as above
                 result.usage = batch_result.usage
+                self.set_cost_breakdown(
+                    input_cost=batch_result.prompt_cost,
+                    output_cost=batch_result.completion_cost,
+                    total_cost=batch_result.cost,
+                    cost_for_built_in_tools_cost_usd_dollar=0.0,
+                )
 
         start_time, end_time, result = self._success_handler_helper_fn(
             start_time=start_time,
diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py
index a37c3ba4405..10db82d1b64 100644
--- a/litellm/proxy/spend_tracking/spend_tracking_utils.py
+++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py
@@ -582,6 +582,7 @@ def get_logging_payload(kwargs, response_obj, start_time, end_time) -> SpendLogs
                 metadata=metadata,
                 standard_logging_payload=standard_logging_payload,
                 omit_when_missing=_omits_session_id_when_missing(metadata),
+                batch_trace_session_id=_get_batch_trace_session_id(call_type=call_type, request_id=id),
             ),
             request_duration_ms=_get_request_duration_ms(start_time, end_time),
             status=_get_status_for_spend_log(
@@ -620,20 +621,44 @@ def _omits_session_id_when_missing(metadata: Mapping[str, object] | None) -> boo
     return general_settings.get("missing_session_id") == "omit"
 
 
+_BATCH_TRACE_CALL_TYPES: Final = frozenset(
+    {
+        CallTypes.create_batch.value,
+        CallTypes.acreate_batch.value,
+        CallTypes.retrieve_batch.value,
+        CallTypes.aretrieve_batch.value,
+    }
+)
+
+
+def _get_batch_trace_session_id(call_type: str | None, request_id: str | None) -> str | None:
+    """A batch's create row and its poller-written cost row both derive their request id
+    from the same batch id (the cost row appends BATCH_COST_REQUEST_ID_SUFFIX), so using
+    that id as the session groups the batch lifecycle into one trace on the logs UI. The
+    poller builds its own logging context, so per-request trace ids can never link them."""
+    if call_type not in _BATCH_TRACE_CALL_TYPES or not request_id:
+        return None
+    return request_id.removesuffix(BATCH_COST_REQUEST_ID_SUFFIX)
+
+
 def _get_session_id_for_spend_log(
     kwargs: Mapping[str, object],
     metadata: Mapping[str, object] | None,
     standard_logging_payload: StandardLoggingPayload | None,
     omit_when_missing: bool,
+    batch_trace_session_id: str | None = None,
 ) -> str | None:
     """Under `omit` only `metadata.session_id`, the key Langfuse reads, counts as a session; `litellm_session_id` may
-    be a copied trace id."""
+    be a copied trace id. Batch call types carry a deterministic session derived from the batch id, which outranks
+    the per-request trace ids because those differ between the create call and the cost poller's row."""
     if omit_when_missing:
         session_id: Final = metadata.get("session_id") if metadata else None
         return str(session_id) if session_id else None
 
     from litellm._uuid import uuid
 
+    if batch_trace_session_id is not None:
+        return batch_trace_session_id
     if standard_logging_payload is not None and standard_logging_payload.get("trace_id") is not None:
         return str(standard_logging_payload.get("trace_id"))
     if kwargs.get("litellm_trace_id") is not None:
diff --git a/tests/batches_tests/test_batches_logging_unit_tests.py b/tests/batches_tests/test_batches_logging_unit_tests.py
index 5bde40d90b0..73adb391481 100644
--- a/tests/batches_tests/test_batches_logging_unit_tests.py
+++ b/tests/batches_tests/test_batches_logging_unit_tests.py
@@ -144,18 +144,20 @@ def test_get_batch_job_total_usage_from_file_content(sample_file_content_dict):
 @pytest.mark.asyncio
 async def test_batch_cost_calculator(sample_file_content_dict):
     """
-    mock litellm.completion_cost to return 0.5
+    mock batch_cost_calculator to return (0.3, 0.2) per line
 
     we know sample_file_content_dict has 2 successful responses
 
-    so we expect the cost to be 0.5 * 2 = 1.0
+    so we expect the cost to be (0.3 + 0.2) * 2 = 1.0, split 0.6 / 0.4
     """
-    with patch("litellm.completion_cost", return_value=0.5):
+    with patch("litellm.cost_calculator.batch_cost_calculator", return_value=(0.3, 0.2)):
         result = _aggregate_batch_cost_usage_models(
             entries=sample_file_content_dict,
             custom_llm_provider="openai",
         )
-        assert result.cost == 1.0  # 0.5 * 2 successful responses
+        assert result.cost == pytest.approx(1.0)  # (0.3 + 0.2) * 2 successful responses
+        assert result.prompt_cost == pytest.approx(0.6)
+        assert result.completion_cost == pytest.approx(0.4)
 
 
 def test_get_response_from_batch_job_output_file(sample_file_content_dict):
@@ -402,6 +404,56 @@ async def test_batch_retrieve_cost_tracking_with_explicit_cost_data():
         assert mock_batch.usage == explicit_usage
 
 
+@pytest.mark.asyncio
+async def test_batch_retrieve_explicit_cost_split_sets_cost_breakdown():
+    """The poller passes the batch's prompt/completion cost split so the spend row's
+    cost_breakdown carries real input/output costs; without it the UI's Cost Breakdown
+    card renders blank for every batch. Regression for the split being dropped."""
+    from litellm.litellm_core_utils.litellm_logging import Logging
+    from litellm.types.utils import CallTypes, LiteLLMBatch
+
+    mock_batch = LiteLLMBatch(
+        id="batch-breakdown-1",
+        object="batch",
+        endpoint="/v1/chat/completions",
+        errors=None,
+        input_file_id="file-input-1",
+        completion_window="24h",
+        status="completed",
+        output_file_id="file-output-1",
+        created_at=1234567890,
+    )
+    mock_batch._hidden_params = {}
+
+    logging_obj = Logging(
+        model="gpt-5-mini",
+        messages=[{"role": "user", "content": "test"}],
+        stream=False,
+        call_type=CallTypes.aretrieve_batch.value,
+        litellm_call_id="test-call-breakdown",
+        function_id="test-function",
+        start_time=time.time(),
+        dynamic_success_callbacks=[],
+    )
+    logging_obj.custom_llm_provider = "openai"
+
+    await logging_obj.async_success_handler(
+        result=mock_batch,
+        start_time=time.time(),
+        end_time=time.time() + 1,
+        batch_cost=0.10,
+        batch_usage=litellm.Usage(prompt_tokens=200, completion_tokens=100, total_tokens=300),
+        batch_models=["gpt-5-mini"],
+        batch_prompt_cost=0.06,
+        batch_completion_cost=0.04,
+    )
+
+    assert logging_obj.cost_breakdown is not None
+    assert logging_obj.cost_breakdown["input_cost"] == 0.06
+    assert logging_obj.cost_breakdown["output_cost"] == 0.04
+    assert logging_obj.cost_breakdown["total_cost"] == 0.10
+
+
 @pytest.mark.asyncio
 async def test_batch_retrieve_cost_tracking_with_unified_file_id_incomplete_batch():
     """
diff --git a/tests/proxy_unit_tests/test_check_batch_cost.py b/tests/proxy_unit_tests/test_check_batch_cost.py
index ff5e8f89d64..9ee0a36f00b 100644
--- a/tests/proxy_unit_tests/test_check_batch_cost.py
+++ b/tests/proxy_unit_tests/test_check_batch_cost.py
@@ -2553,6 +2553,51 @@ class TestBatchCostAttribution:
 
         assert metadata["user_api_key_alias"] == "prod-key"
 
+    @pytest.mark.asyncio
+    async def test_org_id_comes_from_the_creating_key(self):
+        """The spend update writer increments organization spend from user_api_key_org_id,
+        so an org-scoped key's batch cost must carry the key's org id."""
+        from types import SimpleNamespace
+
+        instance = self._instance(
+            key_row=SimpleNamespace(key_alias="prod-key", org_id="org-42"),
+            team_row=SimpleNamespace(team_alias="Team Alpha", organization_id="org-team"),
+        )
+
+        metadata = await instance._build_creator_attribution_metadata(self._job(), "batch-1")
+
+        assert metadata["user_api_key_org_id"] == "org-42"
+
+    @pytest.mark.asyncio
+    async def test_org_id_falls_back_to_the_team_organization(self):
+        """A key with no org of its own still books batch spend against its team's
+        organization, matching how the request path resolves org attribution."""
+        from types import SimpleNamespace
+
+        instance = self._instance(
+            key_row=SimpleNamespace(key_alias="prod-key", org_id=None),
+            team_row=SimpleNamespace(team_alias="Team Alpha", organization_id="org-team"),
+        )
+
+        metadata = await instance._build_creator_attribution_metadata(self._job(), "batch-1")
+
+        assert metadata["user_api_key_org_id"] == "org-team"
+
+    @pytest.mark.asyncio
+    async def test_no_org_leaves_the_key_unset(self):
+        """Without any org the key is absent entirely, so the spend writer's org update
+        stays skipped instead of matching an empty-string organization."""
+        from types import SimpleNamespace
+
+        instance = self._instance(
+            key_row=SimpleNamespace(key_alias="prod-key", org_id=None),
+            team_row=SimpleNamespace(team_alias="Team Alpha", organization_id=None),
+        )
+
+        metadata = await instance._build_creator_attribution_metadata(self._job(), "batch-1")
+
+        assert "user_api_key_org_id" not in metadata
+
 
 class TestPollPageStarvation:
     """LIT-5462 regression: a row that can never be costed used to keep its slot in the
diff --git a/tests/test_litellm/batches/test_batch_utils.py b/tests/test_litellm/batches/test_batch_utils.py
index c86c7c4df03..1fd78870481 100644
--- a/tests/test_litellm/batches/test_batch_utils.py
+++ b/tests/test_litellm/batches/test_batch_utils.py
@@ -489,7 +489,9 @@ def test_aggregate_counts_successful_and_failed_requests(monkeypatch):
 
 
 def test_aggregate_returns_batch_cost_usage_result_dataclass(monkeypatch):
-    monkeypatch.setattr(litellm, "completion_cost", lambda **kw: 1.0)
+    import litellm.cost_calculator as cc
+
+    monkeypatch.setattr(cc, "batch_cost_calculator", lambda **kw: (0.4, 0.6))
     result = bu._aggregate_batch_cost_usage_models(
         entries=[_success_row(usage=_usage(10, 5))], custom_llm_provider="openai"
     )
@@ -500,6 +502,7 @@ def test_aggregate_returns_batch_cost_usage_result_dataclass(monkeypatch):
         1,
         0,
     )
+    assert (result.prompt_cost, result.completion_cost) == (0.4, 0.6)
 
 
 # =========================================================================== #
@@ -507,15 +510,17 @@ def test_aggregate_returns_batch_cost_usage_result_dataclass(monkeypatch):
 # =========================================================================== #
 
 
-def test_cost_from_content_completion_cost_path(monkeypatch):
-    # model_info is None -> litellm.completion_cost per successful row.
+def test_cost_without_model_info_prices_each_row_by_its_response_model(monkeypatch):
+    # model_info is None -> batch_cost_calculator per successful row, model from the response body.
+    import litellm.cost_calculator as cc
+
     calls = []
 
-    def _completion_cost(**kw):
+    def _batch_cost(**kw):
         calls.append(kw)
-        return 0.5
+        return (0.3, 0.2)
 
-    monkeypatch.setattr(litellm, "completion_cost", _completion_cost)
+    monkeypatch.setattr(cc, "batch_cost_calculator", _batch_cost)
     rows = [
         _success_row(usage=_usage(10, 5)),
         _failed_row(),  # excluded -> not costed
@@ -524,8 +529,10 @@ def test_cost_from_content_completion_cost_path(monkeypatch):
 
     result = bu._aggregate_batch_cost_usage_models(entries=rows, custom_llm_provider="openai")
 
-    assert result.cost == 1.0  # 2 successful * 0.5
+    assert result.cost == pytest.approx(1.0)  # 2 successful * (0.3 + 0.2)
+    assert (result.prompt_cost, result.completion_cost) == (pytest.approx(0.6), pytest.approx(0.4))
     assert len(calls) == 2  # failed row not costed
+    assert all(call["model"] == "gpt-4o" and call["model_info"] is None for call in calls)
     assert result.successful_requests == 2
     assert result.failed_requests == 1
 
@@ -578,7 +585,9 @@ def test_aggregate_consumes_entries_in_a_single_pass(monkeypatch):
     """A one-shot generator: any implementation that iterates the entries twice
     (e.g. separate cost and usage passes) sees nothing on the second pass and
     returns wrong totals for at least one of cost/usage/models."""
-    monkeypatch.setattr(litellm, "completion_cost", lambda **kw: 0.5)
+    import litellm.cost_calculator as cc
+
+    monkeypatch.setattr(cc, "batch_cost_calculator", lambda **kw: (0.25, 0.25))
     one_shot = (row for row in [_success_row(usage=_usage(10, 5)), _failed_row(), _success_row(usage=_usage(20, 10))])
 
     result = bu._aggregate_batch_cost_usage_models(entries=one_shot, custom_llm_provider="openai")
@@ -753,12 +762,15 @@ def test_vertex_cost_error_in_line_is_swallowed(monkeypatch):
 
 @pytest.mark.asyncio
 async def test_calculate_batch_cost_and_usage_orchestration(monkeypatch):
+    import litellm.cost_calculator as cc
+
     rows = [_success_row(model="gpt-4o", usage=_usage(10, 5))]
-    monkeypatch.setattr(litellm, "completion_cost", lambda **kw: 2.5)
+    monkeypatch.setattr(cc, "batch_cost_calculator", lambda **kw: (1.5, 1.0))
 
     result = await bu.calculate_batch_cost_and_usage(file_content_dictionary=rows, custom_llm_provider="openai")
 
     assert result.cost == 2.5
+    assert (result.prompt_cost, result.completion_cost) == (1.5, 1.0)
     assert (result.usage.prompt_tokens, result.usage.completion_tokens, result.usage.total_tokens) == (10, 5, 15)
     assert result.models == ["gpt-4o"]
 
@@ -1107,8 +1119,10 @@ async def test_handle_completed_batch_orchestration(monkeypatch):
     async def fake_fetch(batch, custom_llm_provider, litellm_params=None):
         return _vertex_jsonl(rows)
 
+    import litellm.cost_calculator as cc
+
     monkeypatch.setattr(bu, "_fetch_batch_output_file_content", fake_fetch)
-    monkeypatch.setattr(litellm, "completion_cost", lambda **kw: 3.3)
+    monkeypatch.setattr(cc, "batch_cost_calculator", lambda **kw: (2.0, 1.3))
 
     result = await bu._handle_completed_batch(_batch("of"), custom_llm_provider="openai")
 
diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
index 323930eee60..be274befd25 100644
--- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
+++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
@@ -128,6 +128,68 @@ def test_legacy_policy_keeps_trace_id_fallback():
     assert len(str(generated)) == 36
 
 
+def test_batch_lifecycle_rows_derive_the_same_session_from_the_batch_id():
+    """The create call's request id IS the batch id and the poller's cost row appends
+    _batch_cost to it, so deriving the session from the request id lands both rows in one
+    trace on the logs UI even though the poller builds a fresh logging context per cycle."""
+    from litellm.proxy.spend_tracking.spend_tracking_utils import _get_batch_trace_session_id
+
+    create_session: Final = _get_batch_trace_session_id(call_type="acreate_batch", request_id="batch-uid-1")
+    cost_session: Final = _get_batch_trace_session_id(
+        call_type="aretrieve_batch", request_id="batch-uid-1_batch_cost"
+    )
+    assert create_session == cost_session == "batch-uid-1"
+
+
+def test_non_batch_call_types_derive_no_batch_session():
+    from litellm.proxy.spend_tracking.spend_tracking_utils import _get_batch_trace_session_id
+
+    assert _get_batch_trace_session_id(call_type="acompletion", request_id="chatcmpl-1") is None
+
+
+def test_batch_session_outranks_the_per_request_trace_id():
+    """Each batch lifecycle call carries its own auto-generated trace id, so letting the
+    trace id win would scatter the rows across sessions again."""
+    session_id: Final = _get_session_id_for_spend_log(
+        kwargs={"litellm_trace_id": "trace-abc"},
+        metadata={"trace_id": "trace-abc"},
+        standard_logging_payload=_TRACE_ONLY_STANDARD_LOGGING,
+        omit_when_missing=False,
+        batch_trace_session_id="batch-uid-1",
+    )
+    assert session_id == "batch-uid-1"
+
+
+def test_omit_policy_still_suppresses_batch_sessions():
+    session_id: Final = _get_session_id_for_spend_log(
+        kwargs={},
+        metadata=None,
+        standard_logging_payload=None,
+        omit_when_missing=True,
+        batch_trace_session_id="batch-uid-1",
+    )
+    assert session_id is None
+
+
+def test_get_logging_payload_groups_batch_create_and_cost_rows_in_one_session():
+    def _payload(call_type: str) -> SpendLogsPayload:
+        return get_logging_payload(
+            kwargs={
+                "call_type": call_type,
+                "model": "gpt-4o-mini",
+                "litellm_params": {"metadata": {"user_api_key": "test-key"}},
+            },
+            response_obj=litellm.ModelResponse(id="batch-uid-1", choices=[], usage=litellm.Usage()),
+            start_time=datetime.datetime.now(timezone.utc),
+            end_time=datetime.datetime.now(timezone.utc),
+        )
+
+    create_payload: Final = _payload("acreate_batch")
+    cost_payload: Final = _payload("aretrieve_batch")
+    assert cost_payload["request_id"] == "batch-uid-1_batch_cost"
+    assert create_payload["session_id"] == cost_payload["session_id"] == "batch-uid-1"
+
+
 @pytest.mark.parametrize(
     ("request_metadata", "expected"),
     [
diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx
index b4296abe266..a444acb9517 100644
--- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx
@@ -64,10 +64,12 @@ export const getRequestLogsTableColumns = ({
       const sessionAgentCount = log.session_agent_count ?? (isAgent ? sessionCount : 0);
       const sessionMcpCount = log.mcp_tool_call_count ?? (isMcp ? sessionCount : 0);
 
+      if (isBatchCallType(log.call_type)) {
+        return <BatchBadge count={sessionCount > 1 ? sessionCount : undefined} />;
+      }
       if (sessionCount <= 1) {
         if (isMcp) return <McpBadge />;
         if (isAgent) return <AgentBadge />;
-        if (isBatchCallType(log.call_type)) return <BatchBadge />;
         return <LlmBadge />;
       }
 

From 06c860b60a5c4a2d14a02dead9c809b8b7eaf22f Mon Sep 17 00:00:00 2001
From: mubashir1osmani <mubashir.osmani777@gmail.com>
Date: Thu, 3 Sep 2026 18:09:18 -0400
Subject: [PATCH 28/85] fix(ui): type batch results metadata as unknown instead
 of any

---
 .../components/view_logs/LogDetailsDrawer/LogDetailContent.tsx  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx
index 8e0f041b88d..710d82b2f9d 100644
--- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.tsx
@@ -388,7 +388,7 @@ function MetricLabel({ label, tooltip, docsUrl }: { label: string; tooltip: stri
  * Aggregate per-request outcomes for a batch cost row: batch id, success/failure counts
  * from the parsed output and error files, and the models the batch actually ran on.
  */
-function BatchResultsSection({ logEntry, metadata }: { logEntry: LogEntry; metadata: Record<string, any> }) {
+function BatchResultsSection({ logEntry, metadata }: { logEntry: LogEntry; metadata: Record<string, unknown> }) {
   const counts = getBatchRequestCounts(metadata);
   const batchId = getBatchIdFromRequestId(logEntry.request_id);
   const batchModels = getBatchModels(metadata);

From 538cd2c3b00d31fcb516c8326de87883e0e27f94 Mon Sep 17 00:00:00 2001
From: mubashir1osmani <mubashir.osmani777@gmail.com>
Date: Thu, 3 Sep 2026 18:16:31 -0400
Subject: [PATCH 29/85] fix(batches): narrow batch cost kwargs before the
 breakdown and drop node access in test

---
 litellm/litellm_core_utils/litellm_logging.py        |  6 +++++-
 .../LogDetailsDrawer/LogDetailContent.test.tsx       | 12 +++++++-----
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py
index 3a616e7961d..03486f4f729 100644
--- a/litellm/litellm_core_utils/litellm_logging.py
+++ b/litellm/litellm_core_utils/litellm_logging.py
@@ -2906,7 +2906,11 @@ class Logging(LiteLLMLoggingBaseClass):
                 result.usage = batch_usage
                 batch_prompt_cost: Final = kwargs.get("batch_prompt_cost", None)
                 batch_completion_cost: Final = kwargs.get("batch_completion_cost", None)
-                if batch_prompt_cost is not None and batch_completion_cost is not None:
+                if (
+                    isinstance(batch_prompt_cost, float)
+                    and isinstance(batch_completion_cost, float)
+                    and isinstance(batch_cost, float)
+                ):
                     self.set_cost_breakdown(
                         input_cost=batch_prompt_cost,
                         output_cost=batch_completion_cost,
diff --git a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx
index 91778d2a98a..a893e9bffd0 100644
--- a/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/LogDetailsDrawer/LogDetailContent.test.tsx
@@ -163,11 +163,13 @@ describe("LogDetailContent", () => {
         />,
       );
 
-      const section = screen.getByText("Batch Results").closest('[data-slot="card"]') as HTMLElement;
-      expect(within(section).getByText("batch_abc123")).toBeInTheDocument();
-      expect(within(section).getByText("2")).toBeInTheDocument();
-      expect(within(section).getByText("1")).toBeInTheDocument();
-      expect(within(section).getByText("gemini-2.5-flash")).toBeInTheDocument();
+      expect(screen.getByText("Batch Results")).toBeInTheDocument();
+      expect(screen.getByText("batch_abc123")).toBeInTheDocument();
+      expect(screen.getByText("Successful Requests")).toBeInTheDocument();
+      expect(screen.getByText("2")).toBeInTheDocument();
+      expect(screen.getByText("Failed Requests")).toBeInTheDocument();
+      expect(screen.getByText("1")).toBeInTheDocument();
+      expect(screen.getByText("gemini-2.5-flash")).toBeInTheDocument();
     });
 
     it("still renders the batch id when a legacy row carries no counts", () => {

From 7fb4b427cec0db7d3bc85dd6c2055ca987f2390b Mon Sep 17 00:00:00 2001
From: mubashir1osmani <mubashir.osmani777@gmail.com>
Date: Thu, 3 Sep 2026 18:36:05 -0400
Subject: [PATCH 30/85] feat(batches): snapshot the creating key's org on the
 managed object row

---
 .../proxy/common_utils/check_batch_cost.py    | 61 +++++++++++++------
 .../proxy/hooks/managed_files.py              |  1 +
 .../migration.sql                             |  4 ++
 .../litellm_proxy_extras/schema.prisma        |  1 +
 litellm/models/managed_files.py               |  1 +
 schema.prisma                                 |  1 +
 .../proxy_unit_tests/test_check_batch_cost.py | 21 ++++++-
 ..._batch_update_db_managed_output_file_id.py |  4 +-
 8 files changed, 74 insertions(+), 20 deletions(-)
 create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20260903230000_add_org_id_to_managed_object_table/migration.sql

diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
index be6e52681fb..595a8d04bed 100644
--- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
+++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
@@ -112,39 +112,66 @@ class CheckBatchCost:
             verbose_proxy_logger.error(f"CheckBatchCost: could not look up user {user_id} for batch {batch_id}: {e}")
             return {}
 
-    async def _get_key_attribution(self, batch_id: str, api_key: str | None) -> tuple[str | None, str | None]:
-        """Resolve the creating virtual key's (alias, org_id) from its hashed token."""
+    async def _get_key_alias(self, batch_id: str, api_key: str | None) -> str | None:
+        """Resolve the creating virtual key's alias from its hashed token."""
         if not api_key:
-            return None, None
+            return None
         try:
             key_row: prisma_models.LiteLLM_VerificationToken | None = (
                 await self.prisma_client.db.litellm_verificationtoken.find_unique(
                     where={"token": api_key}
                 )
             )
-            if key_row is None:
-                return None, None
-            return getattr(key_row, "key_alias", None), getattr(key_row, "org_id", None)
+            return getattr(key_row, "key_alias", None) if key_row is not None else None
         except Exception as e:
             verbose_proxy_logger.error(f"CheckBatchCost: could not look up key alias for batch {batch_id}: {e}")
-            return None, None
+            return None
 
-    async def _get_team_attribution(self, team_id: str | None) -> tuple[str | None, str | None]:
-        """Resolve a team's (alias, organization_id) from its id."""
+    async def _get_team_alias(self, team_id: str | None) -> str | None:
+        """Resolve a team's alias from its id."""
         if not team_id:
-            return None, None
+            return None
         try:
             team_row: prisma_models.LiteLLM_TeamTable | None = (
                 await self.prisma_client.db.litellm_teamtable.find_unique(
                     where={"team_id": team_id}
                 )
             )
-            if team_row is None:
-                return None, None
-            return getattr(team_row, "team_alias", None), getattr(team_row, "organization_id", None)
+            return getattr(team_row, "team_alias", None) if team_row is not None else None
         except Exception as e:
             verbose_proxy_logger.error(f"CheckBatchCost: could not look up team alias for team {team_id}: {e}")
-            return None, None
+            return None
+
+    async def _get_org_id(self, job: "LiteLLM_ManagedObjectTable", batch_id: str) -> str | None:
+        """Organization to bill the batch against, snapshotted on the row at creation
+        like team_id. Rows created before the org_id column existed carry None, so they
+        fall back to the creating key's org (or its team's) as resolved today."""
+        org_id = getattr(job, "org_id", None)
+        if org_id:
+            return org_id
+        api_key = getattr(job, "api_key", None)
+        team_id = getattr(job, "team_id", None)
+        try:
+            if api_key:
+                key_row: prisma_models.LiteLLM_VerificationToken | None = (
+                    await self.prisma_client.db.litellm_verificationtoken.find_unique(
+                        where={"token": api_key}
+                    )
+                )
+                key_org_id = getattr(key_row, "org_id", None) if key_row is not None else None
+                if key_org_id:
+                    return key_org_id
+            if team_id:
+                team_row: prisma_models.LiteLLM_TeamTable | None = (
+                    await self.prisma_client.db.litellm_teamtable.find_unique(
+                        where={"team_id": team_id}
+                    )
+                )
+                return getattr(team_row, "organization_id", None) if team_row is not None else None
+            return None
+        except Exception as e:
+            verbose_proxy_logger.error(f"CheckBatchCost: could not resolve org for batch {batch_id}: {e}")
+            return None
 
     async def _build_creator_attribution_metadata(
         self, job: "LiteLLM_ManagedObjectTable", batch_id: str
@@ -173,13 +200,13 @@ class CheckBatchCost:
             **(await self._get_user_info(batch_id, job.created_by)),
         }
 
-        key_alias, key_org_id = await self._get_key_attribution(batch_id, api_key)
+        key_alias = await self._get_key_alias(batch_id, api_key)
         if key_alias is not None:
             metadata["user_api_key_alias"] = key_alias
-        team_alias, team_org_id = await self._get_team_attribution(team_id)
+        team_alias = await self._get_team_alias(team_id)
         if team_alias is not None:
             metadata["user_api_key_team_alias"] = team_alias
-        org_id: Final = key_org_id or team_org_id
+        org_id: Final = await self._get_org_id(job, batch_id)
         if org_id is not None:
             metadata["user_api_key_org_id"] = org_id
         if isinstance(request_tags, list) and request_tags:
diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py
index 5cfcf6129f0..4f0ca787280 100644
--- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py
+++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py
@@ -349,6 +349,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                     "file_purpose": file_purpose,
                     "created_by": resolve_resource_owner_id(user_api_key_dict),
                     "team_id": user_api_key_dict.team_id,
+                    "org_id": user_api_key_dict.org_id,
                     "updated_by": user_api_key_dict.user_id,
                     "status": file_object.status,
                     **attribution_columns,
diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260903230000_add_org_id_to_managed_object_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260903230000_add_org_id_to_managed_object_table/migration.sql
new file mode 100644
index 00000000000..bbe980bb66f
--- /dev/null
+++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260903230000_add_org_id_to_managed_object_table/migration.sql
@@ -0,0 +1,4 @@
+-- Add org_id column to LiteLLM_ManagedObjectTable
+-- Snapshots the creating key's organization at submission time, like team_id,
+-- so CheckBatchCost can bill organization spend hours later without re-resolving
+ALTER TABLE "LiteLLM_ManagedObjectTable" ADD COLUMN IF NOT EXISTS "org_id" TEXT;
diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma
index 7604ceadf7a..e386446d5bd 100644
--- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma
+++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma
@@ -1034,6 +1034,7 @@ model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use t
   created_at DateTime @default(now())
   created_by String?
   team_id String?
+  org_id String? // creating key's organization at submission time; CheckBatchCost bills org spend against it
   api_key String?
   request_tags Json? @default("[]")
   updated_at DateTime @updatedAt
diff --git a/litellm/models/managed_files.py b/litellm/models/managed_files.py
index 23d70ef5c48..c90f9b535ea 100644
--- a/litellm/models/managed_files.py
+++ b/litellm/models/managed_files.py
@@ -32,6 +32,7 @@ class LiteLLM_ManagedObjectTable(LiteLLMPydanticObjectBase):
     file_object: LiteLLMBatch | LiteLLMFineTuningJob | ResponsesAPIResponse
     created_by: str | None = None
     team_id: str | None = None
+    org_id: str | None = None
 
 
 class LiteLLM_ManagedVectorStoreTable(LiteLLMPydanticObjectBase):
diff --git a/schema.prisma b/schema.prisma
index 7604ceadf7a..e386446d5bd 100644
--- a/schema.prisma
+++ b/schema.prisma
@@ -1034,6 +1034,7 @@ model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use t
   created_at DateTime @default(now())
   created_by String?
   team_id String?
+  org_id String? // creating key's organization at submission time; CheckBatchCost bills org spend against it
   api_key String?
   request_tags Json? @default("[]")
   updated_at DateTime @updatedAt
diff --git a/tests/proxy_unit_tests/test_check_batch_cost.py b/tests/proxy_unit_tests/test_check_batch_cost.py
index 9ee0a36f00b..92b946e19b1 100644
--- a/tests/proxy_unit_tests/test_check_batch_cost.py
+++ b/tests/proxy_unit_tests/test_check_batch_cost.py
@@ -2553,10 +2553,27 @@ class TestBatchCostAttribution:
 
         assert metadata["user_api_key_alias"] == "prod-key"
 
+    @pytest.mark.asyncio
+    async def test_org_id_snapshotted_on_the_row_wins(self):
+        """The org_id column captures the creating key's organization at submission time,
+        like team_id, so a key later moved to another org still bills the original one."""
+        from types import SimpleNamespace
+
+        instance = self._instance(
+            key_row=SimpleNamespace(key_alias="prod-key", org_id="org-moved-to"),
+            team_row=SimpleNamespace(team_alias="Team Alpha", organization_id="org-team"),
+        )
+
+        metadata = await instance._build_creator_attribution_metadata(
+            self._job(org_id="org-at-creation"), "batch-1"
+        )
+
+        assert metadata["user_api_key_org_id"] == "org-at-creation"
+
     @pytest.mark.asyncio
     async def test_org_id_comes_from_the_creating_key(self):
-        """The spend update writer increments organization spend from user_api_key_org_id,
-        so an org-scoped key's batch cost must carry the key's org id."""
+        """The spend update writer increments organization spend from user_api_key_org_id.
+        A legacy row without the org_id column falls back to the creating key's org."""
         from types import SimpleNamespace
 
         instance = self._instance(
diff --git a/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py b/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py
index ebd33aa2e53..63884e0a779 100644
--- a/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py
+++ b/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py
@@ -390,7 +390,7 @@ async def test_store_unified_object_id_persists_key_and_tags_on_create():
     """Regression (spend loss): the batch create persists the creating key hash and tags so
     CheckBatchCost can write an attributed spend row instead of a blank one the DB drops."""
     instance, store = _in_memory_managed_files()
-    creator = UserAPIKeyAuth(user_id="alice", team_id="team-alpha", api_key="hash-alice")
+    creator = UserAPIKeyAuth(user_id="alice", team_id="team-alpha", api_key="hash-alice", org_id="org-acme")
 
     await instance.store_unified_object_id(
         unified_object_id="unified-b",
@@ -407,6 +407,7 @@ async def test_store_unified_object_id_persists_key_and_tags_on_create():
     assert row["api_key"] == "hash-alice"
     assert row["created_by"] == "alice"
     assert row["team_id"] == "team-alpha"
+    assert row["org_id"] == "org-acme"
     assert row["request_tags"].data == ["env:prod"]
 
 
@@ -471,6 +472,7 @@ async def test_store_unified_object_id_attribution_columns_are_write_once():
     upsert_data = instance.prisma_client.db.litellm_managedobjecttable.upsert.call_args.kwargs["data"]
     assert "api_key" not in upsert_data["update"]
     assert "request_tags" not in upsert_data["update"]
+    assert "org_id" not in upsert_data["update"]
 
 
 @pytest.mark.asyncio

From 9859d1e64ca1e6930d2f3e440a51222e834c7a5f Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Thu, 3 Sep 2026 22:36:28 +0000
Subject: [PATCH 31/85] chore: sync schema.prisma copies from root

---
 litellm/proxy/schema.prisma | 1 +
 1 file changed, 1 insertion(+)

diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma
index 7604ceadf7a..e386446d5bd 100644
--- a/litellm/proxy/schema.prisma
+++ b/litellm/proxy/schema.prisma
@@ -1034,6 +1034,7 @@ model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use t
   created_at DateTime @default(now())
   created_by String?
   team_id String?
+  org_id String? // creating key's organization at submission time; CheckBatchCost bills org spend against it
   api_key String?
   request_tags Json? @default("[]")
   updated_at DateTime @updatedAt

From 0d7976116eff0a9090f846e0066067bd867e67fb Mon Sep 17 00:00:00 2001
From: mubashir1osmani <mubashir.osmani777@gmail.com>
Date: Thu, 3 Sep 2026 18:51:18 -0400
Subject: [PATCH 32/85] fix(batches): resolve team-scoped keys' org at creation
 for the snapshot

---
 .../proxy/hooks/managed_files.py              | 20 +++++++++++++-
 ..._batch_update_db_managed_output_file_id.py | 27 +++++++++++++++++++
 2 files changed, 46 insertions(+), 1 deletion(-)

diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py
index 4f0ca787280..2ed446ca44e 100644
--- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py
+++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py
@@ -277,6 +277,24 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         )
         verbose_logger.debug(f"LiteLLM Managed File object with id={file_id} stored in db: {result}")
 
+    async def _resolve_creator_org_id(self, user_api_key_dict: UserAPIKeyAuth) -> Optional[str]:
+        """Organization to snapshot on the managed object row, like team_id. A key that
+        belongs to an org only through its team carries no org_id on the auth object, so
+        resolve the team's organization at creation time; costing then bills the org the
+        batch was submitted under even if the key or team moves before it completes."""
+        if user_api_key_dict.org_id:
+            return user_api_key_dict.org_id
+        if not user_api_key_dict.team_id:
+            return None
+        try:
+            team_row = await self.prisma_client.db.litellm_teamtable.find_unique(
+                where={"team_id": user_api_key_dict.team_id}
+            )
+            return getattr(team_row, "organization_id", None) if team_row is not None else None
+        except Exception as e:
+            verbose_logger.warning(f"could not resolve org for managed object attribution: {e}")
+            return None
+
     async def store_unified_object_id(
         self,
         unified_object_id: str,
@@ -349,7 +367,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                     "file_purpose": file_purpose,
                     "created_by": resolve_resource_owner_id(user_api_key_dict),
                     "team_id": user_api_key_dict.team_id,
-                    "org_id": user_api_key_dict.org_id,
+                    "org_id": await self._resolve_creator_org_id(user_api_key_dict),
                     "updated_by": user_api_key_dict.user_id,
                     "status": file_object.status,
                     **attribution_columns,
diff --git a/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py b/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py
index 63884e0a779..b5865ab4a13 100644
--- a/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py
+++ b/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py
@@ -375,6 +375,7 @@ def _in_memory_managed_files():
     table.upsert = AsyncMock(side_effect=_upsert)
     prisma = MagicMock()
     prisma.db.litellm_managedobjecttable = table
+    prisma.db.litellm_teamtable.find_unique = AsyncMock(return_value=None)
 
     cache = MagicMock()
     cache.async_set_cache = AsyncMock()
@@ -411,6 +412,32 @@ async def test_store_unified_object_id_persists_key_and_tags_on_create():
     assert row["request_tags"].data == ["env:prod"]
 
 
+@pytest.mark.asyncio
+async def test_store_unified_object_id_resolves_org_through_the_team():
+    """Most keys belong to an org only through their team, so the auth object carries no
+    org_id. The create resolves the team's organization so org spend is snapshotted at
+    submission time instead of never being billed."""
+    from types import SimpleNamespace
+
+    instance, store = _in_memory_managed_files()
+    instance.prisma_client.db.litellm_teamtable.find_unique = AsyncMock(
+        return_value=SimpleNamespace(organization_id="org-via-team")
+    )
+    creator = UserAPIKeyAuth(user_id="alice", team_id="team-alpha", api_key="hash-alice")
+
+    await instance.store_unified_object_id(
+        unified_object_id="unified-b",
+        file_object=_build_batch_response(batch_id="b", status="validating"),
+        litellm_parent_otel_span=None,
+        model_object_id="b",
+        file_purpose="batch",
+        user_api_key_dict=creator,
+        persist_attribution=True,
+    )
+
+    assert store["unified-b"]["org_id"] == "org-via-team"
+
+
 @pytest.mark.asyncio
 async def test_store_unified_object_id_omits_key_and_tags_without_persist_attribution():
     """Regression (spend redirect): a caller that is not the batch create (a poll, or the

From 35371a34c1c873a0afafe9dffc1873fcee42928e Mon Sep 17 00:00:00 2001
From: mubashir1osmani <mubashir.osmani777@gmail.com>
Date: Thu, 3 Sep 2026 19:09:45 -0400
Subject: [PATCH 33/85] fix(batches): keep team org attribution when the key
 lookup fails

---
 .../proxy/common_utils/check_batch_cost.py    | 24 ++++++++++++-------
 .../proxy_unit_tests/test_check_batch_cost.py | 17 +++++++++++++
 2 files changed, 32 insertions(+), 9 deletions(-)

diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
index 595a8d04bed..bd7f3c93ff3 100644
--- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
+++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
@@ -151,8 +151,8 @@ class CheckBatchCost:
             return org_id
         api_key = getattr(job, "api_key", None)
         team_id = getattr(job, "team_id", None)
-        try:
-            if api_key:
+        if api_key:
+            try:
                 key_row: prisma_models.LiteLLM_VerificationToken | None = (
                     await self.prisma_client.db.litellm_verificationtoken.find_unique(
                         where={"token": api_key}
@@ -161,16 +161,22 @@ class CheckBatchCost:
                 key_org_id = getattr(key_row, "org_id", None) if key_row is not None else None
                 if key_org_id:
                     return key_org_id
-            if team_id:
-                team_row: prisma_models.LiteLLM_TeamTable | None = (
-                    await self.prisma_client.db.litellm_teamtable.find_unique(
-                        where={"team_id": team_id}
-                    )
+            except Exception as e:
+                verbose_proxy_logger.error(
+                    f"CheckBatchCost: could not resolve the key's org for batch {batch_id}, "
+                    f"still trying the team's: {e}"
                 )
-                return getattr(team_row, "organization_id", None) if team_row is not None else None
+        if not team_id:
             return None
+        try:
+            team_row: prisma_models.LiteLLM_TeamTable | None = (
+                await self.prisma_client.db.litellm_teamtable.find_unique(
+                    where={"team_id": team_id}
+                )
+            )
+            return getattr(team_row, "organization_id", None) if team_row is not None else None
         except Exception as e:
-            verbose_proxy_logger.error(f"CheckBatchCost: could not resolve org for batch {batch_id}: {e}")
+            verbose_proxy_logger.error(f"CheckBatchCost: could not resolve the team's org for batch {batch_id}: {e}")
             return None
 
     async def _build_creator_attribution_metadata(
diff --git a/tests/proxy_unit_tests/test_check_batch_cost.py b/tests/proxy_unit_tests/test_check_batch_cost.py
index 92b946e19b1..59dfb67c486 100644
--- a/tests/proxy_unit_tests/test_check_batch_cost.py
+++ b/tests/proxy_unit_tests/test_check_batch_cost.py
@@ -2600,6 +2600,23 @@ class TestBatchCostAttribution:
 
         assert metadata["user_api_key_org_id"] == "org-team"
 
+    @pytest.mark.asyncio
+    async def test_key_lookup_failure_still_bills_the_team_org(self):
+        """A key-table error while resolving a legacy row's org must not drop the team's
+        organization: the two lookups fail independently, so org spend still lands."""
+        from types import SimpleNamespace
+
+        instance = self._instance(
+            team_row=SimpleNamespace(team_alias="Team Alpha", organization_id="org-team"),
+        )
+        instance.prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(
+            side_effect=Exception("db down")
+        )
+
+        metadata = await instance._build_creator_attribution_metadata(self._job(), "batch-1")
+
+        assert metadata["user_api_key_org_id"] == "org-team"
+
     @pytest.mark.asyncio
     async def test_no_org_leaves_the_key_unset(self):
         """Without any org the key is absent entirely, so the spend writer's org update

From 8debf8294cbe5e8d66d470b8dda1254668aa65df Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Fri, 4 Sep 2026 12:42:39 -0700
Subject: [PATCH 34/85] fix(batches): resolve a legacy row's org from the key's
 organization_id and keep the Batch label on grouped rows

---
 .../proxy/common_utils/check_batch_cost.py          |  2 +-
 tests/proxy_unit_tests/test_check_batch_cost.py     |  8 ++++----
 .../view_logs/RequestLogsTableColumns.test.tsx      | 13 +++++++++++++
 .../view_logs/RequestLogsTableColumns.tsx           |  2 +-
 .../src/components/view_logs/TypeBadges.test.tsx    |  7 +------
 .../src/components/view_logs/TypeBadges.tsx         |  4 ++--
 6 files changed, 22 insertions(+), 14 deletions(-)

diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
index ce80ba6493f..6b3ccfd3694 100644
--- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
+++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
@@ -158,7 +158,7 @@ class CheckBatchCost:
                         where={"token": api_key}
                     )
                 )
-                key_org_id = getattr(key_row, "org_id", None) if key_row is not None else None
+                key_org_id = getattr(key_row, "organization_id", None) if key_row is not None else None
                 if key_org_id:
                     return key_org_id
             except Exception as e:
diff --git a/tests/proxy_unit_tests/test_check_batch_cost.py b/tests/proxy_unit_tests/test_check_batch_cost.py
index 9ce91a8aee4..f9fb782b052 100644
--- a/tests/proxy_unit_tests/test_check_batch_cost.py
+++ b/tests/proxy_unit_tests/test_check_batch_cost.py
@@ -2561,7 +2561,7 @@ class TestBatchCostAttribution:
         from types import SimpleNamespace
 
         instance = self._instance(
-            key_row=SimpleNamespace(key_alias="prod-key", org_id="org-moved-to"),
+            key_row=SimpleNamespace(key_alias="prod-key", organization_id="org-moved-to"),
             team_row=SimpleNamespace(team_alias="Team Alpha", organization_id="org-team"),
         )
 
@@ -2578,7 +2578,7 @@ class TestBatchCostAttribution:
         from types import SimpleNamespace
 
         instance = self._instance(
-            key_row=SimpleNamespace(key_alias="prod-key", org_id="org-42"),
+            key_row=SimpleNamespace(key_alias="prod-key", organization_id="org-42"),
             team_row=SimpleNamespace(team_alias="Team Alpha", organization_id="org-team"),
         )
 
@@ -2593,7 +2593,7 @@ class TestBatchCostAttribution:
         from types import SimpleNamespace
 
         instance = self._instance(
-            key_row=SimpleNamespace(key_alias="prod-key", org_id=None),
+            key_row=SimpleNamespace(key_alias="prod-key", organization_id=None),
             team_row=SimpleNamespace(team_alias="Team Alpha", organization_id="org-team"),
         )
 
@@ -2625,7 +2625,7 @@ class TestBatchCostAttribution:
         from types import SimpleNamespace
 
         instance = self._instance(
-            key_row=SimpleNamespace(key_alias="prod-key", org_id=None),
+            key_row=SimpleNamespace(key_alias="prod-key", organization_id=None),
             team_row=SimpleNamespace(team_alias="Team Alpha", organization_id=None),
         )
 
diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx
index 3e8e522afe1..9f0e659cb1f 100644
--- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.test.tsx
@@ -141,6 +141,19 @@ describe("Type column", () => {
     expect(screen.getByText("Batch")).toBeInTheDocument();
     expect(screen.queryByText("LLM")).not.toBeInTheDocument();
   });
+
+  it("keeps the Batch label on the grouped create-plus-cost session instead of a row count", () => {
+    const groupedCostRow: Partial<LogEntry> = {
+      request_id: "batch_1_batch_cost",
+      call_type: "aretrieve_batch",
+      session_id: "batch_1",
+      session_total_count: 2,
+    };
+    renderRows([logEntry(groupedCostRow)]);
+
+    expect(screen.getByText("Batch")).toBeInTheDocument();
+    expect(screen.queryByText("2")).not.toBeInTheDocument();
+  });
 });
 
 describe("batch rows", () => {
diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx
index a444acb9517..1ec1087a1a4 100644
--- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsTableColumns.tsx
@@ -65,7 +65,7 @@ export const getRequestLogsTableColumns = ({
       const sessionMcpCount = log.mcp_tool_call_count ?? (isMcp ? sessionCount : 0);
 
       if (isBatchCallType(log.call_type)) {
-        return <BatchBadge count={sessionCount > 1 ? sessionCount : undefined} />;
+        return <BatchBadge />;
       }
       if (sessionCount <= 1) {
         if (isMcp) return <McpBadge />;
diff --git a/ui/litellm-dashboard/src/components/view_logs/TypeBadges.test.tsx b/ui/litellm-dashboard/src/components/view_logs/TypeBadges.test.tsx
index 8a964ea2124..9a3b53685ca 100644
--- a/ui/litellm-dashboard/src/components/view_logs/TypeBadges.test.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/TypeBadges.test.tsx
@@ -45,14 +45,9 @@ describe("TypeBadges", () => {
   });
 
   describe("BatchBadge", () => {
-    it("should render with default 'Batch' text when no count is provided", () => {
+    it("should render 'Batch'", () => {
       render(<BatchBadge />);
       expect(screen.getByText("Batch")).toBeInTheDocument();
     });
-
-    it("should render the count when provided", () => {
-      render(<BatchBadge count={4} />);
-      expect(screen.getByText("4")).toBeInTheDocument();
-    });
   });
 });
diff --git a/ui/litellm-dashboard/src/components/view_logs/TypeBadges.tsx b/ui/litellm-dashboard/src/components/view_logs/TypeBadges.tsx
index 84079848487..db64bfbbe73 100644
--- a/ui/litellm-dashboard/src/components/view_logs/TypeBadges.tsx
+++ b/ui/litellm-dashboard/src/components/view_logs/TypeBadges.tsx
@@ -97,9 +97,9 @@ export const AgentBadge = ({ count }: { count?: number }) => (
   </span>
 );
 
-export const BatchBadge = ({ count }: { count?: number }) => (
+export const BatchBadge = () => (
   <span className="inline-flex items-center gap-1 px-2 py-0.5 bg-teal-50 text-teal-700 border border-teal-200 rounded-full text-[11px] font-medium whitespace-nowrap dark:bg-teal-950 dark:text-teal-300 dark:border-teal-800">
     <LayersIcon />
-    {count != null ? count : "Batch"}
+    Batch
   </span>
 );

From 217cb7da653e025f7c6d10eaae8e291b826fd76f Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Fri, 4 Sep 2026 20:30:33 -0700
Subject: [PATCH 35/85] fix(managed_files): resolve the creator org through the
 cached team lookup

Batch creation snapshotted the team's organization with a direct
litellm_teamtable query on every create. Go through get_team_object
instead, which serves the team auth already cached and only falls back
to the database when the team was never cached.
---
 .../proxy/hooks/managed_files.py              | 13 +++-
 ..._batch_update_db_managed_output_file_id.py | 68 ++++++++++++++-----
 2 files changed, 61 insertions(+), 20 deletions(-)

diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py
index 850e5aadf0e..68782c5516d 100644
--- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py
+++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py
@@ -288,11 +288,18 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             return user_api_key_dict.org_id
         if not user_api_key_dict.team_id:
             return None
+        from litellm.proxy.auth.auth_checks import get_team_object
+        from litellm.proxy.proxy_server import proxy_logging_obj, user_api_key_cache
+
         try:
-            team_row = await self.prisma_client.db.litellm_teamtable.find_unique(
-                where={"team_id": user_api_key_dict.team_id}
+            team: Final = await get_team_object(
+                team_id=user_api_key_dict.team_id,
+                prisma_client=self.prisma_client,
+                user_api_key_cache=user_api_key_cache,
+                parent_otel_span=user_api_key_dict.parent_otel_span,
+                proxy_logging_obj=proxy_logging_obj,
             )
-            return getattr(team_row, "organization_id", None) if team_row is not None else None
+            return team.organization_id
         except Exception as e:
             verbose_logger.warning(f"could not resolve org for managed object attribution: {e}")
             return None
diff --git a/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py b/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py
index b5865ab4a13..d3e668b8987 100644
--- a/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py
+++ b/tests/test_litellm/enterprise/proxy/test_batch_update_db_managed_output_file_id.py
@@ -413,29 +413,63 @@ async def test_store_unified_object_id_persists_key_and_tags_on_create():
 
 
 @pytest.mark.asyncio
-async def test_store_unified_object_id_resolves_org_through_the_team():
+async def test_store_unified_object_id_resolves_org_through_the_cached_team():
     """Most keys belong to an org only through their team, so the auth object carries no
-    org_id. The create resolves the team's organization so org spend is snapshotted at
-    submission time instead of never being billed."""
-    from types import SimpleNamespace
+    org_id. The create reads the team that auth already cached, so org spend is snapshotted
+    at submission time without a database query in the request path."""
+    from litellm.models.team import LiteLLM_TeamTableCachedObj
+    from litellm.proxy.proxy_server import user_api_key_cache
+
+    instance, store = _in_memory_managed_files()
+    creator = UserAPIKeyAuth(user_id="alice", team_id="team-cached", api_key="hash-alice")
+    await user_api_key_cache.async_set_cache(
+        key="team_id:team-cached",
+        value=LiteLLM_TeamTableCachedObj(team_id="team-cached", organization_id="org-via-team"),
+        model_type=LiteLLM_TeamTableCachedObj,
+    )
+    try:
+        await instance.store_unified_object_id(
+            unified_object_id="unified-b",
+            file_object=_build_batch_response(batch_id="b", status="validating"),
+            litellm_parent_otel_span=None,
+            model_object_id="b",
+            file_purpose="batch",
+            user_api_key_dict=creator,
+            persist_attribution=True,
+        )
+    finally:
+        user_api_key_cache.delete_cache(key="team_id:team-cached")
+
+    assert store["unified-b"]["org_id"] == "org-via-team"
+    instance.prisma_client.db.litellm_teamtable.find_unique.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_store_unified_object_id_resolves_org_from_the_db_when_the_team_is_not_cached():
+    """A team no request has run under yet is absent from the auth cache; its organization
+    still comes back from the table so the org is billed rather than dropped."""
+    from litellm.models.team import LiteLLM_TeamTable
+    from litellm.proxy.proxy_server import user_api_key_cache
 
     instance, store = _in_memory_managed_files()
     instance.prisma_client.db.litellm_teamtable.find_unique = AsyncMock(
-        return_value=SimpleNamespace(organization_id="org-via-team")
+        return_value=LiteLLM_TeamTable(team_id="team-uncached", organization_id="org-via-db")
     )
-    creator = UserAPIKeyAuth(user_id="alice", team_id="team-alpha", api_key="hash-alice")
+    creator = UserAPIKeyAuth(user_id="alice", team_id="team-uncached", api_key="hash-alice")
+    try:
+        await instance.store_unified_object_id(
+            unified_object_id="unified-b",
+            file_object=_build_batch_response(batch_id="b", status="validating"),
+            litellm_parent_otel_span=None,
+            model_object_id="b",
+            file_purpose="batch",
+            user_api_key_dict=creator,
+            persist_attribution=True,
+        )
+    finally:
+        user_api_key_cache.delete_cache(key="team_id:team-uncached")
 
-    await instance.store_unified_object_id(
-        unified_object_id="unified-b",
-        file_object=_build_batch_response(batch_id="b", status="validating"),
-        litellm_parent_otel_span=None,
-        model_object_id="b",
-        file_purpose="batch",
-        user_api_key_dict=creator,
-        persist_attribution=True,
-    )
-
-    assert store["unified-b"]["org_id"] == "org-via-team"
+    assert store["unified-b"]["org_id"] == "org-via-db"
 
 
 @pytest.mark.asyncio

From 814c151b02b41e4fb333f4f505203d7db7c4343a Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 5 Sep 2026 01:35:09 -0700
Subject: [PATCH 36/85] fix(batches): mask api base credentials on batch cost
 rows

---
 .../proxy/common_utils/check_batch_cost.py    |  4 +-
 litellm/litellm_core_utils/litellm_logging.py | 16 ++--
 .../proxy_unit_tests/test_check_batch_cost.py | 84 +++++++++++++++++++
 3 files changed, 94 insertions(+), 10 deletions(-)

diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
index 6b3ccfd3694..64a86387c39 100644
--- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
+++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
@@ -685,7 +685,7 @@ class CheckBatchCost:
         from litellm.files.main import afile_content
         from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
         from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
-        from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info
+        from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info, mask_api_base_credentials
         from litellm.proxy.openai_files_endpoints.common_utils import (
             _is_base64_encoded_unified_file_id,
         )
@@ -858,7 +858,7 @@ class CheckBatchCost:
                         "user-agent": CHECK_BATCH_COST_USER_AGENT,
                     }
                 },
-                **({"api_base": deployment_api_base} if deployment_api_base else {}),
+                **({"api_base": mask_api_base_credentials(deployment_api_base)} if deployment_api_base else {}),
                 "metadata": {
                     **(await self._build_creator_attribution_metadata(job, batch_id)),
                     # spend logs read the deployment identity off these metadata keys, so
diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py
index 73c06b891fb..7e800139228 100644
--- a/litellm/litellm_core_utils/litellm_logging.py
+++ b/litellm/litellm_core_utils/litellm_logging.py
@@ -419,6 +419,13 @@ def _provider_response_id(source: object) -> str | None:
     return candidate if isinstance(candidate, str) and candidate else None
 
 
+def mask_api_base_credentials(api_base: str) -> str:
+    if "key=" not in api_base:
+        return api_base
+    key_end: Final = api_base.find("key=") + 4
+    return api_base[:key_end] + "*" * 5 + api_base[-4:]
+
+
 class Logging(LiteLLMLoggingBaseClass):
     global \
         supabaseClient, \
@@ -1160,14 +1167,7 @@ class Logging(LiteLLMLoggingBaseClass):
         return data
 
     def _get_masked_api_base(self, api_base: str) -> str:
-        if "key=" in api_base:
-            # Find the position of "key=" in the string
-            key_index: Final = api_base.find("key=") + 4
-            # Mask the last 5 characters after "key="
-            masked_api_base = api_base[:key_index] + "*" * 5 + api_base[-4:]
-        else:
-            masked_api_base = api_base
-        return str(masked_api_base)
+        return str(mask_api_base_credentials(api_base))
 
     def _pre_call(self, input, api_key, model=None, additional_args={}):
         """
diff --git a/tests/proxy_unit_tests/test_check_batch_cost.py b/tests/proxy_unit_tests/test_check_batch_cost.py
index f9fb782b052..36177b44930 100644
--- a/tests/proxy_unit_tests/test_check_batch_cost.py
+++ b/tests/proxy_unit_tests/test_check_batch_cost.py
@@ -583,6 +583,90 @@ class TestCheckBatchCost:
         assert passed_model_info["input_cost_per_token_batches"] == 2e-06
         assert passed_model_info["output_cost_per_token_batches"] == 4e-06
 
+    @pytest.mark.asyncio
+    async def test_poller_masks_api_base_credentials_before_logging(
+        self, check_batch_cost_instance, mock_prisma_client, mock_llm_router
+    ):
+        """Request rows mask `key=` query credentials out of api_base before it is
+        logged, but the poller skips that pre-call step, so an unmasked deployment
+        api_base would land verbatim on the batch cost row: regression test for the
+        poller masking the same way.
+        """
+        import base64
+        from unittest.mock import patch
+
+        import httpx
+        import respx
+
+        from litellm.litellm_core_utils.litellm_logging import Logging
+
+        mock_prisma_client.db.litellm_managedobjecttable.update_many = AsyncMock(return_value=1)
+        mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock()
+        mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None)
+
+        mock_job = MagicMock()
+        mock_job.id = "job-masked-api-base-1"
+        mock_job.unified_object_id = base64.urlsafe_b64encode(
+            b"litellm_proxy;model_id:model-123;llm_batch_id:batch-456"
+        ).decode()
+        mock_job.created_by = "user-1"
+        mock_prisma_client.db.litellm_managedobjecttable.find_many = AsyncMock(return_value=[mock_job])
+
+        mock_response = MagicMock()
+        mock_response.status = "completed"
+        mock_response.output_file_id = "file-output-123"
+        mock_response.error_file_id = None
+        mock_response.model_dump_json.return_value = '{"id":"batch-1","status":"completed"}'
+        mock_llm_router.aretrieve_batch = AsyncMock(return_value=mock_response)
+        mock_llm_router.get_deployment_credentials_with_provider = MagicMock(return_value={"api_key": "sk-test"})
+
+        mock_deployment = MagicMock()
+        mock_deployment.litellm_params.custom_llm_provider = "openai"
+        mock_deployment.litellm_params.model = "gpt-5.4-mini"
+        mock_deployment.litellm_params.api_base = "https://gateway.example.com/v1?key=AIzaSyVERYSECRET7890"
+        mock_deployment.model_info.model_dump.return_value = {}
+        mock_llm_router.get_deployment = MagicMock(return_value=mock_deployment)
+
+        output_line = json.dumps(
+            {
+                "custom_id": "req-1",
+                "response": {
+                    "status_code": 200,
+                    "body": {
+                        "id": "chatcmpl-1",
+                        "object": "chat.completion",
+                        "model": "gpt-5.4-mini",
+                        "choices": [
+                            {
+                                "index": 0,
+                                "message": {"role": "assistant", "content": "hi"},
+                                "finish_reason": "stop",
+                            }
+                        ],
+                        "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
+                    },
+                },
+                "error": None,
+            }
+        )
+
+        with (
+            respx.mock(assert_all_called=True) as provider,
+            patch.object(  # test-quality-ok: the poller builds Logging inline, the only seam to the row it logs
+                Logging, "async_success_handler", autospec=True
+            ) as success_handler,
+        ):
+            provider.get("https://api.openai.com/v1/files/file-output-123/content").mock(
+                return_value=httpx.Response(200, content=f"{output_line}\n".encode())
+            )
+            await check_batch_cost_instance.check_batch_cost()
+
+        cost_row_calls = [call for call in success_handler.await_args_list if "batch_cost" in call.kwargs]
+        assert len(cost_row_calls) == 1
+        logged_api_base = cost_row_calls[0].args[0].litellm_params["api_base"]
+        assert logged_api_base == "https://gateway.example.com/v1?key=*****7890"
+        assert "VERYSECRET" not in logged_api_base
+
     @pytest.mark.asyncio
     async def test_primary_path_completion_update_includes_batch_processed(
         self, check_batch_cost_instance, mock_prisma_client, mock_llm_router

From 95d721ffe60e6fa8409e2ff3e57ab9a85093fcc7 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 5 Sep 2026 02:06:40 -0700
Subject: [PATCH 37/85] refactor(batches): drop docstrings restating the org
 fallbacks

---
 .../litellm_enterprise/proxy/common_utils/check_batch_cost.py | 3 ---
 enterprise/litellm_enterprise/proxy/hooks/managed_files.py    | 4 ----
 2 files changed, 7 deletions(-)

diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
index 64a86387c39..13e9e5093a8 100644
--- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
+++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py
@@ -143,9 +143,6 @@ class CheckBatchCost:
             return None
 
     async def _get_org_id(self, job: "LiteLLM_ManagedObjectTable", batch_id: str) -> str | None:
-        """Organization to bill the batch against, snapshotted on the row at creation
-        like team_id. Rows created before the org_id column existed carry None, so they
-        fall back to the creating key's org (or its team's) as resolved today."""
         org_id = getattr(job, "org_id", None)
         if org_id:
             return org_id
diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py
index 6474475f1bf..486904d0abe 100644
--- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py
+++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py
@@ -281,10 +281,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         verbose_logger.debug(f"LiteLLM Managed File object with id={file_id} stored in db: {result}")
 
     async def _resolve_creator_org_id(self, user_api_key_dict: UserAPIKeyAuth) -> Optional[str]:
-        """Organization to snapshot on the managed object row, like team_id. A key that
-        belongs to an org only through its team carries no org_id on the auth object, so
-        resolve the team's organization at creation time; costing then bills the org the
-        batch was submitted under even if the key or team moves before it completes."""
         if user_api_key_dict.org_id:
             return user_api_key_dict.org_id
         if not user_api_key_dict.team_id:

From 48cc4efca38b46e1f561da4728954d06bbe223c7 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 5 Sep 2026 20:37:17 -0700
Subject: [PATCH 38/85] fix(least-busy): share in-flight request counts across
 proxy workers

Least-busy kept one dict of in-flight counts per model group in the
router cache, which reads in-memory first, so every worker and replica
routed on its own stale copy and each write overwrote the shared value.

Counts now live in one key per deployment, incremented and read through
Redis when the router has a Redis cache, and in the process-local cache
otherwise.
---
 basedpyright-code-budget.json                 |  12 +-
 litellm/caching/redis_cache.py                |   4 +-
 litellm/router_strategy/least_busy.py         | 372 +++++++++---------
 ruff-strict-budget.json                       |  10 +-
 .../local_testing/test_least_busy_routing.py  |  46 ++-
 .../router_strategy/test_least_busy.py        | 151 +++++++
 type-discipline-budget.json                   |   8 +-
 7 files changed, 376 insertions(+), 227 deletions(-)
 create mode 100644 tests/test_litellm/router_strategy/test_least_busy.py

diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json
index 0b0a61192e6..c21dc76743a 100644
--- a/basedpyright-code-budget.json
+++ b/basedpyright-code-budget.json
@@ -54,10 +54,10 @@
     "limit": 0
   },
   "reportMissingParameterType": {
-    "limit": 5570
+    "limit": 5551
   },
   "reportMissingTypeArgument": {
-    "limit": 15281
+    "limit": 15277
   },
   "reportMissingTypeStubs": {
     "limit": 40
@@ -99,19 +99,19 @@
     "limit": 0
   },
   "reportUnknownArgumentType": {
-    "limit": 44358
+    "limit": 44357
   },
   "reportUnknownLambdaType": {
     "limit": 109
   },
   "reportUnknownMemberType": {
-    "limit": 38271
+    "limit": 38240
   },
   "reportUnknownParameterType": {
-    "limit": 19584
+    "limit": 19558
   },
   "reportUnknownVariableType": {
-    "limit": 29814
+    "limit": 29781
   },
   "reportUnnecessaryCast": {
     "limit": 110
diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py
index aaee7188d86..d4c47914394 100644
--- a/litellm/caching/redis_cache.py
+++ b/litellm/caching/redis_cache.py
@@ -680,7 +680,7 @@ class RedisCache(BaseCache):
             # NON blocking - notify users Redis is throwing an exception
             print_verbose(f"litellm.caching.caching: set() - Got exception from REDIS : {e}")
 
-    def increment_cache(self, key, value: int, ttl: float | None = None, **kwargs) -> int:
+    def increment_cache(self, key, value: int, ttl: float | None = None, refresh_ttl: bool = False, **kwargs) -> int:
         _redis_client: Final = self.redis_client
         start_time = time.time()
         set_ttl: Final = self.get_ttl(ttl=ttl)
@@ -701,7 +701,7 @@ class RedisCache(BaseCache):
             if set_ttl is not None:
                 # check if key already has ttl, if not -> set ttl
                 start_time = time.time()
-                current_ttl: Final = _redis_client.ttl(key)
+                current_ttl: Final = -1 if refresh_ttl else _redis_client.ttl(key)
                 end_time = time.time()
                 _duration = end_time - start_time
                 self.service_logger_obj.service_success_hook(
diff --git a/litellm/router_strategy/least_busy.py b/litellm/router_strategy/least_busy.py
index 1433e8ba4d4..de6a4c4f59a 100644
--- a/litellm/router_strategy/least_busy.py
+++ b/litellm/router_strategy/least_busy.py
@@ -1,17 +1,96 @@
-#### What this does ####
-#   identifies least busy deployment
-#   How is this achieved?
-#   - Before each call, have the router print the state of requests {"deployment": "requests_in_flight"}
-#   - use litellm.input_callbacks to log when a request is just about to be made to a model - {"deployment-id": traffic}
-#   - use litellm.success + failure callbacks to log when a request completed
-#   - in get_available_deployment, for a given model group name -> pick based on traffic
-
-import random
+from collections.abc import Mapping, Sequence
 from typing import Final
 
+from pydantic import TypeAdapter, ValidationError
+from typing_extensions import ReadOnly, TypedDict
+
+from litellm._logging import verbose_router_logger
 from litellm.caching.caching import DualCache
 from litellm.integrations.custom_logger import CustomLogger
 
+IN_FLIGHT_COUNT_TTL_SECONDS: Final = 60 * 60
+
+
+class _ModelInfo(TypedDict, total=False):
+    id: ReadOnly[str | int | None]
+
+
+class _Metadata(TypedDict, total=False):
+    model_group: ReadOnly[str | None]
+
+
+class _LitellmParams(TypedDict, total=False):
+    metadata: ReadOnly[_Metadata | None]
+    model_info: ReadOnly[_ModelInfo | None]
+
+
+class _CallKwargs(TypedDict, total=False):
+    litellm_params: ReadOnly[_LitellmParams | None]
+
+
+class _DeploymentModelInfo(TypedDict):
+    id: ReadOnly[str | int]
+
+
+class _Deployment(TypedDict):
+    model_info: ReadOnly[_DeploymentModelInfo]
+
+
+_CALL_KWARGS: Final = TypeAdapter(_CallKwargs)
+_DEPLOYMENTS: Final = TypeAdapter(list[_Deployment])
+_REDIS_COUNTS: Final = TypeAdapter(dict[str, float | None])
+_MEMORY_COUNTS: Final = TypeAdapter(tuple[float | None, ...])
+
+
+def _request_count_key(model_group: str, deployment_id: str) -> str:
+    return f"{model_group}_request_count:{deployment_id}"
+
+
+def _deployment_ref(kwargs: Mapping[str, object]) -> tuple[str, str] | None:
+    try:
+        call: Final = _CALL_KWARGS.validate_python(kwargs)
+    except ValidationError:
+        return None
+    litellm_params: Final = call.get("litellm_params")
+    metadata: Final = litellm_params.get("metadata") if litellm_params else None
+    model_info: Final = litellm_params.get("model_info") if litellm_params else None
+    model_group: Final = metadata.get("model_group") if metadata else None
+    deployment_id: Final = model_info.get("id") if model_info else None
+    if model_group is None or deployment_id is None:
+        return None
+    return model_group, str(deployment_id)
+
+
+def _request_count_keys(model_group: str, healthy_deployments: Sequence[Mapping[str, object]]) -> tuple[str, ...]:
+    return tuple(
+        _request_count_key(model_group, str(deployment["model_info"]["id"]))
+        for deployment in _DEPLOYMENTS.validate_python(healthy_deployments)
+    )
+
+
+def _as_count(value: float | None) -> int:
+    return 0 if value is None else int(value)
+
+
+def _least_busy(
+    healthy_deployments: Sequence[Mapping[str, object]], counts: tuple[int, ...]
+) -> Mapping[str, object] | None:
+    if not healthy_deployments:
+        return None
+    return healthy_deployments[min(range(len(healthy_deployments)), key=lambda index: counts[index])]
+
+
+def _warn_unreadable(model_group: str, error: Exception) -> None:
+    verbose_router_logger.warning(
+        "least-busy routing could not read the in-flight counts for %s, treating every deployment as idle: %s",
+        model_group,
+        error,
+    )
+
+
+def _warn_unwritable(key: str, error: Exception) -> None:
+    verbose_router_logger.warning("least-busy routing could not update the in-flight count under %s: %s", key, error)
+
 
 class LeastBusyLoggingHandler(CustomLogger):
     test_flag: bool = False
@@ -21,194 +100,101 @@ class LeastBusyLoggingHandler(CustomLogger):
     def __init__(self, router_cache: DualCache):
         self.router_cache = router_cache
 
-    def log_pre_api_call(self, model, messages, kwargs):
-        """
-        Log when a model is being used.
+    def log_pre_api_call(self, model: str, messages: object, kwargs: Mapping[str, object]) -> None:
+        self._increment(kwargs, 1)
 
-        Caching based on model group.
-        """
-        try:
-            if kwargs["litellm_params"].get("metadata") is None:
-                pass
-            else:
-                model_group: Final = kwargs["litellm_params"]["metadata"].get("model_group", None)
-                id = kwargs["litellm_params"].get("model_info", {}).get("id", None)
-                if model_group is None or id is None:
-                    return
-                elif isinstance(id, int):
-                    id = str(id)
+    def log_success_event(
+        self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
+    ) -> None:
+        self._increment(kwargs, -1)
+        if self.test_flag:
+            self.logged_success += 1
 
-                request_count_api_key: Final = f"{model_group}_request_count"
-                # update cache
-                request_count_dict: Final = self.router_cache.get_cache(key=request_count_api_key) or {}
-                request_count_dict[id] = request_count_dict.get(id, 0) + 1
+    def log_failure_event(
+        self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
+    ) -> None:
+        self._increment(kwargs, -1)
+        if self.test_flag:
+            self.logged_failure += 1
 
-                self.router_cache.set_cache(key=request_count_api_key, value=request_count_dict)
-        except Exception:
-            pass
+    async def async_log_success_event(
+        self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
+    ) -> None:
+        await self._async_increment(kwargs, -1)
+        if self.test_flag:
+            self.logged_success += 1
 
-    def log_success_event(self, kwargs, response_obj, start_time, end_time):
-        try:
-            if kwargs["litellm_params"].get("metadata") is None:
-                pass
-            else:
-                model_group: Final = kwargs["litellm_params"]["metadata"].get("model_group", None)
-
-                id = kwargs["litellm_params"].get("model_info", {}).get("id", None)
-                if model_group is None or id is None:
-                    return
-                elif isinstance(id, int):
-                    id = str(id)
-
-                request_count_api_key: Final = f"{model_group}_request_count"
-                # decrement count in cache
-                request_count_dict: Final = self.router_cache.get_cache(key=request_count_api_key) or {}
-                request_count_value: Final[int | None] = request_count_dict.get(id, 0)
-                if request_count_value is None:
-                    return
-                request_count_dict[id] = request_count_value - 1
-                self.router_cache.set_cache(key=request_count_api_key, value=request_count_dict)
-
-                ### TESTING ###
-                if self.test_flag:
-                    self.logged_success += 1
-        except Exception:
-            pass
-
-    def log_failure_event(self, kwargs, response_obj, start_time, end_time):
-        try:
-            if kwargs["litellm_params"].get("metadata") is None:
-                pass
-            else:
-                model_group: Final = kwargs["litellm_params"]["metadata"].get("model_group", None)
-                id = kwargs["litellm_params"].get("model_info", {}).get("id", None)
-                if model_group is None or id is None:
-                    return
-                elif isinstance(id, int):
-                    id = str(id)
-
-                request_count_api_key: Final = f"{model_group}_request_count"
-                # decrement count in cache
-                request_count_dict: Final = self.router_cache.get_cache(key=request_count_api_key) or {}
-                request_count_value: Final[int | None] = request_count_dict.get(id, 0)
-                if request_count_value is None:
-                    return
-                request_count_dict[id] = request_count_value - 1
-                self.router_cache.set_cache(key=request_count_api_key, value=request_count_dict)
-
-                ### TESTING ###
-                if self.test_flag:
-                    self.logged_failure += 1
-        except Exception:
-            pass
-
-    async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
-        try:
-            if kwargs["litellm_params"].get("metadata") is None:
-                pass
-            else:
-                model_group: Final = kwargs["litellm_params"]["metadata"].get("model_group", None)
-
-                id = kwargs["litellm_params"].get("model_info", {}).get("id", None)
-                if model_group is None or id is None:
-                    return
-                elif isinstance(id, int):
-                    id = str(id)
-
-                request_count_api_key: Final = f"{model_group}_request_count"
-                # decrement count in cache
-                request_count_dict: Final = await self.router_cache.async_get_cache(key=request_count_api_key) or {}
-                request_count_value: Final[int | None] = request_count_dict.get(id, 0)
-                if request_count_value is None:
-                    return
-                request_count_dict[id] = request_count_value - 1
-                await self.router_cache.async_set_cache(key=request_count_api_key, value=request_count_dict)
-
-                ### TESTING ###
-                if self.test_flag:
-                    self.logged_success += 1
-        except Exception:
-            pass
-
-    async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
-        try:
-            if kwargs["litellm_params"].get("metadata") is None:
-                pass
-            else:
-                model_group: Final = kwargs["litellm_params"]["metadata"].get("model_group", None)
-                id = kwargs["litellm_params"].get("model_info", {}).get("id", None)
-                if model_group is None or id is None:
-                    return
-                elif isinstance(id, int):
-                    id = str(id)
-
-                request_count_api_key: Final = f"{model_group}_request_count"
-                # decrement count in cache
-                request_count_dict: Final = await self.router_cache.async_get_cache(key=request_count_api_key) or {}
-                request_count_value: Final[int | None] = request_count_dict.get(id, 0)
-                if request_count_value is None:
-                    return
-                request_count_dict[id] = request_count_value - 1
-                await self.router_cache.async_set_cache(key=request_count_api_key, value=request_count_dict)
-
-                ### TESTING ###
-                if self.test_flag:
-                    self.logged_failure += 1
-        except Exception:
-            pass
-
-    def _get_available_deployments(
-        self,
-        healthy_deployments: list,
-        all_deployments: dict,
-    ):
-        """
-        Helper to get deployments using least busy strategy
-        """
-        for d in healthy_deployments:
-            ## if healthy deployment not yet used
-            if d["model_info"]["id"] not in all_deployments:
-                all_deployments[d["model_info"]["id"]] = 0
-        # map deployment to id
-        # pick least busy deployment
-        min_traffic = float("inf")
-        min_deployment = None
-        for k, v in all_deployments.items():
-            if v < min_traffic:
-                min_traffic = v
-                min_deployment = k
-        if min_deployment is not None:
-            ## check if min deployment is a string, if so, cast it to int
-            for m in healthy_deployments:
-                if m["model_info"]["id"] == min_deployment:
-                    return m
-            min_deployment = random.choice(healthy_deployments)
-        else:
-            min_deployment = random.choice(healthy_deployments)
-        return min_deployment
+    async def async_log_failure_event(
+        self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
+    ) -> None:
+        await self._async_increment(kwargs, -1)
+        if self.test_flag:
+            self.logged_failure += 1
 
     def get_available_deployments(
-        self,
-        model_group: str,
-        healthy_deployments: list,
-    ):
-        """
-        Sync helper to get deployments using least busy strategy
-        """
-        request_count_api_key: Final = f"{model_group}_request_count"
-        all_deployments: Final = self.router_cache.get_cache(key=request_count_api_key) or {}
-        return self._get_available_deployments(
-            healthy_deployments=healthy_deployments,
-            all_deployments=all_deployments,
-        )
+        self, model_group: str, healthy_deployments: Sequence[Mapping[str, object]]
+    ) -> Mapping[str, object] | None:
+        keys: Final = _request_count_keys(model_group, healthy_deployments)
+        try:
+            counts: Final = tuple(_as_count(value) for value in self._read_counts(keys))
+        except Exception as e:
+            _warn_unreadable(model_group, e)
+            return _least_busy(healthy_deployments, (0,) * len(keys))
+        return _least_busy(healthy_deployments, counts)
 
-    async def async_get_available_deployments(self, model_group: str, healthy_deployments: list):
-        """
-        Async helper to get deployments using least busy strategy
-        """
-        request_count_api_key: Final = f"{model_group}_request_count"
-        all_deployments: Final = await self.router_cache.async_get_cache(key=request_count_api_key) or {}
-        return self._get_available_deployments(
-            healthy_deployments=healthy_deployments,
-            all_deployments=all_deployments,
-        )
+    async def async_get_available_deployments(
+        self, model_group: str, healthy_deployments: Sequence[Mapping[str, object]]
+    ) -> Mapping[str, object] | None:
+        keys: Final = _request_count_keys(model_group, healthy_deployments)
+        try:
+            counts: Final = tuple(_as_count(value) for value in await self._async_read_counts(keys))
+        except Exception as e:
+            _warn_unreadable(model_group, e)
+            return _least_busy(healthy_deployments, (0,) * len(keys))
+        return _least_busy(healthy_deployments, counts)
+
+    def _increment(self, kwargs: Mapping[str, object], delta: int) -> None:
+        ref: Final = _deployment_ref(kwargs)
+        if ref is None:
+            return
+        key: Final = _request_count_key(*ref)
+        redis_cache: Final = self.router_cache.redis_cache
+        try:
+            if redis_cache is None:
+                self.router_cache.increment_cache(key, delta, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS)
+            else:
+                redis_cache.increment_cache(key, delta, ttl=IN_FLIGHT_COUNT_TTL_SECONDS, refresh_ttl=True)
+        except Exception as e:
+            _warn_unwritable(key, e)
+
+    async def _async_increment(self, kwargs: Mapping[str, object], delta: int) -> None:
+        ref: Final = _deployment_ref(kwargs)
+        if ref is None:
+            return
+        key: Final = _request_count_key(*ref)
+        redis_cache: Final = self.router_cache.redis_cache
+        try:
+            if redis_cache is None:
+                await self.router_cache.async_increment_cache(
+                    key, delta, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS
+                )
+            else:
+                await redis_cache.async_increment(key, delta, ttl=IN_FLIGHT_COUNT_TTL_SECONDS, refresh_ttl=True)
+        except Exception as e:
+            _warn_unwritable(key, e)
+
+    def _read_counts(self, keys: tuple[str, ...]) -> tuple[float | None, ...]:
+        redis_cache: Final = self.router_cache.redis_cache
+        if redis_cache is None:
+            return _MEMORY_COUNTS.validate_python(self.router_cache.batch_get_cache(list(keys), local_only=True))
+        by_key: Final = _REDIS_COUNTS.validate_python(redis_cache.batch_get_cache(key_list=list(keys)))
+        return tuple(by_key.get(key) for key in keys)
+
+    async def _async_read_counts(self, keys: tuple[str, ...]) -> tuple[float | None, ...]:
+        redis_cache: Final = self.router_cache.redis_cache
+        if redis_cache is None:
+            return _MEMORY_COUNTS.validate_python(
+                await self.router_cache.async_batch_get_cache(list(keys), local_only=True)
+            )
+        by_key: Final = _REDIS_COUNTS.validate_python(await redis_cache.async_batch_get_cache(key_list=list(keys)))
+        return tuple(by_key.get(key) for key in keys)
diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json
index fd7b30bc314..ceff8c6e15c 100644
--- a/ruff-strict-budget.json
+++ b/ruff-strict-budget.json
@@ -1,6 +1,6 @@
 {
   "ANN001": {
-    "limit": 2956
+    "limit": 2937
   },
   "ANN002": {
     "limit": 71
@@ -9,10 +9,10 @@
     "limit": 806
   },
   "ANN201": {
-    "limit": 1979
+    "limit": 1972
   },
   "ANN202": {
-    "limit": 831
+    "limit": 830
   },
   "ANN204": {
     "limit": 683
@@ -57,7 +57,7 @@
     "limit": 3
   },
   "BLE001": {
-    "limit": 2916
+    "limit": 2915
   },
   "C401": {
     "limit": 8
@@ -189,7 +189,7 @@
     "limit": 0
   },
   "S110": {
-    "limit": 217
+    "limit": 212
   },
   "S112": {
     "limit": 22
diff --git a/tests/local_testing/test_least_busy_routing.py b/tests/local_testing/test_least_busy_routing.py
index 18ab8bf779d..fb83d4e601f 100644
--- a/tests/local_testing/test_least_busy_routing.py
+++ b/tests/local_testing/test_least_busy_routing.py
@@ -33,8 +33,8 @@ def test_model_added():
         }
     }
     least_busy_logger.log_pre_api_call(model="test", messages=[], kwargs=kwargs)
-    request_count_api_key = f"gpt-3.5-turbo_request_count"
-    assert test_cache.get_cache(key=request_count_api_key) is not None
+    request_count_api_key = "gpt-3.5-turbo_request_count:1234"
+    assert test_cache.get_cache(key=request_count_api_key) == 1
 
 
 def test_get_available_deployments():
@@ -52,8 +52,8 @@ def test_get_available_deployments():
         }
     }
     least_busy_logger.log_pre_api_call(model="test", messages=[], kwargs=kwargs)
-    request_count_api_key = f"{model_group}_request_count"
-    assert test_cache.get_cache(key=request_count_api_key) is not None
+    request_count_api_key = f"{model_group}_request_count:1234"
+    assert test_cache.get_cache(key=request_count_api_key) == 1
 
 
 # test_get_available_deployments()
@@ -104,15 +104,20 @@ async def test_router_get_available_deployments(async_test):
     router.leastbusy_logger.test_flag = True
 
     model_group = "azure-model"
-    request_count_dict = {1: 10, 2: 54, 3: 100}
-    cache_key = f"{model_group}_request_count"
+    request_count_dict = {"1": 10, "2": 54, "3": 100}
+    cache_keys = {
+        deployment_id: f"{model_group}_request_count:{deployment_id}"
+        for deployment_id in request_count_dict
+    }
     if async_test is True:
-        await router.cache.async_set_cache(key=cache_key, value=request_count_dict)
+        for deployment_id, count in request_count_dict.items():
+            await router.cache.async_set_cache(key=cache_keys[deployment_id], value=count)
         deployment = await router.async_get_available_deployment(
             model=model_group, messages=None, request_kwargs={}
         )
     else:
-        router.cache.set_cache(key=cache_key, value=request_count_dict)
+        for deployment_id, count in request_count_dict.items():
+            router.cache.set_cache(key=cache_keys[deployment_id], value=count)
         deployment = router.get_available_deployment(model=model_group, messages=None)
     print(f"deployment: {deployment}")
     assert deployment["model_info"]["id"] == "1"
@@ -124,15 +129,18 @@ async def test_router_get_available_deployments(async_test):
         messages=[{"role": "user", "content": "Hey, how's it going?"}],
     )
 
-    return_dict = router.cache.get_cache(key=cache_key)
-
     # wait 2 seconds
     time.sleep(2)
 
+    return_dict = {
+        deployment_id: router.cache.get_cache(key=cache_key)
+        for deployment_id, cache_key in cache_keys.items()
+    }
+
     assert router.leastbusy_logger.logged_success == 1
-    assert return_dict[1] == 10
-    assert return_dict[2] == 54
-    assert return_dict[3] == 100
+    assert return_dict["1"] == 10
+    assert return_dict["2"] == 54
+    assert return_dict["3"] == 100
 
 
 ## Test with Real calls ##
@@ -192,9 +200,11 @@ async def test_router_atext_completion_streaming():
     await asyncio.sleep(random.uniform(0, 2))
     await router.atext_completion(model=model, prompt=prompt, stream=True)
 
-    cache_key = f"{model}_request_count"
     ## check if calls equally distributed
-    cache_dict = router.cache.get_cache(key=cache_key)
+    cache_dict = {
+        deployment_id: router.cache.get_cache(key=f"{model}_request_count:{deployment_id}")
+        for deployment_id in ("1", "2", "3")
+    }
     for k, v in cache_dict.items():
         assert v == 1, f"Failed. K={k} called v={v} times, cache_dict={cache_dict}"
 
@@ -259,8 +269,10 @@ async def test_router_completion_streaming():
     await asyncio.sleep(random.uniform(0, 2))
     await router.acompletion(model=model, messages=messages, stream=True)
 
-    cache_key = f"{model}_request_count"
     ## check if calls equally distributed
-    cache_dict = router.cache.get_cache(key=cache_key)
+    cache_dict = {
+        deployment_id: router.cache.get_cache(key=f"{model}_request_count:{deployment_id}")
+        for deployment_id in ("1", "2", "3")
+    }
     for k, v in cache_dict.items():
         assert v == 1, f"Failed. K={k} called v={v} times, cache_dict={cache_dict}"
diff --git a/tests/test_litellm/router_strategy/test_least_busy.py b/tests/test_litellm/router_strategy/test_least_busy.py
new file mode 100644
index 00000000000..32a226a5d5b
--- /dev/null
+++ b/tests/test_litellm/router_strategy/test_least_busy.py
@@ -0,0 +1,151 @@
+import json
+from typing import Final
+
+import pytest
+
+from litellm.caching.caching import DualCache
+from litellm.caching.in_memory_cache import InMemoryCache
+from litellm.router_strategy.least_busy import IN_FLIGHT_COUNT_TTL_SECONDS, LeastBusyLoggingHandler
+
+GROUP: Final = "least-busy-group"
+DEPLOYMENT_A: Final[dict[str, object]] = {"model_info": {"id": "dep-a"}}
+DEPLOYMENT_B: Final[dict[str, object]] = {"model_info": {"id": "dep-b"}}
+HEALTHY: Final = [DEPLOYMENT_A, DEPLOYMENT_B]
+
+
+def _call_kwargs(deployment_id: str) -> dict[str, object]:
+    return {"litellm_params": {"metadata": {"model_group": GROUP}, "model_info": {"id": deployment_id}}}
+
+
+class SharedRedisCounters:
+    """Stores JSON strings and hands back a fresh object per read, the way a real Redis client does."""
+
+    def __init__(self) -> None:
+        self.encoded: dict[str, str] = {}
+        self.ttls: dict[str, float] = {}
+
+    def count(self, key: str) -> object:
+        raw: Final = self.encoded.get(key)
+        return None if raw is None else json.loads(raw)
+
+    def get_cache(self, key: str, **kwargs: object) -> object:
+        return self.count(key)
+
+    def set_cache(self, key: str, value: object, **kwargs: object) -> None:
+        self.encoded[key] = json.dumps(value)
+
+    async def async_get_cache(self, key: str, **kwargs: object) -> object:
+        return self.count(key)
+
+    async def async_set_cache(self, key: str, value: object, **kwargs: object) -> None:
+        self.set_cache(key, value)
+
+    def increment_cache(self, key: str, value: int, ttl: float | None = None, refresh_ttl: bool = False) -> int:
+        current: Final = self.count(key) or 0
+        assert isinstance(current, int)
+        incremented: Final = current + value
+        self.encoded[key] = json.dumps(incremented)
+        if ttl is not None and (refresh_ttl or key not in self.ttls):
+            self.ttls[key] = ttl
+        return incremented
+
+    async def async_increment(self, key: str, value: float, ttl: int | None = None, refresh_ttl: bool = False) -> float:
+        return self.increment_cache(key, int(value), ttl, refresh_ttl)
+
+    def batch_get_cache(self, key_list: list[str], **kwargs: object) -> dict[str, object]:
+        return {key: self.count(key) for key in key_list}
+
+    async def async_batch_get_cache(self, key_list: list[str], **kwargs: object) -> dict[str, object]:
+        return self.batch_get_cache(key_list)
+
+
+def _worker(shared: SharedRedisCounters | None) -> LeastBusyLoggingHandler:
+    cache: Final = DualCache(in_memory_cache=InMemoryCache(), redis_cache=shared)  # pyright: ignore[reportArgumentType]  # duck-typed Redis double
+    return LeastBusyLoggingHandler(router_cache=cache)
+
+
+@pytest.mark.asyncio
+async def test_worker_routes_around_a_request_another_worker_started() -> None:
+    shared: Final = SharedRedisCounters()
+    streaming_worker: Final = _worker(shared)
+    picking_worker: Final = _worker(shared)
+
+    picking_worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
+    await picking_worker.async_log_success_event(_call_kwargs("dep-a"), None, None, None)
+
+    streaming_worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
+
+    assert await picking_worker.async_get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B
+
+    await streaming_worker.async_log_success_event(_call_kwargs("dep-a"), None, None, None)
+
+    assert await picking_worker.async_get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A
+
+
+def test_sync_pick_reads_the_shared_counts() -> None:
+    shared: Final = SharedRedisCounters()
+    streaming_worker: Final = _worker(shared)
+    picking_worker: Final = _worker(shared)
+
+    picking_worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
+    picking_worker.log_success_event(_call_kwargs("dep-a"), None, None, None)
+
+    streaming_worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
+
+    assert picking_worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B
+
+    streaming_worker.log_failure_event(_call_kwargs("dep-a"), None, None, None)
+
+    assert picking_worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A
+
+
+def test_redis_counts_keep_a_refreshed_ttl() -> None:
+    shared: Final = SharedRedisCounters()
+    worker: Final = _worker(shared)
+
+    worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
+    worker.log_success_event(_call_kwargs("dep-a"), None, None, None)
+
+    assert shared.count(f"{GROUP}_request_count:dep-a") == 0
+    assert shared.ttls == {f"{GROUP}_request_count:dep-a": IN_FLIGHT_COUNT_TTL_SECONDS}
+
+
+@pytest.mark.asyncio
+async def test_counts_stay_in_memory_without_redis() -> None:
+    worker: Final = _worker(None)
+
+    worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
+
+    assert await worker.async_get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B
+    assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B
+
+    await worker.async_log_success_event(_call_kwargs("dep-a"), None, None, None)
+
+    assert await worker.async_get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A
+    assert worker.router_cache.get_cache(f"{GROUP}_request_count:dep-a") == 0
+
+
+class UnavailableRedis(SharedRedisCounters):
+    def batch_get_cache(self, key_list: list[str], **kwargs: object) -> dict[str, object]:
+        raise ConnectionError("redis is down")
+
+    def increment_cache(self, key: str, value: int, ttl: float | None = None, refresh_ttl: bool = False) -> int:
+        raise ConnectionError("redis is down")
+
+
+def test_redis_outage_never_fails_the_request() -> None:
+    worker: Final = _worker(UnavailableRedis())
+
+    worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
+
+    assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A
+
+
+def test_calls_without_a_deployment_are_ignored() -> None:
+    shared: Final = SharedRedisCounters()
+    worker: Final = _worker(shared)
+
+    worker.log_pre_api_call(model="m", messages=[], kwargs={"litellm_params": {"metadata": None}})
+    worker.log_pre_api_call(model="m", messages=[], kwargs={})
+
+    assert shared.encoded == {}
diff --git a/type-discipline-budget.json b/type-discipline-budget.json
index e7186dfe186..e413e6db2c6 100644
--- a/type-discipline-budget.json
+++ b/type-discipline-budget.json
@@ -1,9 +1,9 @@
 {
   "LIT001": {
-    "limit": 22180
+    "limit": 22176
   },
   "LIT002": {
-    "limit": 26729
+    "limit": 26721
   },
   "LIT003": {
     "limit": 261
@@ -27,10 +27,10 @@
     "limit": 0
   },
   "LIT010": {
-    "limit": 16426
+    "limit": 16412
   },
   "LIT011": {
-    "limit": 5506
+    "limit": 5505
   },
   "LIT012": {
     "limit": 4486

From c5aa4f07185c91f7c055e614734d5e216c6d043d Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 5 Sep 2026 21:14:58 -0700
Subject: [PATCH 39/85] fix(least-busy): fall back to per-worker counts when
 Redis is unreadable

Keep each worker's own in-flight counter up to date alongside the shared one,
so a Redis outage routes on that worker's counts the way it did before this
branch instead of treating every deployment as idle.

Floor a counter at zero when a decrement finds the key gone, which happens when
a request outlives the 1 hour TTL, so an expired counter cannot settle at -1 and
win every pick.
---
 litellm/router_strategy/least_busy.py         | 97 +++++++++++--------
 .../router_strategy/test_least_busy.py        | 39 +++++++-
 2 files changed, 95 insertions(+), 41 deletions(-)

diff --git a/litellm/router_strategy/least_busy.py b/litellm/router_strategy/least_busy.py
index de6a4c4f59a..00d27b5f8d9 100644
--- a/litellm/router_strategy/least_busy.py
+++ b/litellm/router_strategy/least_busy.py
@@ -39,7 +39,7 @@ class _Deployment(TypedDict):
 _CALL_KWARGS: Final = TypeAdapter(_CallKwargs)
 _DEPLOYMENTS: Final = TypeAdapter(list[_Deployment])
 _REDIS_COUNTS: Final = TypeAdapter(dict[str, float | None])
-_MEMORY_COUNTS: Final = TypeAdapter(tuple[float | None, ...])
+_MEMORY_COUNTS: Final = TypeAdapter(tuple[float | None, ...] | None)
 
 
 def _request_count_key(model_group: str, deployment_id: str) -> str:
@@ -68,8 +68,20 @@ def _request_count_keys(model_group: str, healthy_deployments: Sequence[Mapping[
     )
 
 
-def _as_count(value: float | None) -> int:
-    return 0 if value is None else int(value)
+def _as_counts(values: Sequence[float | None]) -> tuple[int, ...]:
+    return tuple(0 if value is None else int(value) for value in values)
+
+
+def _shared_counts(raw: object, keys: tuple[str, ...]) -> tuple[int, ...]:
+    by_key: Final = _REDIS_COUNTS.validate_python(raw)
+    return _as_counts([by_key.get(key) for key in keys])
+
+
+def _local_counts(raw: object, keys: tuple[str, ...]) -> tuple[int, ...]:
+    values: Final = _MEMORY_COUNTS.validate_python(raw)
+    if values is None or len(values) != len(keys):
+        return (0,) * len(keys)
+    return _as_counts(values)
 
 
 def _least_busy(
@@ -82,7 +94,8 @@ def _least_busy(
 
 def _warn_unreadable(model_group: str, error: Exception) -> None:
     verbose_router_logger.warning(
-        "least-busy routing could not read the in-flight counts for %s, treating every deployment as idle: %s",
+        "least-busy routing could not read the shared in-flight counts for %s, "
+        "falling back to this worker's own counts: %s",
         model_group,
         error,
     )
@@ -135,23 +148,31 @@ class LeastBusyLoggingHandler(CustomLogger):
         self, model_group: str, healthy_deployments: Sequence[Mapping[str, object]]
     ) -> Mapping[str, object] | None:
         keys: Final = _request_count_keys(model_group, healthy_deployments)
-        try:
-            counts: Final = tuple(_as_count(value) for value in self._read_counts(keys))
-        except Exception as e:
-            _warn_unreadable(model_group, e)
-            return _least_busy(healthy_deployments, (0,) * len(keys))
-        return _least_busy(healthy_deployments, counts)
+        redis_cache: Final = self.router_cache.redis_cache
+        if redis_cache is not None:
+            try:
+                shared: Final = _shared_counts(redis_cache.batch_get_cache(key_list=list(keys)), keys)
+            except Exception as e:
+                _warn_unreadable(model_group, e)
+            else:
+                return _least_busy(healthy_deployments, shared)
+        local: Final = _local_counts(self.router_cache.batch_get_cache(list(keys), local_only=True), keys)
+        return _least_busy(healthy_deployments, local)
 
     async def async_get_available_deployments(
         self, model_group: str, healthy_deployments: Sequence[Mapping[str, object]]
     ) -> Mapping[str, object] | None:
         keys: Final = _request_count_keys(model_group, healthy_deployments)
-        try:
-            counts: Final = tuple(_as_count(value) for value in await self._async_read_counts(keys))
-        except Exception as e:
-            _warn_unreadable(model_group, e)
-            return _least_busy(healthy_deployments, (0,) * len(keys))
-        return _least_busy(healthy_deployments, counts)
+        redis_cache: Final = self.router_cache.redis_cache
+        if redis_cache is not None:
+            try:
+                shared: Final = _shared_counts(await redis_cache.async_batch_get_cache(key_list=list(keys)), keys)
+            except Exception as e:
+                _warn_unreadable(model_group, e)
+            else:
+                return _least_busy(healthy_deployments, shared)
+        local: Final = _local_counts(await self.router_cache.async_batch_get_cache(list(keys), local_only=True), keys)
+        return _least_busy(healthy_deployments, local)
 
     def _increment(self, kwargs: Mapping[str, object], delta: int) -> None:
         ref: Final = _deployment_ref(kwargs)
@@ -160,10 +181,16 @@ class LeastBusyLoggingHandler(CustomLogger):
         key: Final = _request_count_key(*ref)
         redis_cache: Final = self.router_cache.redis_cache
         try:
+            local: Final = self.router_cache.increment_cache(
+                key, delta, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS
+            )
+            if local < 0:
+                self.router_cache.set_cache(key, 0, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS)
             if redis_cache is None:
-                self.router_cache.increment_cache(key, delta, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS)
-            else:
-                redis_cache.increment_cache(key, delta, ttl=IN_FLIGHT_COUNT_TTL_SECONDS, refresh_ttl=True)
+                return
+            shared: Final = redis_cache.increment_cache(key, delta, ttl=IN_FLIGHT_COUNT_TTL_SECONDS, refresh_ttl=True)
+            if shared < 0:
+                redis_cache.set_cache(key, 0, ttl=IN_FLIGHT_COUNT_TTL_SECONDS)
         except Exception as e:
             _warn_unwritable(key, e)
 
@@ -174,27 +201,17 @@ class LeastBusyLoggingHandler(CustomLogger):
         key: Final = _request_count_key(*ref)
         redis_cache: Final = self.router_cache.redis_cache
         try:
+            local: Final = await self.router_cache.async_increment_cache(
+                key, delta, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS
+            )
+            if local is not None and local < 0:
+                await self.router_cache.async_set_cache(key, 0, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS)
             if redis_cache is None:
-                await self.router_cache.async_increment_cache(
-                    key, delta, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS
-                )
-            else:
-                await redis_cache.async_increment(key, delta, ttl=IN_FLIGHT_COUNT_TTL_SECONDS, refresh_ttl=True)
+                return
+            shared: Final = await redis_cache.async_increment(
+                key, delta, ttl=IN_FLIGHT_COUNT_TTL_SECONDS, refresh_ttl=True
+            )
+            if shared < 0:
+                await redis_cache.async_set_cache(key, 0, ttl=IN_FLIGHT_COUNT_TTL_SECONDS)
         except Exception as e:
             _warn_unwritable(key, e)
-
-    def _read_counts(self, keys: tuple[str, ...]) -> tuple[float | None, ...]:
-        redis_cache: Final = self.router_cache.redis_cache
-        if redis_cache is None:
-            return _MEMORY_COUNTS.validate_python(self.router_cache.batch_get_cache(list(keys), local_only=True))
-        by_key: Final = _REDIS_COUNTS.validate_python(redis_cache.batch_get_cache(key_list=list(keys)))
-        return tuple(by_key.get(key) for key in keys)
-
-    async def _async_read_counts(self, keys: tuple[str, ...]) -> tuple[float | None, ...]:
-        redis_cache: Final = self.router_cache.redis_cache
-        if redis_cache is None:
-            return _MEMORY_COUNTS.validate_python(
-                await self.router_cache.async_batch_get_cache(list(keys), local_only=True)
-            )
-        by_key: Final = _REDIS_COUNTS.validate_python(await redis_cache.async_batch_get_cache(key_list=list(keys)))
-        return tuple(by_key.get(key) for key in keys)
diff --git a/tests/test_litellm/router_strategy/test_least_busy.py b/tests/test_litellm/router_strategy/test_least_busy.py
index 32a226a5d5b..55702c6fe73 100644
--- a/tests/test_litellm/router_strategy/test_least_busy.py
+++ b/tests/test_litellm/router_strategy/test_least_busy.py
@@ -133,14 +133,51 @@ class UnavailableRedis(SharedRedisCounters):
         raise ConnectionError("redis is down")
 
 
-def test_redis_outage_never_fails_the_request() -> None:
+@pytest.mark.asyncio
+async def test_a_redis_outage_falls_back_to_this_workers_own_counts() -> None:
     worker: Final = _worker(UnavailableRedis())
 
     worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
 
+    assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B
+    assert await worker.async_get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B
+
+    await worker.async_log_success_event(_call_kwargs("dep-a"), None, None, None)
+
     assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A
 
 
+def test_a_shared_counter_that_expired_mid_request_cannot_go_negative() -> None:
+    shared: Final = SharedRedisCounters()
+    worker: Final = _worker(shared)
+
+    worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
+    shared.encoded.clear()
+    worker.log_success_event(_call_kwargs("dep-a"), None, None, None)
+
+    assert shared.count(f"{GROUP}_request_count:dep-a") == 0
+
+    worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
+
+    assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B
+
+
+@pytest.mark.asyncio
+async def test_a_local_counter_that_expired_mid_request_cannot_go_negative() -> None:
+    worker: Final = _worker(None)
+    in_memory: Final = worker.router_cache.in_memory_cache
+
+    worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
+    in_memory.delete_cache(f"{GROUP}_request_count:dep-a")
+    await worker.async_log_success_event(_call_kwargs("dep-a"), None, None, None)
+
+    assert worker.router_cache.get_cache(f"{GROUP}_request_count:dep-a") == 0
+
+    worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
+
+    assert await worker.async_get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B
+
+
 def test_calls_without_a_deployment_are_ignored() -> None:
     shared: Final = SharedRedisCounters()
     worker: Final = _worker(shared)

From a9bc2cb50b4ba443e72b7d03a33447cc8724b037 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sat, 5 Sep 2026 22:38:26 -0700
Subject: [PATCH 40/85] fix(least-busy): clamp the shared in-flight count to
 zero in one call

A decrement whose matching increment is gone, because the counter key
expired while the request was still in flight, used to recreate the key
at -1, and a deployment with a negative count looks permanently idle, so
it collects every pick from then on. The repair write that followed the
decrement could also land after another pod's increment and erase it.

The increment, the clamp at zero and the TTL refresh now run as a single
Lua call, so nothing can interleave between them.
---
 litellm/caching/redis_cache.py                | 39 ++++++++++++++++++-
 litellm/router_strategy/least_busy.py         | 10 +----
 .../router_strategy/test_least_busy.py        | 20 +++++-----
 3 files changed, 50 insertions(+), 19 deletions(-)

diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py
index d4c47914394..4eccde5742b 100644
--- a/litellm/caching/redis_cache.py
+++ b/litellm/caching/redis_cache.py
@@ -20,6 +20,8 @@ from contextvars import ContextVar
 from datetime import timedelta
 from typing import TYPE_CHECKING, Any, Final, Protocol, TypeVar, cast
 
+from pydantic import TypeAdapter
+
 import litellm
 from litellm._logging import print_verbose, verbose_logger
 from litellm.constants import (
@@ -80,11 +82,22 @@ class _AsyncRedisCommands(Protocol):
 
     def pipeline(self, transaction: bool = True) -> "Pipeline[bytes]": ...
 
+    def eval(self, script: str, numkeys: int, *keys_and_args: str | bytes | float) -> Awaitable[object]: ...
+
 
 _BREAKER_GUARD_FRAME_NAMES: Final = frozenset(
     {"<lambda>", "wrapper", "_run_under_circuit_breaker", "_run_under_circuit_breaker_sync"}
 )
 
+_INCREMENT_WITH_FLOOR_LUA: Final = (
+    "local count = redis.call('INCRBY', KEYS[1], ARGV[1]) "
+    "if count < 0 then redis.call('SET', KEYS[1], 0) count = 0 end "
+    "redis.call('EXPIRE', KEYS[1], ARGV[2]) "
+    "return count"
+)
+
+_LUA_COUNT: Final = TypeAdapter(int)
+
 
 def _get_call_stack_info(num_frames: int = 2) -> str:
     """
@@ -680,7 +693,7 @@ class RedisCache(BaseCache):
             # NON blocking - notify users Redis is throwing an exception
             print_verbose(f"litellm.caching.caching: set() - Got exception from REDIS : {e}")
 
-    def increment_cache(self, key, value: int, ttl: float | None = None, refresh_ttl: bool = False, **kwargs) -> int:
+    def increment_cache(self, key, value: int, ttl: float | None = None, **kwargs) -> int:
         _redis_client: Final = self.redis_client
         start_time = time.time()
         set_ttl: Final = self.get_ttl(ttl=ttl)
@@ -701,7 +714,7 @@ class RedisCache(BaseCache):
             if set_ttl is not None:
                 # check if key already has ttl, if not -> set ttl
                 start_time = time.time()
-                current_ttl: Final = -1 if refresh_ttl else _redis_client.ttl(key)
+                current_ttl: Final = _redis_client.ttl(key)
                 end_time = time.time()
                 _duration = end_time - start_time
                 self.service_logger_obj.service_success_hook(
@@ -736,6 +749,20 @@ class RedisCache(BaseCache):
             )
             raise e
 
+    def increment_with_floor(self, key: str, value: int, ttl: int) -> int:
+        """Add ``value`` to ``key``, clamp the result at zero, and refresh the TTL, in one Lua call.
+
+        A counter whose key expired while a request was still in flight would otherwise be
+        recreated negative by that request's decrement. Clamping inside the same call is what
+        keeps it safe: a separate corrective write could land after another pod's increment and
+        erase it. Returns the resulting count.
+        """
+        namespaced_key: Final = self.check_and_fix_namespace(key=key)
+        count: Final[object] = self.redis_client.eval(  # pyright: ignore[reportAttributeAccessIssue]  # stubs omit eval
+            _INCREMENT_WITH_FLOOR_LUA, 1, namespaced_key, value, ttl
+        )
+        return _LUA_COUNT.validate_python(count)
+
     @_redis_circuit_breaker_guard
     async def async_scan_iter(self, pattern: str, count: int = 100) -> list:
         start_time: Final = time.time()
@@ -1241,6 +1268,14 @@ class RedisCache(BaseCache):
             result = result.decode()
         return float(result)
 
+    @_redis_circuit_breaker_guard
+    async def async_increment_with_floor(self, key: str, value: int, ttl: int) -> int:
+        """Async twin of ``increment_with_floor``, sharing its Lua script and its guarantees."""
+        _redis_client: Final = self._async_commands()
+        namespaced_key: Final = self.check_and_fix_namespace(key=key)
+        count: Final = await _redis_client.eval(_INCREMENT_WITH_FLOOR_LUA, 1, namespaced_key, value, ttl)
+        return _LUA_COUNT.validate_python(count)
+
     async def flush_cache_buffer(self):
         print_verbose(f"flushing to redis....reached size of buffer {len(self.redis_batch_writing_buffer)}")
         await self.async_set_cache_pipeline(self.redis_batch_writing_buffer)
diff --git a/litellm/router_strategy/least_busy.py b/litellm/router_strategy/least_busy.py
index 00d27b5f8d9..771d2bb4328 100644
--- a/litellm/router_strategy/least_busy.py
+++ b/litellm/router_strategy/least_busy.py
@@ -188,9 +188,7 @@ class LeastBusyLoggingHandler(CustomLogger):
                 self.router_cache.set_cache(key, 0, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS)
             if redis_cache is None:
                 return
-            shared: Final = redis_cache.increment_cache(key, delta, ttl=IN_FLIGHT_COUNT_TTL_SECONDS, refresh_ttl=True)
-            if shared < 0:
-                redis_cache.set_cache(key, 0, ttl=IN_FLIGHT_COUNT_TTL_SECONDS)
+            redis_cache.increment_with_floor(key, delta, IN_FLIGHT_COUNT_TTL_SECONDS)
         except Exception as e:
             _warn_unwritable(key, e)
 
@@ -208,10 +206,6 @@ class LeastBusyLoggingHandler(CustomLogger):
                 await self.router_cache.async_set_cache(key, 0, local_only=True, ttl=IN_FLIGHT_COUNT_TTL_SECONDS)
             if redis_cache is None:
                 return
-            shared: Final = await redis_cache.async_increment(
-                key, delta, ttl=IN_FLIGHT_COUNT_TTL_SECONDS, refresh_ttl=True
-            )
-            if shared < 0:
-                await redis_cache.async_set_cache(key, 0, ttl=IN_FLIGHT_COUNT_TTL_SECONDS)
+            await redis_cache.async_increment_with_floor(key, delta, IN_FLIGHT_COUNT_TTL_SECONDS)
         except Exception as e:
             _warn_unwritable(key, e)
diff --git a/tests/test_litellm/router_strategy/test_least_busy.py b/tests/test_litellm/router_strategy/test_least_busy.py
index 55702c6fe73..eb9591811ac 100644
--- a/tests/test_litellm/router_strategy/test_least_busy.py
+++ b/tests/test_litellm/router_strategy/test_least_busy.py
@@ -40,17 +40,16 @@ class SharedRedisCounters:
     async def async_set_cache(self, key: str, value: object, **kwargs: object) -> None:
         self.set_cache(key, value)
 
-    def increment_cache(self, key: str, value: int, ttl: float | None = None, refresh_ttl: bool = False) -> int:
+    def increment_with_floor(self, key: str, value: int, ttl: int) -> int:
         current: Final = self.count(key) or 0
         assert isinstance(current, int)
-        incremented: Final = current + value
+        incremented: Final = max(0, current + value)
         self.encoded[key] = json.dumps(incremented)
-        if ttl is not None and (refresh_ttl or key not in self.ttls):
-            self.ttls[key] = ttl
+        self.ttls[key] = ttl
         return incremented
 
-    async def async_increment(self, key: str, value: float, ttl: int | None = None, refresh_ttl: bool = False) -> float:
-        return self.increment_cache(key, int(value), ttl, refresh_ttl)
+    async def async_increment_with_floor(self, key: str, value: int, ttl: int) -> int:
+        return self.increment_with_floor(key, value, ttl)
 
     def batch_get_cache(self, key_list: list[str], **kwargs: object) -> dict[str, object]:
         return {key: self.count(key) for key in key_list}
@@ -102,12 +101,14 @@ def test_sync_pick_reads_the_shared_counts() -> None:
 def test_redis_counts_keep_a_refreshed_ttl() -> None:
     shared: Final = SharedRedisCounters()
     worker: Final = _worker(shared)
+    key: Final = f"{GROUP}_request_count:dep-a"
+    shared.ttls[key] = 5
 
     worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
     worker.log_success_event(_call_kwargs("dep-a"), None, None, None)
 
-    assert shared.count(f"{GROUP}_request_count:dep-a") == 0
-    assert shared.ttls == {f"{GROUP}_request_count:dep-a": IN_FLIGHT_COUNT_TTL_SECONDS}
+    assert shared.count(key) == 0
+    assert shared.ttls == {key: IN_FLIGHT_COUNT_TTL_SECONDS}
 
 
 @pytest.mark.asyncio
@@ -129,7 +130,7 @@ class UnavailableRedis(SharedRedisCounters):
     def batch_get_cache(self, key_list: list[str], **kwargs: object) -> dict[str, object]:
         raise ConnectionError("redis is down")
 
-    def increment_cache(self, key: str, value: int, ttl: float | None = None, refresh_ttl: bool = False) -> int:
+    def increment_with_floor(self, key: str, value: int, ttl: int) -> int:
         raise ConnectionError("redis is down")
 
 
@@ -159,6 +160,7 @@ def test_a_shared_counter_that_expired_mid_request_cannot_go_negative() -> None:
 
     worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
 
+    assert shared.count(f"{GROUP}_request_count:dep-a") == 1
     assert worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_B
 
 

From caf9bbbd5a535e61da844e1a0225a4f8a6312b8d Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sun, 6 Sep 2026 00:20:05 -0700
Subject: [PATCH 41/85] fix(least-busy): keep the shared count readable,
 counted once, and off the loop

A Redis outage read as "every deployment is idle", because batch_get_cache
swallows the failure and answers with an empty dict. batch_get_counts and its
async twin raise instead, so a worker that cannot reach Redis falls back to its
own numbers rather than routing on zeros.

The counter's TTL is now set only on a key that has none, so a +1 left behind by
a worker that died mid-request ages out an hour after the key was created. It
used to be refreshed on every touch, which kept that stuck count alive for as
long as the group took traffic.

Two least-busy groups counted the same request twice, since the pre-call list
kept a selector per group while the success list deduped by class. The selector
now goes on through add_litellm_input_callback, which dedupes the same way.

A prompt-management model picked its deployment on the synchronous path, so the
new Redis read landed on the event loop and configured routing plugins never
ran. It awaits the async selector now.
---
 basedpyright-code-budget.json                 | 12 ++--
 litellm/caching/redis_cache.py                | 36 +++++++++-
 litellm/router.py                             |  8 ++-
 litellm/router_strategy/least_busy.py         | 10 +--
 ruff-strict-budget.json                       | 10 +--
 .../test_litellm/caching/test_redis_cache.py  | 44 ++++++++++++
 .../router_strategy/test_least_busy.py        | 67 +++++++++----------
 .../test_router_routing_groups.py             | 38 +++++++++++
 .../test_router_routing_plugins.py            | 30 +++++++++
 type-discipline-budget.json                   |  8 +--
 10 files changed, 199 insertions(+), 64 deletions(-)

diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json
index c21dc76743a..32d3730aa32 100644
--- a/basedpyright-code-budget.json
+++ b/basedpyright-code-budget.json
@@ -54,10 +54,10 @@
     "limit": 0
   },
   "reportMissingParameterType": {
-    "limit": 5551
+    "limit": 5532
   },
   "reportMissingTypeArgument": {
-    "limit": 15277
+    "limit": 15273
   },
   "reportMissingTypeStubs": {
     "limit": 40
@@ -105,13 +105,13 @@
     "limit": 109
   },
   "reportUnknownMemberType": {
-    "limit": 38240
+    "limit": 38208
   },
   "reportUnknownParameterType": {
-    "limit": 19558
+    "limit": 19532
   },
   "reportUnknownVariableType": {
-    "limit": 29781
+    "limit": 29751
   },
   "reportUnnecessaryCast": {
     "limit": 110
@@ -135,7 +135,7 @@
     "limit": 21
   },
   "reportUnusedFunction": {
-    "limit": 138
+    "limit": 137
   },
   "reportUnusedImport": {
     "limit": 542
diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py
index 4eccde5742b..3abc6e6f3c9 100644
--- a/litellm/caching/redis_cache.py
+++ b/litellm/caching/redis_cache.py
@@ -92,11 +92,18 @@ _BREAKER_GUARD_FRAME_NAMES: Final = frozenset(
 _INCREMENT_WITH_FLOOR_LUA: Final = (
     "local count = redis.call('INCRBY', KEYS[1], ARGV[1]) "
     "if count < 0 then redis.call('SET', KEYS[1], 0) count = 0 end "
-    "redis.call('EXPIRE', KEYS[1], ARGV[2]) "
+    "if redis.call('TTL', KEYS[1]) < 0 then redis.call('EXPIRE', KEYS[1], ARGV[2]) end "
     "return count"
 )
 
 _LUA_COUNT: Final = TypeAdapter(int)
+_OPTIONAL_COUNTS: Final = TypeAdapter(tuple[int | None, ...])
+
+
+def _decoded_counts(values: Sequence[bytes | str | None]) -> tuple[int | None, ...]:
+    return _OPTIONAL_COUNTS.validate_python(
+        tuple(value.decode("utf-8") if isinstance(value, bytes) else value for value in values)
+    )
 
 
 def _get_call_stack_info(num_frames: int = 2) -> str:
@@ -749,13 +756,19 @@ class RedisCache(BaseCache):
             )
             raise e
 
+    @_redis_circuit_breaker_guard_sync
     def increment_with_floor(self, key: str, value: int, ttl: int) -> int:
-        """Add ``value`` to ``key``, clamp the result at zero, and refresh the TTL, in one Lua call.
+        """Add ``value`` to ``key``, clamp the result at zero, and give a new key ``ttl``, in one Lua call.
 
         A counter whose key expired while a request was still in flight would otherwise be
         recreated negative by that request's decrement. Clamping inside the same call is what
         keeps it safe: a separate corrective write could land after another pod's increment and
-        erase it. Returns the resulting count.
+        erase it.
+
+        The TTL is set only on a key that has none, so a counter expires ``ttl`` after it was
+        created rather than ``ttl`` after it was last touched. Refreshing it on every touch
+        would keep a count a dead worker never decremented alive for as long as the group
+        takes traffic. Returns the resulting count.
         """
         namespaced_key: Final = self.check_and_fix_namespace(key=key)
         count: Final[object] = self.redis_client.eval(  # pyright: ignore[reportAttributeAccessIssue]  # stubs omit eval
@@ -763,6 +776,23 @@ class RedisCache(BaseCache):
         )
         return _LUA_COUNT.validate_python(count)
 
+    @_redis_circuit_breaker_guard_sync
+    def batch_get_counts(self, key_list: list[str]) -> tuple[int | None, ...]:
+        """Read integer counters for ``key_list``, in order, raising when Redis cannot answer.
+
+        ``batch_get_cache`` swallows every failure and returns an empty dict, which the caller
+        cannot tell apart from "every counter is unset". A caller that has to fall back to its
+        own numbers when Redis is unreachable needs the failure, not a dict of zeros.
+        """
+        namespaced_keys: Final = [self.check_and_fix_namespace(key=key) for key in key_list]
+        return _decoded_counts(self._run_redis_mget_operation(keys=namespaced_keys))
+
+    @_redis_circuit_breaker_guard
+    async def async_batch_get_counts(self, key_list: list[str]) -> tuple[int | None, ...]:
+        """Async twin of ``batch_get_counts``, raising on failure the same way."""
+        namespaced_keys: Final = [self.check_and_fix_namespace(key=key) for key in key_list]
+        return _decoded_counts(await self._async_run_redis_mget_operation(keys=namespaced_keys))
+
     @_redis_circuit_breaker_guard
     async def async_scan_iter(self, pattern: str, count: int = 100) -> list:
         start_time: Final = time.time()
diff --git a/litellm/router.py b/litellm/router.py
index 3f450661946..f5b7924fb58 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -1212,7 +1212,7 @@ class Router:
                 selector = LeastBusyLoggingHandler(router_cache=self.cache)
                 if register_callbacks:
                     if isinstance(litellm.input_callback, list):
-                        litellm.input_callback.append(selector)
+                        litellm.logging_callback_manager.add_litellm_input_callback(selector)
                     else:
                         litellm.input_callback = [selector]
             case RoutingStrategy.USAGE_BASED_ROUTING.value:
@@ -4139,10 +4139,12 @@ class Router:
                 }
             )
         litellm_logging_object = cast(LiteLLMLogging, litellm_logging_object)
-        prompt_management_deployment: Final = self.get_available_deployment(
+        specific_deployment: Final = kwargs.pop("specific_deployment", None)
+        prompt_management_deployment: Final = await self.async_get_available_deployment(
             model=model,
             messages=[{"role": "user", "content": "prompt"}],
-            specific_deployment=kwargs.pop("specific_deployment", None),
+            specific_deployment=specific_deployment,
+            request_kwargs=kwargs,
         )
 
         self._update_kwargs_with_deployment(deployment=prompt_management_deployment, kwargs=kwargs)
diff --git a/litellm/router_strategy/least_busy.py b/litellm/router_strategy/least_busy.py
index 771d2bb4328..ab6d702bbc2 100644
--- a/litellm/router_strategy/least_busy.py
+++ b/litellm/router_strategy/least_busy.py
@@ -38,7 +38,6 @@ class _Deployment(TypedDict):
 
 _CALL_KWARGS: Final = TypeAdapter(_CallKwargs)
 _DEPLOYMENTS: Final = TypeAdapter(list[_Deployment])
-_REDIS_COUNTS: Final = TypeAdapter(dict[str, float | None])
 _MEMORY_COUNTS: Final = TypeAdapter(tuple[float | None, ...] | None)
 
 
@@ -72,11 +71,6 @@ def _as_counts(values: Sequence[float | None]) -> tuple[int, ...]:
     return tuple(0 if value is None else int(value) for value in values)
 
 
-def _shared_counts(raw: object, keys: tuple[str, ...]) -> tuple[int, ...]:
-    by_key: Final = _REDIS_COUNTS.validate_python(raw)
-    return _as_counts([by_key.get(key) for key in keys])
-
-
 def _local_counts(raw: object, keys: tuple[str, ...]) -> tuple[int, ...]:
     values: Final = _MEMORY_COUNTS.validate_python(raw)
     if values is None or len(values) != len(keys):
@@ -151,7 +145,7 @@ class LeastBusyLoggingHandler(CustomLogger):
         redis_cache: Final = self.router_cache.redis_cache
         if redis_cache is not None:
             try:
-                shared: Final = _shared_counts(redis_cache.batch_get_cache(key_list=list(keys)), keys)
+                shared: Final = _as_counts(redis_cache.batch_get_counts(list(keys)))
             except Exception as e:
                 _warn_unreadable(model_group, e)
             else:
@@ -166,7 +160,7 @@ class LeastBusyLoggingHandler(CustomLogger):
         redis_cache: Final = self.router_cache.redis_cache
         if redis_cache is not None:
             try:
-                shared: Final = _shared_counts(await redis_cache.async_batch_get_cache(key_list=list(keys)), keys)
+                shared: Final = _as_counts(await redis_cache.async_batch_get_counts(list(keys)))
             except Exception as e:
                 _warn_unreadable(model_group, e)
             else:
diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json
index ceff8c6e15c..d63a69de76f 100644
--- a/ruff-strict-budget.json
+++ b/ruff-strict-budget.json
@@ -1,6 +1,6 @@
 {
   "ANN001": {
-    "limit": 2937
+    "limit": 2918
   },
   "ANN002": {
     "limit": 71
@@ -9,10 +9,10 @@
     "limit": 806
   },
   "ANN201": {
-    "limit": 1972
+    "limit": 1965
   },
   "ANN202": {
-    "limit": 830
+    "limit": 829
   },
   "ANN204": {
     "limit": 683
@@ -57,7 +57,7 @@
     "limit": 3
   },
   "BLE001": {
-    "limit": 2915
+    "limit": 2914
   },
   "C401": {
     "limit": 8
@@ -189,7 +189,7 @@
     "limit": 0
   },
   "S110": {
-    "limit": 212
+    "limit": 207
   },
   "S112": {
     "limit": 22
diff --git a/tests/test_litellm/caching/test_redis_cache.py b/tests/test_litellm/caching/test_redis_cache.py
index 2f412e7382b..6b2df118611 100644
--- a/tests/test_litellm/caching/test_redis_cache.py
+++ b/tests/test_litellm/caching/test_redis_cache.py
@@ -525,6 +525,50 @@ def test_circuit_breaker_open_keeps_sync_batch_get_cache_as_a_miss(sync_batch_re
     assert sync_batch_redis_cache.batch_get_cache(key_list=["lit6729"]) == {}
 
 
+def test_batch_get_counts_raises_where_batch_get_cache_reports_a_miss(sync_batch_redis_cache):
+    """A caller that must fall back when Redis is unreachable needs the failure, not zeros.
+
+    The batch read answers a dead Redis with an empty dict, which a counting caller cannot tell
+    apart from "every counter is unset". Least-busy routing read that as an idle deployment and
+    kept sending traffic to it instead of falling back to this worker's own in-flight counts.
+    """
+    assert sync_batch_redis_cache.batch_get_cache(key_list=["lit7039"]) == {}
+
+    with pytest.raises(OSError, match="redis unavailable"):
+        sync_batch_redis_cache.batch_get_counts(["lit7039"])
+
+
+@pytest.mark.asyncio
+async def test_async_batch_get_counts_raises_where_async_batch_get_cache_reports_a_miss(redis_no_ping: None):
+    """Async twin: the async batch read hides the same failure behind an empty dict."""
+    failing_client = AsyncMock()
+    failing_client.mget.side_effect = OSError("redis unavailable")
+    with patch(  # test-quality-ok: RedisCache.__init__ builds its client eagerly, with no injection point
+        "litellm._redis.get_redis_client", return_value=MagicMock()
+    ):
+        cache = RedisCache(host="127.0.0.1", port=6379)
+
+    with patch.object(cache, "init_async_client", return_value=failing_client):
+        assert await cache.async_batch_get_cache(key_list=["lit7039"]) == {}
+
+        with pytest.raises(OSError, match="redis unavailable"):
+            await cache.async_batch_get_counts(["lit7039"])
+
+
+@pytest.mark.parametrize("stored", [b"3", "3"])
+def test_batch_get_counts_reads_counters_in_order_and_keeps_unset_keys_apart(stored, redis_no_ping: None):
+    """Counters come back positionally, so an unset key has to stay a hole rather than shift the
+    rest of the row onto the wrong deployments, and a count has to survive whether the client
+    hands it back as bytes or as text."""
+    with patch(  # test-quality-ok: RedisCache.__init__ builds its client eagerly, with no injection point
+        "litellm._redis.get_redis_client", return_value=MagicMock()
+    ):
+        cache = RedisCache(host="127.0.0.1", port=6379)
+    cache.redis_client.mget.return_value = [stored, None, b"0"]
+
+    assert cache.batch_get_counts(["dep-a", "dep-b", "dep-c"]) == (3, None, 0)
+
+
 @pytest.fixture
 def sync_batch_cache_with_service_logger(redis_no_ping: None) -> Iterator[tuple[RedisCache, ServiceLogging]]:
     service_logger = ServiceLogging(mock_testing=True)
diff --git a/tests/test_litellm/router_strategy/test_least_busy.py b/tests/test_litellm/router_strategy/test_least_busy.py
index eb9591811ac..9efa526fc02 100644
--- a/tests/test_litellm/router_strategy/test_least_busy.py
+++ b/tests/test_litellm/router_strategy/test_least_busy.py
@@ -1,4 +1,3 @@
-import json
 from typing import Final
 
 import pytest
@@ -18,44 +17,34 @@ def _call_kwargs(deployment_id: str) -> dict[str, object]:
 
 
 class SharedRedisCounters:
-    """Stores JSON strings and hands back a fresh object per read, the way a real Redis client does."""
+    """Mirrors what Redis gives the handler: increments clamped at zero, a TTL set once when
+    the key is created, and ordered reads that raise rather than invent a value."""
 
     def __init__(self) -> None:
-        self.encoded: dict[str, str] = {}
-        self.ttls: dict[str, float] = {}
+        self.counts: dict[str, int] = {}
+        self.ttls: dict[str, int] = {}
 
-    def count(self, key: str) -> object:
-        raw: Final = self.encoded.get(key)
-        return None if raw is None else json.loads(raw)
+    def count(self, key: str) -> int | None:
+        return self.counts.get(key)
 
-    def get_cache(self, key: str, **kwargs: object) -> object:
-        return self.count(key)
-
-    def set_cache(self, key: str, value: object, **kwargs: object) -> None:
-        self.encoded[key] = json.dumps(value)
-
-    async def async_get_cache(self, key: str, **kwargs: object) -> object:
-        return self.count(key)
-
-    async def async_set_cache(self, key: str, value: object, **kwargs: object) -> None:
-        self.set_cache(key, value)
+    def expire(self, key: str) -> None:
+        self.counts.pop(key, None)
+        self.ttls.pop(key, None)
 
     def increment_with_floor(self, key: str, value: int, ttl: int) -> int:
-        current: Final = self.count(key) or 0
-        assert isinstance(current, int)
-        incremented: Final = max(0, current + value)
-        self.encoded[key] = json.dumps(incremented)
-        self.ttls[key] = ttl
+        incremented: Final = max(0, self.counts.get(key, 0) + value)
+        self.counts[key] = incremented
+        self.ttls.setdefault(key, ttl)
         return incremented
 
     async def async_increment_with_floor(self, key: str, value: int, ttl: int) -> int:
         return self.increment_with_floor(key, value, ttl)
 
-    def batch_get_cache(self, key_list: list[str], **kwargs: object) -> dict[str, object]:
-        return {key: self.count(key) for key in key_list}
+    def batch_get_counts(self, key_list: list[str]) -> tuple[int | None, ...]:
+        return tuple(self.counts.get(key) for key in key_list)
 
-    async def async_batch_get_cache(self, key_list: list[str], **kwargs: object) -> dict[str, object]:
-        return self.batch_get_cache(key_list)
+    async def async_batch_get_counts(self, key_list: list[str]) -> tuple[int | None, ...]:
+        return self.batch_get_counts(key_list)
 
 
 def _worker(shared: SharedRedisCounters | None) -> LeastBusyLoggingHandler:
@@ -98,17 +87,25 @@ def test_sync_pick_reads_the_shared_counts() -> None:
     assert picking_worker.get_available_deployments(GROUP, HEALTHY) is DEPLOYMENT_A
 
 
-def test_redis_counts_keep_a_refreshed_ttl() -> None:
+def test_the_handler_never_pushes_a_counters_ttl_forward() -> None:
+    """A worker that dies mid-request leaves a +1 nobody will ever decrement. Redis expires that
+    stuck count an hour after the key was created, which only works while nothing writes the TTL
+    again: a handler that refreshed it on every touch would keep the count alive for as long as
+    the group takes traffic, and the deployment would read busier than it is forever."""
     shared: Final = SharedRedisCounters()
     worker: Final = _worker(shared)
     key: Final = f"{GROUP}_request_count:dep-a"
-    shared.ttls[key] = 5
 
+    worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
+
+    assert shared.ttls == {key: IN_FLIGHT_COUNT_TTL_SECONDS}
+
+    shared.ttls[key] = 5
     worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
     worker.log_success_event(_call_kwargs("dep-a"), None, None, None)
 
-    assert shared.count(key) == 0
-    assert shared.ttls == {key: IN_FLIGHT_COUNT_TTL_SECONDS}
+    assert shared.count(key) == 1
+    assert shared.ttls == {key: 5}
 
 
 @pytest.mark.asyncio
@@ -127,10 +124,10 @@ async def test_counts_stay_in_memory_without_redis() -> None:
 
 
 class UnavailableRedis(SharedRedisCounters):
-    def batch_get_cache(self, key_list: list[str], **kwargs: object) -> dict[str, object]:
+    def increment_with_floor(self, key: str, value: int, ttl: int) -> int:
         raise ConnectionError("redis is down")
 
-    def increment_with_floor(self, key: str, value: int, ttl: int) -> int:
+    def batch_get_counts(self, key_list: list[str]) -> tuple[int | None, ...]:
         raise ConnectionError("redis is down")
 
 
@@ -153,7 +150,7 @@ def test_a_shared_counter_that_expired_mid_request_cannot_go_negative() -> None:
     worker: Final = _worker(shared)
 
     worker.log_pre_api_call(model="m", messages=[], kwargs=_call_kwargs("dep-a"))
-    shared.encoded.clear()
+    shared.expire(f"{GROUP}_request_count:dep-a")
     worker.log_success_event(_call_kwargs("dep-a"), None, None, None)
 
     assert shared.count(f"{GROUP}_request_count:dep-a") == 0
@@ -187,4 +184,4 @@ def test_calls_without_a_deployment_are_ignored() -> None:
     worker.log_pre_api_call(model="m", messages=[], kwargs={"litellm_params": {"metadata": None}})
     worker.log_pre_api_call(model="m", messages=[], kwargs={})
 
-    assert shared.encoded == {}
+    assert shared.counts == {}
diff --git a/tests/test_litellm/router_strategy/test_router_routing_groups.py b/tests/test_litellm/router_strategy/test_router_routing_groups.py
index 5599c5aad63..0ce0ed5b37e 100644
--- a/tests/test_litellm/router_strategy/test_router_routing_groups.py
+++ b/tests/test_litellm/router_strategy/test_router_routing_groups.py
@@ -12,6 +12,7 @@ import pytest
 
 import litellm
 from litellm import Router
+from litellm.integrations.custom_logger import CustomLogger
 from litellm.types.router import RoutingGroup, RoutingStrategy
 
 
@@ -435,6 +436,43 @@ def test_update_settings_unregisters_group_selectors_when_groups_removed(monkeyp
     assert router._group_selectors == {}
 
 
+def test_two_least_busy_groups_count_a_request_once(monkeypatch):
+    """
+    Least-busy counts a request up from the pre-call hooks on `litellm.input_callback` and
+    back down from the success hooks on `litellm.callbacks`. The success list drops a second
+    selector of the same class, so a pre-call list that kept both counted every request twice
+    and released it once, and the deployment's in-flight count climbed until it looked pinned.
+    """
+    monkeypatch.setattr(litellm, "callbacks", [])
+    monkeypatch.setattr(litellm, "input_callback", [])
+
+    router = _build_router(
+        routing_strategy="least-busy",
+        routing_groups=[
+            {
+                "group_name": "fast",
+                "models": ["filtered-model"],
+                "routing_strategy": "least-busy",
+            }
+        ],
+    )
+    kwargs = {
+        "litellm_params": {
+            "metadata": {"model_group": "filtered-model"},
+            "model_info": {"id": "deploy-1"},
+        }
+    }
+
+    for callback in litellm.input_callback:
+        if isinstance(callback, CustomLogger):
+            callback.log_pre_api_call(model="filtered-model", messages=[], kwargs=kwargs)
+    for callback in litellm.callbacks:
+        if isinstance(callback, CustomLogger):
+            callback.log_success_event(kwargs, None, None, None)
+
+    assert router.cache.get_cache("filtered-model_request_count:deploy-1") == 0
+
+
 # ---------------------------------------------------------------------------
 # Direct helper coverage
 # ---------------------------------------------------------------------------
diff --git a/tests/test_litellm/router_strategy/test_router_routing_plugins.py b/tests/test_litellm/router_strategy/test_router_routing_plugins.py
index 293af36080a..c49b22ea367 100644
--- a/tests/test_litellm/router_strategy/test_router_routing_plugins.py
+++ b/tests/test_litellm/router_strategy/test_router_routing_plugins.py
@@ -164,6 +164,36 @@ async def test_async_completion_with_unsupported_strategy_rejects_configured_plu
         await router.acompletion(model="smart-router", messages=[{"role": "user", "content": "hi"}])
 
 
+@pytest.mark.asyncio
+async def test_prompt_management_model_still_runs_the_plugin_pipeline():
+    """
+    A prompt-management model routes through its own factory, which picked the deployment
+    on the synchronous path. Plugins never run there, so the guard turned every such request
+    into an error message about the caller's own API choice, on an async call the caller made
+    correctly. It also read the in-flight counts with a blocking call inside the event loop.
+    """
+    router = Router(
+        model_list=[
+            {
+                "model_name": "cached-claude",
+                "litellm_params": {
+                    "model": "anthropic_cache_control_hook/claude-sonnet-5",
+                    "prompt_id": "cache-points",
+                },
+            }
+        ],
+        routing_strategy="least-busy",
+        plugins=[BlockEverything()],
+    )
+
+    with pytest.raises(ValueError, match="No deployments left after routing-plugin filtering"):
+        await router.acompletion(
+            model="cached-claude",
+            messages=[{"role": "user", "content": "hi"}],
+            litellm_call_id="lit-7039",
+        )
+
+
 @pytest.mark.asyncio
 async def test_router_without_plugins_is_unaffected():
     """Regression guard: a Router with no `plugins` configured behaves exactly as before."""
diff --git a/type-discipline-budget.json b/type-discipline-budget.json
index e413e6db2c6..0c0952289e2 100644
--- a/type-discipline-budget.json
+++ b/type-discipline-budget.json
@@ -1,9 +1,9 @@
 {
   "LIT001": {
-    "limit": 22176
+    "limit": 22174
   },
   "LIT002": {
-    "limit": 26721
+    "limit": 26715
   },
   "LIT003": {
     "limit": 261
@@ -27,10 +27,10 @@
     "limit": 0
   },
   "LIT010": {
-    "limit": 16412
+    "limit": 16398
   },
   "LIT011": {
-    "limit": 5505
+    "limit": 5504
   },
   "LIT012": {
     "limit": 4486

From ef5f51abca77d3081d04796ce9b3191dc2409f9b Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sun, 6 Sep 2026 01:59:15 -0700
Subject: [PATCH 42/85] fix(least-busy): count for every router, and keep the
 expiry through a clamp

Two routers in one process shared a single handler, because the callback
manager dedupes on the class name plus the handler's public attributes and the
handler had none. The second router's requests were never counted. The handler
now carries the id of the cache it was built on, so routers with different
caches both register while the two selectors one router builds for its routing
groups still collapse into one.

Clamping a negative count back to zero used SET, which drops the key's TTL, so
the next write started the hour over. It uses INCRBY by the negative amount now,
which leaves the expiry alone.

The Lua script had no test that ran it, so tests/local_testing covers both the
sync and async paths against a real Redis, and the file is wired into the
CircleCI job that provides one.
---
 .circleci/config.yml                          |  1 +
 litellm/caching/redis_cache.py                |  2 +-
 litellm/router_strategy/least_busy.py         |  1 +
 .../test_redis_increment_with_floor.py        | 80 +++++++++++++++++++
 .../test_router_routing_groups.py             | 38 +++++++++
 5 files changed, 121 insertions(+), 1 deletion(-)
 create mode 100644 tests/local_testing/test_redis_increment_with_floor.py

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 6e368a3debe..d10962864bf 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -1440,6 +1440,7 @@ jobs:
             TEST_FILES=$(printf "%s\n" \
               tests/local_testing/test_dual_cache.py \
               tests/local_testing/test_redis_batch_optimizations.py \
+              tests/local_testing/test_redis_increment_with_floor.py \
               tests/local_testing/test_router_utils.py)
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
diff --git a/litellm/caching/redis_cache.py b/litellm/caching/redis_cache.py
index 3abc6e6f3c9..106c1580110 100644
--- a/litellm/caching/redis_cache.py
+++ b/litellm/caching/redis_cache.py
@@ -91,7 +91,7 @@ _BREAKER_GUARD_FRAME_NAMES: Final = frozenset(
 
 _INCREMENT_WITH_FLOOR_LUA: Final = (
     "local count = redis.call('INCRBY', KEYS[1], ARGV[1]) "
-    "if count < 0 then redis.call('SET', KEYS[1], 0) count = 0 end "
+    "if count < 0 then count = redis.call('INCRBY', KEYS[1], -count) end "
     "if redis.call('TTL', KEYS[1]) < 0 then redis.call('EXPIRE', KEYS[1], ARGV[2]) end "
     "return count"
 )
diff --git a/litellm/router_strategy/least_busy.py b/litellm/router_strategy/least_busy.py
index ab6d702bbc2..14e6592e1fd 100644
--- a/litellm/router_strategy/least_busy.py
+++ b/litellm/router_strategy/least_busy.py
@@ -106,6 +106,7 @@ class LeastBusyLoggingHandler(CustomLogger):
 
     def __init__(self, router_cache: DualCache):
         self.router_cache = router_cache
+        self.router_cache_id = str(id(router_cache))
 
     def log_pre_api_call(self, model: str, messages: object, kwargs: Mapping[str, object]) -> None:
         self._increment(kwargs, 1)
diff --git a/tests/local_testing/test_redis_increment_with_floor.py b/tests/local_testing/test_redis_increment_with_floor.py
new file mode 100644
index 00000000000..e358d5f31e0
--- /dev/null
+++ b/tests/local_testing/test_redis_increment_with_floor.py
@@ -0,0 +1,80 @@
+"""Least-busy routing keeps its in-flight counters in Redis, and the clamp at zero plus the
+create-once TTL both live inside a Lua script. Nothing but a real Redis runs that script, so
+these are the only tests that fail when the script itself is wrong."""
+
+import os
+import uuid
+from typing import Final
+
+import pytest
+from dotenv import load_dotenv
+
+load_dotenv()
+
+from litellm.caching.redis_cache import RedisCache
+
+TTL: Final = 600
+
+
+@pytest.fixture
+def counter():
+    cache: Final = RedisCache(host=os.getenv("REDIS_HOST"), port=os.getenv("REDIS_PORT"))
+    key: Final = f"lit7039-{uuid.uuid4()}"
+    yield cache, key, cache.check_and_fix_namespace(key=key)
+    cache.delete_cache(key)
+
+
+def test_a_counter_adds_every_increment_and_reads_back_what_it_holds(counter):
+    cache, key, _ = counter
+
+    assert cache.increment_with_floor(key, 3, TTL) == 3
+    assert cache.increment_with_floor(key, 2, TTL) == 5
+    assert cache.batch_get_counts([key]) == (5,)
+
+
+def test_a_decrement_past_zero_leaves_the_counter_at_zero(counter):
+    """A worker whose counter expired mid-request decrements a key that is no longer there.
+    Without the clamp that deployment reads negative, and least-busy pins every later request
+    on it until the count climbs back to zero."""
+    cache, key, _ = counter
+
+    assert cache.increment_with_floor(key, 1, TTL) == 1
+    assert cache.increment_with_floor(key, -5, TTL) == 0
+    assert cache.batch_get_counts([key]) == (0,)
+
+
+def test_traffic_never_pushes_a_counters_expiry_back_out(counter):
+    """The TTL is what releases a count whose worker died mid-request. Rewriting it on every
+    touch would keep that stuck count alive for as long as the group takes traffic."""
+    cache, key, namespaced_key = counter
+
+    cache.increment_with_floor(key, 1, TTL)
+    assert cache.redis_client.ttl(namespaced_key) > TTL - 60
+
+    cache.redis_client.expire(namespaced_key, 30)
+    cache.increment_with_floor(key, 1, TTL)
+
+    assert cache.redis_client.ttl(namespaced_key) <= 30
+
+
+def test_clamping_to_zero_keeps_the_expiry_it_already_had(counter):
+    cache, key, namespaced_key = counter
+
+    cache.increment_with_floor(key, 1, TTL)
+    cache.redis_client.expire(namespaced_key, 30)
+
+    assert cache.increment_with_floor(key, -5, TTL) == 0
+    assert cache.redis_client.ttl(namespaced_key) <= 30
+
+
+@pytest.mark.asyncio
+async def test_the_async_counter_behaves_the_same_way(counter):
+    cache, key, namespaced_key = counter
+
+    assert await cache.async_increment_with_floor(key, 2, TTL) == 2
+    assert await cache.async_batch_get_counts([key]) == (2,)
+
+    cache.redis_client.expire(namespaced_key, 30)
+
+    assert await cache.async_increment_with_floor(key, -9, TTL) == 0
+    assert cache.redis_client.ttl(namespaced_key) <= 30
diff --git a/tests/test_litellm/router_strategy/test_router_routing_groups.py b/tests/test_litellm/router_strategy/test_router_routing_groups.py
index 0ce0ed5b37e..af390c3292b 100644
--- a/tests/test_litellm/router_strategy/test_router_routing_groups.py
+++ b/tests/test_litellm/router_strategy/test_router_routing_groups.py
@@ -473,6 +473,44 @@ def test_two_least_busy_groups_count_a_request_once(monkeypatch):
     assert router.cache.get_cache("filtered-model_request_count:deploy-1") == 0
 
 
+def test_two_routers_in_one_process_each_count_their_own_requests(monkeypatch):
+    """
+    Least-busy hangs its counting off litellm's global callback lists, and those lists keep one
+    logger per class unless the instances differ in a plain attribute. Two routers in one process
+    (a second Router, or a per-request `user_config` one) therefore have to register separately:
+    a second router whose selector is dropped counts nothing, reads zero for every deployment,
+    and sends every request to whichever one is listed first.
+    """
+    monkeypatch.setattr(litellm, "callbacks", [])
+    monkeypatch.setattr(litellm, "input_callback", [])
+
+    first = _build_router(routing_strategy="least-busy")
+    second = _build_router(routing_strategy="least-busy")
+    kwargs = {
+        "litellm_params": {
+            "metadata": {"model_group": "filtered-model"},
+            "model_info": {"id": "deploy-1"},
+        }
+    }
+
+    for callback in litellm.input_callback:
+        if isinstance(callback, CustomLogger):
+            callback.log_pre_api_call(model="filtered-model", messages=[], kwargs=kwargs)
+
+    assert second.cache.get_cache("filtered-model_request_count:deploy-1") == 1
+    assert (
+        second.get_available_deployment(model="filtered-model", messages=[])["model_info"]["id"]
+        == "deploy-2"
+    )
+
+    for callback in litellm.callbacks:
+        if isinstance(callback, CustomLogger):
+            callback.log_success_event(kwargs, None, None, None)
+
+    assert first.cache.get_cache("filtered-model_request_count:deploy-1") == 0
+    assert second.cache.get_cache("filtered-model_request_count:deploy-1") == 0
+
+
 # ---------------------------------------------------------------------------
 # Direct helper coverage
 # ---------------------------------------------------------------------------

From 314e8905c583792d08207cdcd1bfa8e22d985f51 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Sun, 6 Sep 2026 02:44:45 -0700
Subject: [PATCH 43/85] fix(router): let prompt-management plugins see the
 caller's own messages

The prompt-management factory picks its deployment with a placeholder message. That
was inert while the pick ran on the synchronous path, which never runs the routing
plugin pipeline. Now that the pick runs the pipeline, a plugin classifying request
content would score the placeholder instead of the conversation, and the narrowing
it writes decides which deployments the real call may use.
---
 litellm/router.py                             |  2 +-
 .../test_router_routing_plugins.py            | 47 +++++++++++++++++++
 2 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/litellm/router.py b/litellm/router.py
index f5b7924fb58..398d87ce843 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -4142,7 +4142,7 @@ class Router:
         specific_deployment: Final = kwargs.pop("specific_deployment", None)
         prompt_management_deployment: Final = await self.async_get_available_deployment(
             model=model,
-            messages=[{"role": "user", "content": "prompt"}],
+            messages=cast(list[dict[str, str]], messages),  # cast-ok: selection reads messages structurally
             specific_deployment=specific_deployment,
             request_kwargs=kwargs,
         )
diff --git a/tests/test_litellm/router_strategy/test_router_routing_plugins.py b/tests/test_litellm/router_strategy/test_router_routing_plugins.py
index c49b22ea367..0a54addce5e 100644
--- a/tests/test_litellm/router_strategy/test_router_routing_plugins.py
+++ b/tests/test_litellm/router_strategy/test_router_routing_plugins.py
@@ -56,6 +56,18 @@ class BlockEverything:
         return context
 
 
+class MessageRecorder:
+    """Records what each plugin pass was handed, then blocks so the request stops there."""
+
+    def __init__(self):
+        self.seen = []
+
+    async def run(self, context: RoutingContext) -> RoutingContext:
+        self.seen.append(list(context.raw_messages))
+        context.candidate_models = []
+        return context
+
+
 def _smart_router_model_list():
     return [
         {
@@ -194,6 +206,41 @@ async def test_prompt_management_model_still_runs_the_plugin_pipeline():
         )
 
 
+@pytest.mark.asyncio
+async def test_prompt_management_plugins_see_the_callers_own_messages():
+    """
+    The prompt-management factory picks its deployment with a placeholder message, which was
+    harmless while that pick ran on the synchronous path (plugins never ran there at all). Now
+    that the pick runs the plugin pipeline, a plugin that classifies request content would score
+    the placeholder instead of the conversation, and the narrowing it produces decides which
+    deployments the real call is allowed to use.
+    """
+    recorder = MessageRecorder()
+    router = Router(
+        model_list=[
+            {
+                "model_name": "cached-claude",
+                "litellm_params": {
+                    "model": "anthropic_cache_control_hook/claude-sonnet-5",
+                    "prompt_id": "cache-points",
+                },
+            }
+        ],
+        routing_strategy="least-busy",
+        plugins=[recorder],
+    )
+    messages = [{"role": "user", "content": "wire me $40,000 to account 12345"}]
+
+    with pytest.raises(ValueError, match="No deployments left after routing-plugin filtering"):
+        await router.acompletion(
+            model="cached-claude",
+            messages=messages,
+            litellm_call_id="lit-7039",
+        )
+
+    assert recorder.seen == [messages]
+
+
 @pytest.mark.asyncio
 async def test_router_without_plugins_is_unaffected():
     """Regression guard: a Router with no `plugins` configured behaves exactly as before."""

From 4cc0180eab7482a59722e68999def58e11077a72 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 7 Sep 2026 17:06:34 -0700
Subject: [PATCH 44/85] fix(router): keep unresolved drop_params strings so DB
 rows and env refs survive

The drop_params validator collapsed every string it did not recognize to None. A pre-fix DB row holds the flag as ciphertext, so a partial PATCH rebuilt the deployment without it and dropped the key from the stored row, and /model/new turned an os.environ/ reference into nothing before the loader could resolve it. The validator now returns the raw value when it is not a boolean flag, the field admits strings the way timeout already does, and the flag set follows pydantic's lax bool parsing instead of a hand-rolled true/false pair
---
 litellm/litellm_core_utils/core_helpers.py    | 15 +++++-----
 litellm/types/router.py                       |  7 +++--
 .../litellm_core_utils/test_core_helpers.py   | 11 +++++--
 .../test_model_management_endpoints.py        | 27 ++++++++++++++++-
 .../proxy/proxy_server/test_proxy_config.py   | 30 +++++++++++++++++++
 tests/test_litellm/types/test_router.py       | 23 ++++++++++++++
 ui/litellm-dashboard/src/lib/http/schema.d.ts |  4 +--
 7 files changed, 101 insertions(+), 16 deletions(-)

diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py
index 671b2ddce99..bd7a1b8f384 100644
--- a/litellm/litellm_core_utils/core_helpers.py
+++ b/litellm/litellm_core_utils/core_helpers.py
@@ -5,6 +5,7 @@ from collections.abc import Iterable, Mapping
 from typing import TYPE_CHECKING, Any, Final, Literal
 
 import httpx
+from pydantic import TypeAdapter, ValidationError
 
 from litellm._logging import verbose_logger
 from litellm.types.llms.openai import AllMessageValues, OpenAIChatCompletionFinishReason
@@ -37,16 +38,16 @@ def safe_divide_seconds(seconds: float, denominator: float, default: float | Non
     return float(seconds / denominator)
 
 
+_DROP_PARAMS_BOOL: Final = TypeAdapter(bool)
+
+
 def normalize_drop_params(value: object) -> bool | None:
     if isinstance(value, bool):
         return value
-    if isinstance(value, str):
-        lowered: Final = value.strip().lower()
-        if lowered == "true":
-            return True
-        if lowered == "false":
-            return False
-    return None
+    try:
+        return _DROP_PARAMS_BOOL.validate_python(value.strip() if isinstance(value, str) else value)
+    except ValidationError:
+        return None
 
 
 def safe_divide(
diff --git a/litellm/types/router.py b/litellm/types/router.py
index e2a04f9da77..47aea6430c3 100644
--- a/litellm/types/router.py
+++ b/litellm/types/router.py
@@ -315,7 +315,7 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
     timeout: float | str | httpx.Timeout | None = None  # if str, pass in as os.environ/
     stream_timeout: float | str | None = None  # timeout when making stream=True calls, if str, pass in as os.environ/
     max_retries: int | None = None
-    drop_params: bool | None = None
+    drop_params: bool | str | None = None
     organization: str | None = None  # for openai orgs
     configurable_clientside_auth_params: CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS = None
     litellm_credential_name: str | None = None
@@ -408,8 +408,9 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
 
     @field_validator("drop_params", mode="before")
     @classmethod
-    def coerce_drop_params(cls, value: object) -> bool | None:
-        return normalize_drop_params(value)
+    def coerce_drop_params(cls, value: object) -> object:
+        normalized: Final = normalize_drop_params(value)
+        return value if normalized is None else normalized
 
     def __contains__(self, key) -> bool:
         # Define custom behavior for the 'in' operator
diff --git a/tests/test_litellm/litellm_core_utils/test_core_helpers.py b/tests/test_litellm/litellm_core_utils/test_core_helpers.py
index ab43141af23..8797f7c3591 100644
--- a/tests/test_litellm/litellm_core_utils/test_core_helpers.py
+++ b/tests/test_litellm/litellm_core_utils/test_core_helpers.py
@@ -268,11 +268,16 @@ class TestRedactNestedMatchAndRegexKeys:
         (" TRUE ", True),
         ("false", False),
         ("False", False),
+        ("yes", True),
+        ("off", False),
+        ("1", True),
+        (1, True),
+        (0, False),
         (None, None),
-        ("yes", None),
         ("", None),
-        (1, None),
-        (0, None),
+        ("os.environ/DROP_PARAMS", None),
+        ("v2:gcm:not-a-flag", None),
+        (2, None),
     ],
 )
 def test_normalize_drop_params(value, expected):
diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py
index d90338f8480..c02f886fc31 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py
@@ -30,7 +30,7 @@ from litellm.proxy.management_endpoints.model_management_endpoints import (
 )
 from litellm.proxy.utils import PrismaClient
 from litellm.router import Router
-from litellm.types.router import Deployment, LiteLLM_Params, updateDeployment
+from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo, updateDeployment, updateLiteLLMParams
 
 
 async def _passthrough_row(update_data):
@@ -3070,6 +3070,31 @@ class TestUpdateDBModelBlocked:
         assert "blocked" not in result
 
 
+class TestUpdateDBModelKeepsLegacyDropParams:
+    def test_partial_patch_keeps_encrypted_string_drop_params(self, monkeypatch):
+        from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper
+        from litellm.proxy.management_endpoints.model_management_endpoints import update_db_model
+
+        monkeypatch.setenv("LITELLM_SALT_KEY", "sk-1234")
+        legacy_row = Deployment(
+            model_name="gpt-5-nano",
+            litellm_params=LiteLLM_Params(
+                model="openai/gpt-5-nano",
+                api_key=encrypt_value_helper(value="sk-old"),
+                drop_params=encrypt_value_helper(value="true"),
+            ),
+            model_info=ModelInfo(id="legacy-row"),
+        )
+
+        result = update_db_model(
+            db_model=legacy_row,
+            updated_patch=updateDeployment(litellm_params=updateLiteLLMParams(api_key="sk-new")),
+        )
+
+        stored = json.loads(result["litellm_params"])
+        assert decrypt_value_helper(value=stored["drop_params"], key="drop_params") == "true"
+
+
 def _build_db_model_with_pricing():
     """Wildcard deployment with custom pricing in litellm_params; Deployment.__init__
     mirrors SPECIAL_MODEL_INFO_PARAMS into model_info, so both blobs hold the rate."""
diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py
index 2babfe432f3..a4be9574f89 100644
--- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py
+++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py
@@ -19,6 +19,7 @@ import pytest
 
 import litellm
 from litellm.proxy._types import CommonProxyErrors
+from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper
 from litellm.proxy.proxy_server import (
     ProxyConfig,
     _is_remote_module_url,
@@ -2401,6 +2402,35 @@ def test_ProxyConfig__add_deployment_resolves_env_refs_on_arbitrary_field(monkey
     assert deployment.litellm_params.some_future_field == "resolved-custom-value"
 
 
+@pytest.mark.parametrize(
+    "stored_drop_params",
+    ["true", "os.environ/DROP_PARAMS_FLAG"],
+)
+def test_ProxyConfig__add_deployment_turns_stored_drop_params_string_into_bool(monkeypatch, stored_drop_params):
+    monkeypatch.setenv("LITELLM_SALT_KEY", "sk-1234")
+    monkeypatch.setenv("DROP_PARAMS_FLAG", "true")
+    fake_router = MagicMock()
+    fake_router.upsert_deployment = MagicMock(return_value=True)
+    monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", fake_router)
+    pc = ProxyConfig()
+    db_model = SimpleNamespace(
+        model_id="model-1",
+        model_name="gpt-5-nano",
+        model_info={"id": "model-1"},
+        litellm_params={
+            "model": encrypt_value_helper(value="openai/gpt-5-nano"),
+            "drop_params": encrypt_value_helper(value=stored_drop_params),
+        },
+        blocked=False,
+    )
+
+    added = pc._add_deployment(db_models=[db_model])
+    deployment = fake_router.upsert_deployment.call_args.kwargs["deployment"]
+
+    assert added == 1
+    assert deployment.litellm_params.drop_params is True
+
+
 # ---------------------------------------------------------------------------
 # ProxyConfig.decrypt_model_list_from_db
 # ---------------------------------------------------------------------------
diff --git a/tests/test_litellm/types/test_router.py b/tests/test_litellm/types/test_router.py
index accd3b32a0d..2f4a29473c3 100644
--- a/tests/test_litellm/types/test_router.py
+++ b/tests/test_litellm/types/test_router.py
@@ -1,8 +1,10 @@
 import pytest
+from pydantic import ValidationError
 
 from litellm.types.router import (
     SPECIAL_MODEL_INFO_PARAMS,
     Deployment,
+    GenericLiteLLMParams,
     LiteLLM_Params,
     ModelInfo,
 )
@@ -89,3 +91,24 @@ def test_pricing_strings_are_coerced_to_float():
 def test_invalid_pricing_is_rejected():
     with pytest.raises(ValueError, match='validation error for ModelInfo'):
         ModelInfo(id="x", input_cost_per_token="free")
+
+
+@pytest.mark.parametrize(
+    "value, expected",
+    [
+        (True, True),
+        ("true", True),
+        (" False ", False),
+        ("yes", True),
+        (None, None),
+        ("os.environ/DROP_PARAMS", "os.environ/DROP_PARAMS"),
+        ("v2:gcm:ciphertext-from-a-pre-fix-row", "v2:gcm:ciphertext-from-a-pre-fix-row"),
+    ],
+)
+def test_drop_params_coerces_flags_and_keeps_unresolved_strings(value, expected):
+    assert GenericLiteLLMParams(drop_params=value).drop_params == expected
+
+
+def test_drop_params_rejects_non_flag_non_string_values():
+    with pytest.raises(ValidationError):
+        GenericLiteLLMParams(drop_params=2)
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index f7552bfb397..7b5edf6acfd 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -29397,7 +29397,7 @@ export interface components {
             /** Default Api Key Tpm Limit */
             default_api_key_tpm_limit?: number | null;
             /** Drop Params */
-            drop_params?: boolean | null;
+            drop_params?: boolean | string | null;
             /** Gcs Bucket Name */
             gcs_bucket_name?: string | null;
             /** Google Maps Grounding Cost Per Query */
@@ -39570,7 +39570,7 @@ export interface components {
             /** Default Api Key Tpm Limit */
             default_api_key_tpm_limit?: number | null;
             /** Drop Params */
-            drop_params?: boolean | null;
+            drop_params?: boolean | string | null;
             /** Gcs Bucket Name */
             gcs_bucket_name?: string | null;
             /** Google Maps Grounding Cost Per Query */

From 1067697c7b471200cd26ee93834f71b09d013415 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 7 Sep 2026 17:11:56 -0700
Subject: [PATCH 45/85] refactor(utils): gate the triton branch on
 bool(drop_params) like every other provider

---
 litellm/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/litellm/utils.py b/litellm/utils.py
index 59db4162410..50282674868 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -4302,7 +4302,7 @@ def get_optional_params(
             non_default_params=non_default_params,
             optional_params=optional_params,
             model=model,
-            drop_params=drop_params if drop_params is not None else False,
+            drop_params=bool(drop_params),
         )
 
     elif custom_llm_provider == "maritalk":

From 192ea9ec80ed97d771f9258320ac948080e87d2e Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 7 Sep 2026 17:55:37 -0700
Subject: [PATCH 46/85] fix(policy_engine): fail open on streaming shapes
 post_call pipelines cannot govern yet

A post_call pipeline now releases the original stream instead of refusing the
request on every shape it has no handler for: a background request, a pipeline
guardrail without the unified apply_guardrail interface, a route with no
endpoint translation, a buffered stream no translation resolves, and a rewrite
the translation cannot write back (tool-call edits, text edits on translations
without write-back, n>1 chat, an unended Anthropic stream, a Responses dump
with no event envelope). Each case logs a warning naming the policy and
guardrail. Real blocks and writable text masks are unchanged.
---
 .../chat/guardrail_translation/handler.py     |   3 +-
 .../guardrail_translation/base_translation.py |   5 +-
 .../chat/guardrail_translation/handler.py     |   5 +-
 .../guardrail_translation/handler.py          |   7 +-
 .../proxy/policy_engine/pipeline_executor.py  |  66 ++--
 litellm/proxy/utils.py                        | 226 ++++++--------
 .../policy_engine/test_pipeline_executor.py   | 137 ++++++++-
 .../proxy_logging/test_guardrail_pipeline.py  | 282 +++++++++++-------
 8 files changed, 437 insertions(+), 294 deletions(-)

diff --git a/litellm/llms/anthropic/chat/guardrail_translation/handler.py b/litellm/llms/anthropic/chat/guardrail_translation/handler.py
index da30d85e26b..e486be12fe2 100644
--- a/litellm/llms/anthropic/chat/guardrail_translation/handler.py
+++ b/litellm/llms/anthropic/chat/guardrail_translation/handler.py
@@ -1027,7 +1027,8 @@ class AnthropicMessagesHandler(BaseTranslation):
         Get the string so far, check the apply guardrail to the string so far, and return the list of responses so far.
         With ``deliver_ended_stream_rewrites``, an ended stream whose guardrail rewrote the text gets the rewrite
         written back across the buffered chunks (full rewritten text in the first ``text_delta``, the rest blanked);
-        a rewrite on a stream that never reported a ``stop_reason`` has no write-back and fails closed instead.
+        a rewrite on a stream that never reported a ``stop_reason`` has no write-back and is reported as
+        undeliverable, so the pipeline executor discards it and releases the original chunks.
         """
         from litellm.integrations.custom_guardrail import ModifyResponseException
 
diff --git a/litellm/llms/base_llm/guardrail_translation/base_translation.py b/litellm/llms/base_llm/guardrail_translation/base_translation.py
index 770fac6e443..afd8e0f67f7 100644
--- a/litellm/llms/base_llm/guardrail_translation/base_translation.py
+++ b/litellm/llms/base_llm/guardrail_translation/base_translation.py
@@ -56,8 +56,9 @@ class BaseTranslation(ABC):
     """Whether ``process_output_streaming_response`` accepts
     ``deliver_ended_stream_rewrites=True`` and, on an ended (fully buffered)
     stream, writes guardrail text rewrites back across ``responses_so_far`` so
-    a buffered pipeline can release rewritten chunks instead of withholding the
-    stream. Tool-call rewrites stay undeliverable everywhere."""
+    a buffered pipeline can release rewritten chunks. Tool-call rewrites, and
+    text rewrites on every other translation, are undeliverable: the pipeline
+    executor discards them and releases the original chunks."""
 
     @staticmethod
     def transform_user_api_key_dict_to_metadata(
diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py
index 4b7b1cc2700..80292aef2cf 100644
--- a/litellm/llms/openai/chat/guardrail_translation/handler.py
+++ b/litellm/llms/openai/chat/guardrail_translation/handler.py
@@ -1015,7 +1015,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         content-carrying chunk and the rest are blanked, the same shape the
         in-flight write-back uses. Chunks carrying only finish_reason or usage
         stay untouched. A rewrite on a stream carrying more than one distinct
-        choice index fails closed."""
+        choice index is reported as undeliverable, so the pipeline executor
+        discards it and releases the original chunks."""
         post_guardrail_texts: Final = self._string_choice_contents(guardrailed_response)
         changed: Final = tuple(
             after
@@ -1030,7 +1031,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         if len(stream_choice_indices) != 1:
             # stream_chunk_builder collapses every choice into one index-0
             # choice, so a rewrite of the rebuilt response cannot be attributed
-            # back to a single choice on an n>1 stream: withhold the stream
+            # back to a single choice on an n>1 stream: report it undeliverable
             # rather than deliver the rewrite on the wrong choice
             from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
 
diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py
index b543c7f1e17..b0f79552bc5 100644
--- a/litellm/llms/openai/responses/guardrail_translation/handler.py
+++ b/litellm/llms/openai/responses/guardrail_translation/handler.py
@@ -699,7 +699,8 @@ class OpenAIResponsesHandler(BaseTranslation):
         ``response.content_part.done``, ``response.output_item.done``) are synced
         to the rewritten envelope too, so a client reading deltas sees the
         rewrite instead of the raw model output; a rewrite observed where no
-        write-back is possible fails closed instead of releasing raw output.
+        write-back is possible is reported as undeliverable, so the pipeline
+        executor discards it and releases the original events.
         """
         if not responses_so_far:
             return responses_so_far
@@ -788,7 +789,7 @@ class OpenAIResponsesHandler(BaseTranslation):
         # ------------------------------------------------------------------ #
         # Case 2: response.output_item.done — extract tool calls only, then  #
         # fall through to the text fallback when a caller expects rewrites   #
-        # delivered, so a buffer truncated here still fails closed on text.  #
+        # delivered, so a truncated buffer still reports text undeliverable. #
         # ------------------------------------------------------------------ #
         if final_chunk.get("type") == "response.output_item.done":
             model_response_stream: Final = (
@@ -813,7 +814,7 @@ class OpenAIResponsesHandler(BaseTranslation):
         # Fallback: apply guardrail to the accumulated text string.           #
         # No structured write-back is possible here; guardrails that only     #
         # need to block/flag (not rewrite) still work correctly, and a        #
-        # rewrite a caller expects delivered fails closed instead.            #
+        # rewrite a caller expects delivered is reported undeliverable.       #
         # ------------------------------------------------------------------ #
         string_so_far: Final = self.get_streaming_string_so_far(responses_so_far)
         if string_so_far:
diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py
index 160938b723c..970ac20487c 100644
--- a/litellm/proxy/policy_engine/pipeline_executor.py
+++ b/litellm/proxy/policy_engine/pipeline_executor.py
@@ -5,6 +5,7 @@ Runs guardrails sequentially per pipeline step definitions, handling
 pass/fail actions (allow, block, next, modify_response) and data forwarding.
 """
 
+import copy
 import time
 from collections.abc import Mapping, Sequence
 from typing import TYPE_CHECKING, Any, Final, Literal
@@ -77,7 +78,7 @@ class _StreamRewriteObserver(CustomGuardrail):
     which for guardrails like Bedrock's ANONYMIZED action is only known at runtime. Text
     rewrites are deliverable on translations that write them back across the buffered chunks
     (``delivers_ended_stream_text_rewrites``); tool-call rewrites and text rewrites on any
-    other translation make the gate withhold the stream."""
+    other translation are discarded by the executor, which releases the original chunks."""
 
     def __init__(self, inner: CustomGuardrail) -> None:
         super().__init__(guardrail_name=inner.guardrail_name)
@@ -133,6 +134,19 @@ def _prepare_hook_input(
     return hook_input, scans_raw_request
 
 
+def _release_original_chunks(
+    guardrail_name: str,
+    streaming_chunks: list[object],  # mutable-ok: shared buffered-stream chunks, restored in place
+    originals: Sequence[object],
+) -> None:
+    streaming_chunks[:] = originals  # rebind-ok: the caller's buffer is the stream the client receives
+    verbose_proxy_logger.warning(
+        "Pipeline: guardrail '%s' rewrote the streamed response in a way this endpoint's streaming "
+        "pipeline cannot deliver yet; the rewrite was discarded and the original stream released",
+        guardrail_name,
+    )
+
+
 class PipelineExecutor:
     """Executes guardrail pipelines with ordered, conditional step logic."""
 
@@ -263,29 +277,37 @@ class PipelineExecutor:
         litellm_logging_obj: "LiteLLMLoggingObj | None",
     ) -> None:
         """Run one streaming post_call step through the endpoint translation, delivering
-        text rewrites on translations that support ended-stream write-back and raising
-        ``UndeliverableStreamRewrite`` for any rewrite that cannot reach the client."""
+        text rewrites on translations that support ended-stream write-back. A rewrite that
+        cannot reach the client yet (a tool-call rewrite, a text rewrite on a translation
+        without write-back, or one the translation refused with
+        ``UndeliverableStreamRewrite``) is discarded: the buffered chunks go back to the
+        originals and the step passes, so the client gets the stream the merge base sent."""
         observer: Final = _StreamRewriteObserver(callback)
         deliver_rewrites: Final = type(endpoint_translation).delivers_ended_stream_text_rewrites
-        if deliver_rewrites:
-            await endpoint_translation.process_output_streaming_response(
-                responses_so_far=streaming_chunks,
-                guardrail_to_apply=observer,
-                litellm_logging_obj=litellm_logging_obj,
-                user_api_key_dict=user_api_key_dict,
-                request_data=hook_input,
-                deliver_ended_stream_rewrites=True,
-            )
-        else:
-            await endpoint_translation.process_output_streaming_response(
-                responses_so_far=streaming_chunks,
-                guardrail_to_apply=observer,
-                litellm_logging_obj=litellm_logging_obj,
-                user_api_key_dict=user_api_key_dict,
-                request_data=hook_input,
-            )
+        originals: Final = copy.deepcopy(streaming_chunks)
+        try:
+            if deliver_rewrites:
+                await endpoint_translation.process_output_streaming_response(
+                    responses_so_far=streaming_chunks,
+                    guardrail_to_apply=observer,
+                    litellm_logging_obj=litellm_logging_obj,
+                    user_api_key_dict=user_api_key_dict,
+                    request_data=hook_input,
+                    deliver_ended_stream_rewrites=True,
+                )
+            else:
+                await endpoint_translation.process_output_streaming_response(
+                    responses_so_far=streaming_chunks,
+                    guardrail_to_apply=observer,
+                    litellm_logging_obj=litellm_logging_obj,
+                    user_api_key_dict=user_api_key_dict,
+                    request_data=hook_input,
+                )
+        except UndeliverableStreamRewrite:
+            _release_original_chunks(step.guardrail, streaming_chunks, originals)
+            return
         if observer.rewrote_tool_calls or (observer.rewrote_texts and not deliver_rewrites):
-            raise UndeliverableStreamRewrite(step.guardrail)
+            _release_original_chunks(step.guardrail, streaming_chunks, originals)
 
     @staticmethod
     async def _run_step(
@@ -386,8 +408,6 @@ class PipelineExecutor:
                 )  # mutable-ok: modified-data contract is a plain dict
             return ("pass", response if isinstance(response, dict) else None, None, None)
 
-        except UndeliverableStreamRewrite:
-            raise
         except Exception as e:
             if CustomGuardrail._is_guardrail_intervention(e):
                 error_msg: Final = _extract_error_message(e)
diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index ca8e48b5dfa..d7bf832bca4 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -19,7 +19,7 @@ from email.mime.text import MIMEText
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, Protocol, TypeVar, Union, cast, overload
 
-from typing_extensions import NotRequired, ReadOnly, TypedDict
+from typing_extensions import ReadOnly, TypedDict
 
 from litellm import _custom_logger_compatible_callbacks_literal
 from litellm.constants import (
@@ -155,7 +155,7 @@ from litellm.proxy.hooks.sensitive_data_routing import (
 )
 from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup, add_guardrails_from_auth_metadata
 from litellm.proxy.management_helpers.key_settings_audit import with_settings_updated_at
-from litellm.proxy.policy_engine.pipeline_executor import PipelineExecutor, UndeliverableStreamRewrite
+from litellm.proxy.policy_engine.pipeline_executor import PipelineExecutor
 from litellm.repositories.budget_repository import BudgetRepository
 from litellm.repositories.config_repository import ConfigRepository
 from litellm.repositories.table_repositories import (
@@ -518,108 +518,72 @@ def _pipeline_step_supports_unified_streaming(guardrail_name: str) -> bool:
     return callback is not None and PipelineExecutor.supports_unified_execution(callback)
 
 
-class _PipelineErrorBody(TypedDict):
-    message: ReadOnly[str]
-    type: ReadOnly[str]
-    policies: ReadOnly[tuple[str, ...]]
-    guardrails: NotRequired[ReadOnly[tuple[str, ...]]]
-
-
-class _PipelineErrorDetail(TypedDict):
-    error: ReadOnly[_PipelineErrorBody]
-
-
-def _undeliverable_stream_rewrite_error(policy_name: str, guardrail_name: str) -> HTTPException:
-    detail: Final[_PipelineErrorDetail] = {
-        "error": {
-            "message": (
-                f"Streaming response withheld by policy pipeline '{policy_name}' because guardrail "
-                f"'{guardrail_name}' rewrote the streamed output in a way this endpoint's streaming "
-                "pipeline cannot deliver (a tool-call rewrite, or a text rewrite on a route without "
-                "stream write-back). Retry with stream=false, or drop it from the pipeline steps so "
-                "guardrails.add applies it to streamed output."
-            ),
-            "type": "guardrail_pipeline_error",
-            "policies": (policy_name,),
-            "guardrails": (guardrail_name,),
-        }
-    }
-    return HTTPException(status_code=400, detail=detail)
-
-
-def _raise_for_streaming_post_call_pipelines(data: Mapping[str, object], user_api_key_dict: UserAPIKeyAuth) -> None:
-    """
-    Reject up front the requests whose post_call pipelines could never run.
-
-    Background responses skip the post_call hooks entirely, so a pipeline
-    governing one would silently never execute. Streaming responses execute
-    pipelines against the buffered stream through the endpoint guardrail
-    translation of the request route, releasing the buffered chunks on allow
-    (rewritten in place when a guardrail rewrote text and the translation
-    delivers ended-stream rewrites; a rewrite the translation cannot deliver
-    fails closed at runtime instead). That needs every step's guardrail to
-    support the unified apply_guardrail interface, and needs the route to have
-    a translation at all; anything else keeps the 400 rather than letting
-    ungoverned output stream through.
-    """
-    is_stream: Final = data.get("stream") is True
-    is_background: Final = data.get("background") is True
-    if not is_stream and not is_background:
-        return
-    post_call_pipelines: Final = tuple(
+def _post_call_pipelines(data: Mapping[str, object]) -> tuple[tuple[str, "GuardrailPipeline"], ...]:
+    return tuple(
         (policy_name, pipeline) for policy_name, pipeline in _policy_pipelines(data) if pipeline.mode == "post_call"
     )
+
+
+def _warn_background_skips_post_call_pipelines(data: Mapping[str, object]) -> None:
+    if data.get("background") is not True:
+        return
+    policy_names: Final = tuple(policy_name for policy_name, _pipeline in _post_call_pipelines(data))
+    if not policy_names:
+        return
+    verbose_proxy_logger.warning(
+        "Policies with post_call guardrail pipelines do not run on background responses yet; "
+        "the response is released ungoverned by them: %s",
+        ", ".join(policy_names),
+    )
+
+
+def _pipeline_is_streamable(policy_name: str, pipeline: "GuardrailPipeline") -> bool:
+    unsupported: Final = tuple(
+        dict.fromkeys(
+            step.guardrail for step in pipeline.steps if not _pipeline_step_supports_unified_streaming(step.guardrail)
+        )
+    )
+    if not unsupported:
+        return True
+    verbose_proxy_logger.warning(
+        "Policy '%s' has post_call pipeline guardrails without the unified apply_guardrail interface, "
+        "which streaming pipelines need; the stream is released ungoverned by it: %s",
+        policy_name,
+        ", ".join(unsupported),
+    )
+    return False
+
+
+def _streamable_post_call_pipelines(
+    request_data: Mapping[str, object], user_api_key_dict: UserAPIKeyAuth
+) -> tuple[tuple[str, "GuardrailPipeline"], ...]:
+    """
+    The post_call pipelines a streaming response can be gated through.
+
+    Streaming pipelines scan the buffered stream through the endpoint guardrail
+    translation of the request route, so every step's guardrail needs the
+    unified apply_guardrail interface and the route needs a translation. A
+    pipeline that cannot be run that way yet is left out and the stream is
+    released the way it was before pipelines ran on streams at all, with a
+    warning naming what went ungoverned.
+    """
+    post_call_pipelines: Final = _post_call_pipelines(request_data)
     if not post_call_pipelines:
-        return
-    post_call_policies: Final = tuple(policy_name for policy_name, _pipeline in post_call_pipelines)
-    if is_background:
-        background_detail: Final[_PipelineErrorDetail] = {
-            "error": {
-                "message": (
-                    "Policies with post_call guardrail pipelines cannot govern background "
-                    f"responses: {', '.join(post_call_policies)}. Retry with background=false."
-                ),
-                "type": "guardrail_pipeline_error",
-                "policies": post_call_policies,
-            }
-        }
-        raise HTTPException(status_code=400, detail=background_detail)
-    step_guardrails: Final = tuple(
-        dict.fromkeys(step.guardrail for _policy_name, pipeline in post_call_pipelines for step in pipeline.steps)
-    )
-    unsupported_guardrails: Final = tuple(
-        guardrail for guardrail in step_guardrails if not _pipeline_step_supports_unified_streaming(guardrail)
-    )
-    if unsupported_guardrails:
-        unsupported_detail: Final[_PipelineErrorDetail] = {
-            "error": {
-                "message": (
-                    "Policies with post_call guardrail pipelines cannot govern streaming responses "
-                    "because these pipeline guardrails do not support the unified apply_guardrail "
-                    f"interface: {', '.join(unsupported_guardrails)}. Retry with stream=false, or drop "
-                    "them from the pipeline steps so guardrails.add scans them on streamed output."
-                ),
-                "type": "guardrail_pipeline_error",
-                "policies": post_call_policies,
-                "guardrails": unsupported_guardrails,
-            }
-        }
-        raise HTTPException(status_code=400, detail=unsupported_detail)
+        return ()
     route: Final = user_api_key_dict.request_route
-    if not route or resolve_endpoint_translation(user_api_key_dict, None) is not None:
-        return
-    route_detail: Final[_PipelineErrorDetail] = {
-        "error": {
-            "message": (
-                "Policies with post_call guardrail pipelines cannot govern streaming responses on "
-                f"route {route} because it has no endpoint guardrail translation to scan the stream "
-                f"through: {', '.join(post_call_policies)}. Retry with stream=false."
-            ),
-            "type": "guardrail_pipeline_error",
-            "policies": post_call_policies,
-        }
-    }
-    raise HTTPException(status_code=400, detail=route_detail)
+    if route and resolve_endpoint_translation(user_api_key_dict, None) is None:
+        verbose_proxy_logger.warning(
+            "Policies with post_call guardrail pipelines cannot scan streaming responses on route %s yet "
+            "(no endpoint guardrail translation); the stream is released ungoverned by them: %s",
+            route,
+            ", ".join(policy_name for policy_name, _pipeline in post_call_pipelines),
+        )
+        return ()
+    return tuple(
+        (policy_name, pipeline)
+        for policy_name, pipeline in post_call_pipelines
+        if _pipeline_is_streamable(policy_name, pipeline)
+    )
 
 
 def _prompt_block_text(block: object) -> str:
@@ -1990,7 +1954,7 @@ class ProxyLogging:
         )
 
         try:
-            _raise_for_streaming_post_call_pipelines(data, user_api_key_dict)
+            _warn_background_skips_post_call_pipelines(data)
 
             # Execute guardrail pipelines before the normal callback loop
             data, _ = await self._maybe_execute_pipelines(  # rebind-ok: pipeline edits feed the callback loop below
@@ -3371,11 +3335,7 @@ class ProxyLogging:
         1. /chat/completions
         """
         caps: Final = ProxyLogging._callback_capabilities()
-        post_call_pipelines: Final = tuple(
-            (policy_name, pipeline)
-            for policy_name, pipeline in _policy_pipelines(request_data)
-            if pipeline.mode == "post_call"
-        )
+        post_call_pipelines: Final = _streamable_post_call_pipelines(request_data, user_api_key_dict)
         # Fast path: no real overrides. Internal proxy CustomLogger callbacks
         # (e.g. _PROXY_MaxBudgetLimiter, ManagedFiles) inherit the default
         # ``async for chunk: yield chunk`` body, so wrapping the iterator
@@ -3486,9 +3446,11 @@ class ProxyLogging:
         output, rewritten in place when one rewrote text and the translation
         delivers ended-stream rewrites (later steps then re-scan the rewritten
         chunks, so rewrites chain). A rewrite the translation cannot deliver
-        (a tool-call rewrite, or a text rewrite on a route without write-back)
-        withholds the stream with a 400; a block or modify_response terminates
-        with the translation's block chunks or the raised error.
+        yet (a tool-call rewrite, or a text rewrite on a route without
+        write-back) is discarded by the executor and the original chunks are
+        released, as is a buffered shape no translation resolves; a block or
+        modify_response terminates with the translation's block chunks or the
+        raised error.
         """
         buffered: Final[list[object]] = []  # mutable-ok: accumulates the stream before the pipeline verdict
         async for item in response:
@@ -3498,39 +3460,27 @@ class ProxyLogging:
 
         resolved: Final = resolve_endpoint_translation(user_api_key_dict, buffered[0])
         if resolved is None:
-            policy_names: Final = tuple(policy_name for policy_name, _pipeline in pipelines)
-            raise ProxyException(
-                message=(
-                    "Policy pipelines could not govern this streaming response shape; "
-                    f"the response was withheld: {', '.join(policy_names)}."
-                ),
-                type="guardrail_pipeline_error",
-                param=None,
-                code=500,
+            verbose_proxy_logger.warning(
+                "Policies with post_call guardrail pipelines cannot scan this streaming response shape yet; "
+                "the stream is released ungoverned by them: %s",
+                ", ".join(policy_name for policy_name, _pipeline in pipelines),
             )
+            for buffered_item in buffered:
+                yield buffered_item
+            return
         call_type, endpoint_translation = resolved
 
         for policy_name, pipeline in pipelines:
-            try:
-                result: PipelineExecutionResult = await PipelineExecutor.execute_steps(
-                    steps=pipeline.steps,
-                    mode="post_call",
-                    data=request_data,
-                    user_api_key_dict=user_api_key_dict,
-                    call_type=call_type,
-                    policy_name=policy_name,
-                    streaming_chunks=buffered,
-                    endpoint_translation=endpoint_translation,
-                )
-            except UndeliverableStreamRewrite as rewrite:
-                async for error_chunk in unified_guardrail.emit_streaming_http_error(
-                    _undeliverable_stream_rewrite_error(policy_name, rewrite.guardrail_name),
-                    call_type,
-                    buffered,
-                    request_data,
-                ):
-                    yield error_chunk
-                return
+            result: PipelineExecutionResult = await PipelineExecutor.execute_steps(
+                steps=pipeline.steps,
+                mode="post_call",
+                data=request_data,
+                user_api_key_dict=user_api_key_dict,
+                call_type=call_type,
+                policy_name=policy_name,
+                streaming_chunks=buffered,
+                endpoint_translation=endpoint_translation,
+            )
             try:
                 ProxyLogging._handle_pipeline_result(
                     result, data=request_data, policy_name=policy_name, original_response=buffered
diff --git a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py
index d399b4f01cb..0685bc6aa1e 100644
--- a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py
+++ b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py
@@ -4,6 +4,7 @@ Tests for the pipeline executor.
 Uses mock guardrails to validate pipeline execution without external services.
 """
 
+import logging
 from unittest.mock import MagicMock
 
 import pytest
@@ -941,7 +942,55 @@ class _TextTranslation:
         return responses_so_far
 
 
-async def _run_streaming_step(returned_texts, translation):
+class _WritingTranslation:
+    """Writes the guardrail's text (and tool-call) outputs back into the buffered chunks the way the
+    chat/Responses/Messages handlers do on an ended stream."""
+
+    delivers_ended_stream_text_rewrites = True
+
+    async def process_output_streaming_response(
+        self,
+        responses_so_far,
+        guardrail_to_apply,
+        litellm_logging_obj=None,
+        user_api_key_dict=None,
+        request_data=None,
+        deliver_ended_stream_rewrites=False,
+    ):
+        assert deliver_ended_stream_rewrites is True
+        outputs = await guardrail_to_apply.apply_guardrail(
+            inputs={"texts": [responses_so_far[0]["text"]], "tool_calls": [dict(responses_so_far[0]["tool_call"])]},
+            request_data=request_data or {},
+            input_type="response",
+            logging_obj=litellm_logging_obj,
+        )
+        responses_so_far[0]["text"] = outputs["texts"][0]
+        responses_so_far[0]["tool_call"] = outputs["tool_calls"][0]
+        return responses_so_far
+
+
+class _RefusingTranslation:
+    delivers_ended_stream_text_rewrites = True
+
+    async def process_output_streaming_response(
+        self,
+        responses_so_far,
+        guardrail_to_apply,
+        litellm_logging_obj=None,
+        user_api_key_dict=None,
+        request_data=None,
+        deliver_ended_stream_rewrites=False,
+    ):
+        responses_so_far[0]["text"] = "half-written"
+        raise UndeliverableStreamRewrite(guardrail_to_apply.guardrail_name)
+
+
+def _chunk():
+    return {"text": "hello world", "tool_call": {"function": {"name": "lookup", "arguments": '{"ssn": "123"}'}}}
+
+
+async def _run_streaming_step(translation, streaming_chunks=None):
+    chunks = [object()] if streaming_chunks is None else streaming_chunks
     return await PipelineExecutor.execute_steps(
         steps=[PipelineStep(guardrail="masker", on_pass="allow", on_fail="next", on_error="next")],
         mode="post_call",
@@ -949,31 +998,41 @@ async def _run_streaming_step(returned_texts, translation):
         user_api_key_dict=MagicMock(),
         call_type="completion",
         policy_name="p",
-        streaming_chunks=[object()],
+        streaming_chunks=chunks,
         endpoint_translation=translation,
     )
 
 
+def _assert_passed_with_discard_warning(result, caplog):
+    assert result.terminal_action == "allow"
+    assert [step.outcome for step in result.step_results] == ["pass"]
+    assert any("'masker'" in record.getMessage() and "discarded" in record.getMessage() for record in caplog.records)
+
+
 @pytest.mark.asyncio
-async def test_streaming_step_rewrite_escapes_execute_steps_regardless_of_step_actions(monkeypatch):
+async def test_streaming_step_discards_text_rewrite_when_translation_lacks_write_back(monkeypatch, caplog):
     monkeypatch.setattr(litellm, "callbacks", [_TextReturningGuardrail(["hello [MASKED]"])])
     translation = _TextTranslation()
+    chunks = [_chunk()]
 
-    with pytest.raises(UndeliverableStreamRewrite) as info:
-        await _run_streaming_step(["hello [MASKED]"], translation)
+    with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
+        result = await _run_streaming_step(translation, chunks)
 
-    assert info.value.guardrail_name == "masker"
+    _assert_passed_with_discard_warning(result, caplog)
+    assert chunks == [_chunk()]
     assert translation.seen_guardrail_names == ["masker"]
 
 
 @pytest.mark.asyncio
-async def test_streaming_step_unchanged_texts_in_another_container_allow(monkeypatch):
+async def test_streaming_step_unchanged_texts_in_another_container_allow(monkeypatch, caplog):
     monkeypatch.setattr(litellm, "callbacks", [_TextReturningGuardrail(("hello world",))])
 
-    result = await _run_streaming_step(("hello world",), _TextTranslation())
+    with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
+        result = await _run_streaming_step(_TextTranslation())
 
     assert result.terminal_action == "allow"
     assert [step.outcome for step in result.step_results] == ["pass"]
+    assert not any("discarded" in record.getMessage() for record in caplog.records)
 
 
 class _InPlaceMutatingGuardrail(CustomGuardrail):
@@ -989,10 +1048,64 @@ class _InPlaceMutatingGuardrail(CustomGuardrail):
 
 
 @pytest.mark.asyncio
-async def test_streaming_step_in_place_rewrite_still_withholds_stream(monkeypatch):
+async def test_streaming_step_in_place_rewrite_is_discarded_without_write_back(monkeypatch, caplog):
     monkeypatch.setattr(litellm, "callbacks", [_InPlaceMutatingGuardrail()])
+    chunks = [_chunk()]
 
-    with pytest.raises(UndeliverableStreamRewrite) as info:
-        await _run_streaming_step(["hello [MASKED]"], _TextTranslation())
+    with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
+        result = await _run_streaming_step(_TextTranslation(), chunks)
 
-    assert info.value.guardrail_name == "masker"
+    _assert_passed_with_discard_warning(result, caplog)
+    assert chunks == [_chunk()]
+
+
+class _TextAndToolCallRewritingGuardrail(CustomGuardrail):
+    def __init__(self, rewrite_tool_call):
+        super().__init__(guardrail_name="masker", event_hook="post_call", default_on=True)
+        self.rewrite_tool_call = rewrite_tool_call
+
+    async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None):
+        tool_calls = (
+            [{"function": {"name": "lookup", "arguments": '{"ssn": "[MASKED]"}'}}]
+            if self.rewrite_tool_call
+            else inputs["tool_calls"]
+        )
+        return {**inputs, "texts": ["hello [MASKED]"], "tool_calls": tool_calls}
+
+
+@pytest.mark.asyncio
+async def test_streaming_step_delivers_text_rewrite_through_writing_translation(monkeypatch, caplog):
+    monkeypatch.setattr(litellm, "callbacks", [_TextAndToolCallRewritingGuardrail(rewrite_tool_call=False)])
+    chunks = [_chunk()]
+
+    with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
+        result = await _run_streaming_step(_WritingTranslation(), chunks)
+
+    assert result.terminal_action == "allow"
+    assert chunks[0]["text"] == "hello [MASKED]"
+    assert chunks[0]["tool_call"]["function"]["arguments"] == '{"ssn": "123"}'
+    assert not any("discarded" in record.getMessage() for record in caplog.records)
+
+
+@pytest.mark.asyncio
+async def test_streaming_step_discards_tool_call_rewrite_and_restores_written_text(monkeypatch, caplog):
+    monkeypatch.setattr(litellm, "callbacks", [_TextAndToolCallRewritingGuardrail(rewrite_tool_call=True)])
+    chunks = [_chunk()]
+
+    with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
+        result = await _run_streaming_step(_WritingTranslation(), chunks)
+
+    _assert_passed_with_discard_warning(result, caplog)
+    assert chunks == [_chunk()]
+
+
+@pytest.mark.asyncio
+async def test_streaming_step_restores_chunks_when_translation_refuses_the_rewrite(monkeypatch, caplog):
+    monkeypatch.setattr(litellm, "callbacks", [_TextReturningGuardrail(["hello [MASKED]"])])
+    chunks = [_chunk()]
+
+    with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
+        result = await _run_streaming_step(_RefusingTranslation(), chunks)
+
+    _assert_passed_with_discard_warning(result, caplog)
+    assert chunks == [_chunk()]
diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
index 2e73bebb07c..ad2b4a0efbb 100644
--- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
+++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
@@ -11,6 +11,7 @@ from __future__ import annotations
 
 import asyncio
 import json
+import logging
 from typing import Any, Callable, Dict, List
 from unittest.mock import AsyncMock, MagicMock, patch
 
@@ -24,9 +25,9 @@ from litellm.integrations.custom_guardrail import (
     ModifyResponseException,
 )
 from litellm.integrations.prometheus import PrometheusLogger
-from litellm.proxy._types import ProxyException, UserAPIKeyAuth
+from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.common_utils.callback_utils import add_guardrail_to_applied_guardrails_header
-from litellm.proxy.utils import ProxyLogging, _raise_for_streaming_post_call_pipelines
+from litellm.proxy.utils import ProxyLogging, _streamable_post_call_pipelines
 from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import ContentFilterGuardrail
 from litellm.types.guardrails import BlockedWord, ContentFilterAction, GuardrailEventHooks
 from litellm.types.proxy.policy_engine.pipeline_types import (
@@ -1384,76 +1385,87 @@ async def test_pre_call_pipeline_managed_parallel_guardrail_runs_exactly_once(
     assert seen["count"] == 1
 
 
+def _warnings(caplog: pytest.LogCaptureFixture) -> List[str]:
+    return [record.getMessage() for record in caplog.records if record.levelno >= logging.WARNING]
+
+
 @pytest.mark.asyncio
-async def test_pre_call_hook_rejects_streaming_request_with_post_call_pipeline(
-    proxy_logging, make_user_api_key_auth, monkeypatch
+async def test_streaming_request_whose_pipeline_guardrail_is_missing_streams_verbatim(
+    proxy_logging, make_user_api_key_auth, monkeypatch, caplog
 ):
     monkeypatch.setattr(litellm, "callbacks", [])
     data = _post_call_pipeline_data(stream=True)
+    chunks = _stream_chunks()
+    delivered: List[Any] = []
 
-    with pytest.raises(HTTPException) as info:
-        await proxy_logging.pre_call_hook(
+    with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
+        out = await proxy_logging.pre_call_hook(
             user_api_key_dict=make_user_api_key_auth(),
             data=data,
             call_type="completion",
             guardrails_only=True,
         )
+        async for item in proxy_logging.async_post_call_streaming_iterator_hook(
+            user_api_key_dict=make_user_api_key_auth(request_route="/v1/chat/completions"),
+            response=_async_chunk_iter(chunks),
+            request_data=data,
+        ):
+            delivered.append(item)
 
-    assert info.value.status_code == 400
-    assert info.value.detail["error"]["policies"] == ("response-governance",)
-    assert info.value.detail["error"]["guardrails"] == ("gr-post",)
-    assert "stream=false" in info.value.detail["error"]["message"]
+    assert out is not None
+    assert out.get("stream") is True
+    assert [item is chunk for item, chunk in zip(delivered, chunks)] == [True, True]
+    assert len(delivered) == 2
+    assert any("response-governance" in message and "gr-post" in message for message in _warnings(caplog))
 
 
 @pytest.mark.asyncio
-async def test_pre_call_hook_rejects_background_request_with_post_call_pipeline(
-    proxy_logging, make_user_api_key_auth, monkeypatch
+async def test_pre_call_hook_accepts_background_request_with_post_call_pipeline(
+    proxy_logging, make_user_api_key_auth, monkeypatch, caplog
 ):
     monkeypatch.setattr(litellm, "callbacks", [])
     data = _post_call_pipeline_data(background=True)
 
-    with pytest.raises(HTTPException) as info:
-        await proxy_logging.pre_call_hook(
+    with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
+        out = await proxy_logging.pre_call_hook(
             user_api_key_dict=make_user_api_key_auth(),
             data=data,
             call_type="aresponses",
             guardrails_only=True,
         )
 
-    assert info.value.status_code == 400
-    assert info.value.detail["error"]["policies"] == ("response-governance",)
-    assert "background=false" in info.value.detail["error"]["message"]
+    assert out is not None
+    assert out.get("background") is True
+    assert any("response-governance" in message and "background" in message for message in _warnings(caplog))
 
 
-def test_raise_for_streaming_post_call_pipelines_ignores_non_streaming_and_pre_call(make_user_api_key_auth):
-    post_call = GuardrailPipeline(mode="post_call", steps=[PipelineStep(guardrail="g", on_fail="block")])
-    pre_call = GuardrailPipeline(mode="pre_call", steps=[PipelineStep(guardrail="g", on_fail="block")])
-    auth = make_user_api_key_auth(request_route="/custom/stream")
+@pytest.mark.asyncio
+async def test_pre_call_hook_stays_quiet_on_background_request_without_post_call_pipeline(
+    proxy_logging, make_user_api_key_auth, monkeypatch, caplog
+):
+    seen: Dict[str, Any] = {}
+    monkeypatch.setattr(litellm, "callbacks", [_unified_stream_guardrail(seen)])
+    pre_call = GuardrailPipeline(mode="pre_call", steps=[PipelineStep(guardrail="gr-post", on_fail="block")])
+    data = {
+        "model": "m",
+        "messages": [{"role": "user", "content": "hi"}],
+        "background": True,
+        "metadata": {
+            "_guardrail_pipelines": [("request-governance", pre_call)],
+            "_pipeline_managed_guardrails": {"gr-post"},
+        },
+    }
 
-    assert (
-        _raise_for_streaming_post_call_pipelines(
-            {"stream": False, "metadata": {"_guardrail_pipelines": [("p", post_call)]}}, auth
+    with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
+        out = await proxy_logging.pre_call_hook(
+            user_api_key_dict=make_user_api_key_auth(),
+            data=data,
+            call_type="aresponses",
+            guardrails_only=True,
         )
-        is None
-    )
-    assert (
-        _raise_for_streaming_post_call_pipelines(
-            {"background": False, "metadata": {"_guardrail_pipelines": [("p", post_call)]}}, auth
-        )
-        is None
-    )
-    assert (
-        _raise_for_streaming_post_call_pipelines({"metadata": {"_guardrail_pipelines": [("p", post_call)]}}, auth)
-        is None
-    )
-    assert (
-        _raise_for_streaming_post_call_pipelines(
-            {"stream": True, "metadata": {"_guardrail_pipelines": [("p", pre_call)]}}, auth
-        )
-        is None
-    )
-    assert _raise_for_streaming_post_call_pipelines({"stream": True}, auth) is None
-    assert _raise_for_streaming_post_call_pipelines({"background": True}, auth) is None
+
+    assert out is not None
+    assert not any("background" in message for message in _warnings(caplog))
 
 
 # ---------------------------------------------------------------------------
@@ -1485,6 +1497,56 @@ async def _async_chunk_iter(chunks: List[Any]):
         yield chunk
 
 
+def test_streamable_post_call_pipelines_keeps_supported_and_drops_unsupported(
+    make_user_api_key_auth, monkeypatch, caplog
+):
+    class NativeOnlyGuardrail(CustomGuardrail):
+        pass
+
+    supported = _unified_stream_guardrail({})
+    native_only = NativeOnlyGuardrail(guardrail_name="gr-native", event_hook=GuardrailEventHooks.post_call)
+    monkeypatch.setattr(litellm, "callbacks", [supported, native_only])
+    governed = GuardrailPipeline(mode="post_call", steps=[PipelineStep(guardrail="gr-post", on_fail="block")])
+    ungoverned = GuardrailPipeline(
+        mode="post_call",
+        steps=[PipelineStep(guardrail="gr-post", on_fail="next"), PipelineStep(guardrail="gr-native", on_fail="block")],
+    )
+    pre_call = GuardrailPipeline(mode="pre_call", steps=[PipelineStep(guardrail="gr-native", on_fail="block")])
+    data = {"metadata": {"_guardrail_pipelines": [("governed", governed), ("ungoverned", ungoverned), ("req", pre_call)]}}
+
+    with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
+        streamable = _streamable_post_call_pipelines(data, make_user_api_key_auth(request_route="/v1/chat/completions"))
+
+    assert streamable == (("governed", governed),)
+    assert any("'ungoverned'" in message and "gr-native" in message for message in _warnings(caplog))
+    assert not any("'governed'" in message for message in _warnings(caplog))
+
+
+def test_streamable_post_call_pipelines_is_empty_on_route_without_translation(
+    make_user_api_key_auth, monkeypatch, caplog
+):
+    monkeypatch.setattr(litellm, "callbacks", [_unified_stream_guardrail({})])
+    governed = GuardrailPipeline(mode="post_call", steps=[PipelineStep(guardrail="gr-post", on_fail="block")])
+    data = {"metadata": {"_guardrail_pipelines": [("governed", governed)]}}
+
+    with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
+        streamable = _streamable_post_call_pipelines(data, make_user_api_key_auth(request_route="/custom/stream"))
+
+    assert streamable == ()
+    assert any("/custom/stream" in message and "governed" in message for message in _warnings(caplog))
+
+
+def test_streamable_post_call_pipelines_is_empty_without_post_call_pipelines(make_user_api_key_auth, caplog):
+    pre_call = GuardrailPipeline(mode="pre_call", steps=[PipelineStep(guardrail="g", on_fail="block")])
+    auth = make_user_api_key_auth(request_route="/custom/stream")
+
+    with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
+        assert _streamable_post_call_pipelines({"metadata": {"_guardrail_pipelines": [("p", pre_call)]}}, auth) == ()
+        assert _streamable_post_call_pipelines({"stream": True}, auth) == ()
+
+    assert _warnings(caplog) == []
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("request_route", [None, "/v1/chat/completions"])
 async def test_pre_call_hook_allows_streaming_when_pipeline_guardrail_supports_unified(
@@ -1507,21 +1569,25 @@ async def test_pre_call_hook_allows_streaming_when_pipeline_guardrail_supports_u
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize("native_lifecycle", [False, True])
-async def test_pre_call_hook_rejects_streaming_when_pipeline_guardrail_lacks_unified_support(
-    proxy_logging, make_user_api_key_auth, monkeypatch, native_lifecycle
+async def test_streaming_iterator_hook_releases_stream_when_pipeline_guardrail_lacks_unified_support(
+    proxy_logging, make_user_api_key_auth, monkeypatch, native_lifecycle, caplog
 ):
+    seen: Dict[str, Any] = {}
     if native_lifecycle:
 
         class NativeOnlyGuardrail(CustomGuardrail):
             use_native_lifecycle_hooks = True
 
             async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None):
+                seen["count"] = seen.get("count", 0) + 1
                 return inputs
 
     else:
 
         class NativeOnlyGuardrail(CustomGuardrail):
-            pass
+            async def async_post_call_success_hook(self, data, user_api_key_dict, response):
+                seen["count"] = seen.get("count", 0) + 1
+                return response
 
     monkeypatch.setattr(
         litellm,
@@ -1529,18 +1595,29 @@ async def test_pre_call_hook_rejects_streaming_when_pipeline_guardrail_lacks_uni
         [NativeOnlyGuardrail(guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=False)],
     )
     data = _post_call_pipeline_data(stream=True)
+    chunks = _stream_chunks()
+    delivered: List[Any] = []
 
-    with pytest.raises(HTTPException) as info:
-        await proxy_logging.pre_call_hook(
+    with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
+        out = await proxy_logging.pre_call_hook(
             user_api_key_dict=make_user_api_key_auth(),
             data=data,
             call_type="completion",
             guardrails_only=True,
         )
+        async for item in proxy_logging.async_post_call_streaming_iterator_hook(
+            user_api_key_dict=make_user_api_key_auth(request_route="/v1/chat/completions"),
+            response=_async_chunk_iter(chunks),
+            request_data=data,
+        ):
+            delivered.append(item)
 
-    assert info.value.status_code == 400
-    assert info.value.detail["error"]["guardrails"] == ("gr-post",)
-    assert "apply_guardrail" in info.value.detail["error"]["message"]
+    assert out is not None
+    assert out.get("stream") is True
+    assert [item is chunk for item, chunk in zip(delivered, chunks)] == [True, True]
+    assert len(delivered) == 2
+    assert seen.get("count") is None
+    assert any("'response-governance'" in message and "gr-post" in message for message in _warnings(caplog))
 
 
 @pytest.mark.asyncio
@@ -1614,25 +1691,34 @@ async def test_pre_call_hook_allows_streaming_when_content_filter_category_masks
 
 
 @pytest.mark.asyncio
-async def test_pre_call_hook_rejects_streaming_when_route_has_no_guardrail_translation(
-    proxy_logging, make_user_api_key_auth, monkeypatch
+async def test_streaming_iterator_hook_releases_stream_when_route_has_no_guardrail_translation(
+    proxy_logging, make_user_api_key_auth, monkeypatch, caplog
 ):
     seen: Dict[str, Any] = {}
     monkeypatch.setattr(litellm, "callbacks", [_unified_stream_guardrail(seen)])
     data = _post_call_pipeline_data(stream=True)
+    chunks = _stream_chunks()
+    delivered: List[Any] = []
 
-    with pytest.raises(HTTPException) as info:
-        await proxy_logging.pre_call_hook(
+    with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
+        out = await proxy_logging.pre_call_hook(
             user_api_key_dict=make_user_api_key_auth(request_route="/custom/stream"),
             data=data,
             call_type="completion",
             guardrails_only=True,
         )
+        async for item in proxy_logging.async_post_call_streaming_iterator_hook(
+            user_api_key_dict=make_user_api_key_auth(request_route="/custom/stream"),
+            response=_async_chunk_iter(chunks),
+            request_data=data,
+        ):
+            delivered.append(item)
 
-    assert info.value.status_code == 400
-    assert info.value.detail["error"]["policies"] == ("response-governance",)
-    assert "/custom/stream" in info.value.detail["error"]["message"]
+    assert out is not None
+    assert [item is chunk for item, chunk in zip(delivered, chunks)] == [True, True]
+    assert len(delivered) == 2
     assert seen.get("count") is None
+    assert any("/custom/stream" in message and "response-governance" in message for message in _warnings(caplog))
 
 
 @pytest.mark.asyncio
@@ -1714,8 +1800,8 @@ def _echoed_tool_call_dicts(arguments: str) -> List[Dict[str, Any]]:
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize("on_fail, on_error", [("block", None), ("next", "next")])
-async def test_streaming_iterator_hook_pipeline_withholds_runtime_tool_call_rewrite(
-    proxy_logging, make_user_api_key_auth, monkeypatch, on_fail, on_error
+async def test_streaming_iterator_hook_pipeline_releases_originals_on_runtime_tool_call_rewrite(
+    proxy_logging, make_user_api_key_auth, monkeypatch, on_fail, on_error, caplog
 ):
     transform = lambda inputs: {"tool_calls": _echoed_tool_call_dicts('{"ssn": "[MASKED]"}')}  # noqa: E731
     monkeypatch.setattr(litellm, "callbacks", [_rewriting_stream_guardrail(transform)])
@@ -1724,7 +1810,7 @@ async def test_streaming_iterator_hook_pipeline_withholds_runtime_tool_call_rewr
     data = _post_call_pipeline_data(step=step, stream=True)
     delivered: List[Any] = []
 
-    async def _drain() -> None:
+    with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
         async for item in proxy_logging.async_post_call_streaming_iterator_hook(
             user_api_key_dict=make_user_api_key_auth(request_route="/v1/chat/completions"),
             response=_async_chunk_iter(_tool_call_stream_chunks()),
@@ -1732,16 +1818,10 @@ async def test_streaming_iterator_hook_pipeline_withholds_runtime_tool_call_rewr
         ):
             delivered.append(item)
 
-    with pytest.raises(HTTPException) as info:
-        await _drain()
-
-    error = info.value.detail["error"]
-    assert delivered == []
-    assert info.value.status_code == 400
-    assert error["type"] == "guardrail_pipeline_error"
-    assert error["policies"] == ("response-governance",)
-    assert error["guardrails"] == ("gr-post",)
-    assert "stream=false" in error["message"]
+    assert len(delivered) == 2
+    assert delivered[0].choices[0].delta.tool_calls[0].function.arguments == '{"ssn": "123"}'
+    assert delivered[1].choices[0].finish_reason == "tool_calls"
+    assert any("'gr-post'" in message and "discarded" in message for message in _warnings(caplog))
 
 
 @pytest.mark.asyncio
@@ -1849,30 +1929,28 @@ async def test_streaming_iterator_hook_pipeline_releases_stream_echoed_in_anothe
 
 
 @pytest.mark.asyncio
-async def test_streaming_iterator_hook_pipeline_withholds_unresolvable_response_shape(
-    proxy_logging, make_user_api_key_auth, monkeypatch
+async def test_streaming_iterator_hook_pipeline_releases_originals_on_unresolvable_response_shape(
+    proxy_logging, make_user_api_key_auth, monkeypatch, caplog
 ):
     seen: Dict[str, Any] = {}
     monkeypatch.setattr(litellm, "callbacks", [_unified_stream_guardrail(seen)])
     monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
     data = _post_call_pipeline_data(stream=True)
+    chunks = [object(), object()]
     delivered: List[Any] = []
 
-    async def _drain() -> None:
+    with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
         async for item in proxy_logging.async_post_call_streaming_iterator_hook(
             user_api_key_dict=make_user_api_key_auth(),
-            response=_async_chunk_iter([object(), object()]),
+            response=_async_chunk_iter(chunks),
             request_data=data,
         ):
             delivered.append(item)
 
-    with pytest.raises(ProxyException) as info:
-        await _drain()
-
-    assert delivered == []
-    assert info.value.code == "500"
-    assert "withheld" in info.value.message
+    assert [item is chunk for item, chunk in zip(delivered, chunks)] == [True, True]
+    assert len(delivered) == 2
     assert seen.get("count") is None
+    assert any("response-governance" in message and "shape" in message for message in _warnings(caplog))
 
 
 def _anthropic_sse_chunks() -> List[bytes]:
@@ -1953,9 +2031,9 @@ async def test_streaming_iterator_hook_pipeline_delivers_text_rewrite_on_anthrop
 
 
 @pytest.mark.asyncio
-async def test_pipeline_executor_withholds_text_rewrite_when_translation_lacks_write_back(monkeypatch):
+async def test_pipeline_executor_discards_text_rewrite_when_translation_lacks_write_back(monkeypatch, caplog):
     from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
-    from litellm.proxy.policy_engine.pipeline_executor import PipelineExecutor, UndeliverableStreamRewrite
+    from litellm.proxy.policy_engine.pipeline_executor import PipelineExecutor
 
     class NoWriteBackTranslation(BaseTranslation):
         async def process_input_messages(self, data, guardrail_to_apply, litellm_logging_obj):
@@ -1984,45 +2062,23 @@ async def test_pipeline_executor_withholds_text_rewrite_when_translation_lacks_w
 
     transform = lambda inputs: {"texts": ["hello [MASKED]"]}  # noqa: E731
     monkeypatch.setattr(litellm, "callbacks", [_rewriting_stream_guardrail(transform)])
+    chunks = _stream_chunks()
 
-    with pytest.raises(UndeliverableStreamRewrite):
-        await PipelineExecutor.execute_steps(
+    with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
+        result = await PipelineExecutor.execute_steps(
             steps=[PipelineStep(guardrail="gr-post", on_pass="allow", on_fail="block")],
             mode="post_call",
             data={"metadata": {}},
             user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"),
             call_type="acompletion",
             policy_name="response-governance",
-            streaming_chunks=_stream_chunks(),
+            streaming_chunks=chunks,
             endpoint_translation=NoWriteBackTranslation(),
         )
 
-
-@pytest.mark.asyncio
-async def test_streaming_iterator_hook_pipeline_gates_without_iterator_overrides(
-    proxy_logging, make_user_api_key_auth, monkeypatch
-):
-    monkeypatch.setattr(litellm, "callbacks", [])
-    monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
-    data = _post_call_pipeline_data(stream=True)
-    delivered: List[Any] = []
-
-    async def _drain() -> None:
-        async for item in proxy_logging.async_post_call_streaming_iterator_hook(
-            user_api_key_dict=make_user_api_key_auth(request_route="/v1/chat/completions"),
-            response=_async_chunk_iter(_stream_chunks()),
-            request_data=data,
-        ):
-            delivered.append(item)
-
-    with pytest.raises(HTTPException) as info:
-        await _drain()
-
-    assert delivered == []
-    assert info.value.status_code == 400
-    assert info.value.detail["error"]["pipeline_context"]["step_results"] == [
-        {"guardrail": "gr-post", "outcome": "error", "action": "block"}
-    ]
+    assert result.terminal_action == "allow"
+    assert [chunk.choices[0].delta.content for chunk in chunks] == ["hello ", "world"]
+    assert any("'gr-post'" in message and "discarded" in message for message in _warnings(caplog))
 
 
 @pytest.mark.asyncio

From 2f397fa12812afba555dbdeae407597e95968123 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 7 Sep 2026 18:06:29 -0700
Subject: [PATCH 47/85] fix(drop_params): honor string flags in
 litellm_settings and responses, and fail open on non-flag values

---
 litellm/proxy/proxy_server.py                 |  3 ++
 litellm/responses/main.py                     |  5 ++-
 litellm/types/router.py                       |  6 ++-
 .../proxy/proxy_server/test_proxy_config.py   | 43 +++++++++++++++++++
 .../test_responses_api_request_body.py        |  4 +-
 tests/test_litellm/types/test_router.py       |  7 ++-
 6 files changed, 59 insertions(+), 9 deletions(-)

diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 0915b8dd1b9..76acd414976 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -279,6 +279,7 @@ from litellm.litellm_core_utils.audio_utils.utils import resolve_speech_media_ty
 from litellm.litellm_core_utils.core_helpers import (
     _get_parent_otel_span_from_kwargs,
     get_litellm_metadata_from_kwargs,
+    normalize_drop_params,
 )
 from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@@ -5508,6 +5509,8 @@ class ProxyConfig:
 
                     parse_budget_reset_time(value)
                     setattr(litellm, key, value)
+                elif key == "drop_params":
+                    litellm.drop_params = bool(normalize_drop_params(value))
                 else:
                     verbose_proxy_logger.debug(
                         "%s setting litellm.%s=%s%s",
diff --git a/litellm/responses/main.py b/litellm/responses/main.py
index 5e74b7324b4..52ca6ebb07a 100644
--- a/litellm/responses/main.py
+++ b/litellm/responses/main.py
@@ -17,6 +17,7 @@ from litellm.completion_extras.litellm_responses_transformation.transformation i
 from litellm.constants import request_timeout
 from litellm.integrations.anthropic_cache_control_hook import CARRY_UNMATCHED_MESSAGE_POINTS
 from litellm.litellm_core_utils.asyncify import run_async_function
+from litellm.litellm_core_utils.core_helpers import normalize_drop_params
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     update_responses_input_with_model_file_ids,
@@ -1253,7 +1254,7 @@ def responses(
             responses_api_provider_config=responses_api_provider_config,
             response_api_optional_params=response_api_optional_params,
             allowed_openai_params=allowed_openai_params,
-            drop_params=request_drop_params if isinstance(request_drop_params, bool) else None,
+            drop_params=normalize_drop_params(request_drop_params),
         )
 
         litellm_logging_obj.update_from_kwargs(
@@ -2081,7 +2082,7 @@ def compact_responses(
             responses_api_provider_config=responses_api_provider_config,
             response_api_optional_params=response_api_optional_params,
             allowed_openai_params=None,
-            drop_params=request_drop_params if isinstance(request_drop_params, bool) else None,
+            drop_params=normalize_drop_params(request_drop_params),
         )
 
         # Pre Call logging
diff --git a/litellm/types/router.py b/litellm/types/router.py
index 47aea6430c3..e8aec027a5e 100644
--- a/litellm/types/router.py
+++ b/litellm/types/router.py
@@ -408,9 +408,11 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
 
     @field_validator("drop_params", mode="before")
     @classmethod
-    def coerce_drop_params(cls, value: object) -> object:
+    def coerce_drop_params(cls, value: object) -> bool | str | None:
         normalized: Final = normalize_drop_params(value)
-        return value if normalized is None else normalized
+        if normalized is not None:
+            return normalized
+        return value if isinstance(value, str) else None
 
     def __contains__(self, key) -> bool:
         # Define custom behavior for the 'in' operator
diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py
index a4be9574f89..c57cd387d13 100644
--- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py
+++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py
@@ -2431,6 +2431,49 @@ def test_ProxyConfig__add_deployment_turns_stored_drop_params_string_into_bool(m
     assert deployment.litellm_params.drop_params is True
 
 
+def test_ProxyConfig__add_deployment_keeps_loading_rows_after_a_non_flag_drop_params(monkeypatch):
+    monkeypatch.setenv("LITELLM_SALT_KEY", "sk-1234")
+    fake_router = MagicMock()
+    fake_router.upsert_deployment = MagicMock(return_value=True)
+    monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", fake_router)
+    pc = ProxyConfig()
+
+    def db_model(model_id, drop_params):
+        return SimpleNamespace(
+            model_id=model_id,
+            model_name="gpt-5-nano",
+            model_info={"id": model_id},
+            litellm_params={
+                "model": encrypt_value_helper(value="openai/gpt-5-nano"),
+                "drop_params": encrypt_value_helper(value=drop_params),
+            },
+            blocked=False,
+        )
+
+    added = pc._add_deployment(db_models=[db_model("bad-row", 2), db_model("good-after", "true")])
+    deployments = [call.kwargs["deployment"] for call in fake_router.upsert_deployment.call_args_list]
+
+    assert added == 2
+    assert [d.litellm_params.drop_params for d in deployments] == [None, True]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("configured, expected", [("true", True), ("false", False)])
+async def test_ProxyConfig_load_config_turns_litellm_settings_drop_params_string_into_bool(
+    tmp_path, monkeypatch, configured, expected
+):
+    f = tmp_path / "c.yaml"
+    f.write_text(f'model_list: []\nlitellm_settings:\n  drop_params: "{configured}"\n')
+    monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
+    monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False)
+    monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False)
+    monkeypatch.setattr(litellm, "drop_params", not expected)
+
+    await ProxyConfig().load_config(router=None, config_file_path=str(f))
+
+    assert litellm.drop_params is expected
+
+
 # ---------------------------------------------------------------------------
 # ProxyConfig.decrypt_model_list_from_db
 # ---------------------------------------------------------------------------
diff --git a/tests/test_litellm/responses/test_responses_api_request_body.py b/tests/test_litellm/responses/test_responses_api_request_body.py
index 3e60906ec6d..5fced458208 100644
--- a/tests/test_litellm/responses/test_responses_api_request_body.py
+++ b/tests/test_litellm/responses/test_responses_api_request_body.py
@@ -246,8 +246,10 @@ async def test_aresponses_keeps_include_obfuscation_in_stream_options():
 
 
 @pytest.mark.asyncio
+@pytest.mark.parametrize("drop_params", [True, "true"])
 async def test_aresponses_request_level_drop_params_drops_bedrock_mantle_service_tier(
     monkeypatch,
+    drop_params,
 ):
     """
     Request-level drop_params=True (as the proxy injects for agentic CLIs) must
@@ -271,7 +273,7 @@ async def test_aresponses_request_level_drop_params_drops_bedrock_mantle_service
             aws_region_name="us-east-1",
             input="hi",
             service_tier="priority",
-            drop_params=True,
+            drop_params=drop_params,
         )
 
         mock_post.assert_called_once()
diff --git a/tests/test_litellm/types/test_router.py b/tests/test_litellm/types/test_router.py
index 2f4a29473c3..47fc08167e3 100644
--- a/tests/test_litellm/types/test_router.py
+++ b/tests/test_litellm/types/test_router.py
@@ -1,5 +1,4 @@
 import pytest
-from pydantic import ValidationError
 
 from litellm.types.router import (
     SPECIAL_MODEL_INFO_PARAMS,
@@ -109,6 +108,6 @@ def test_drop_params_coerces_flags_and_keeps_unresolved_strings(value, expected)
     assert GenericLiteLLMParams(drop_params=value).drop_params == expected
 
 
-def test_drop_params_rejects_non_flag_non_string_values():
-    with pytest.raises(ValidationError):
-        GenericLiteLLMParams(drop_params=2)
+@pytest.mark.parametrize("value", [2, 2.5, [], {}])
+def test_drop_params_ignores_non_flag_non_string_values(value):
+    assert GenericLiteLLMParams(drop_params=value).drop_params is None

From b7c2decb7db01b54463d94112f7943032d9309da Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 7 Sep 2026 19:00:47 -0700
Subject: [PATCH 48/85] fix(drop_params): honor string values in litellm_params
 and the LITELLM_DROP_PARAMS env var

get_litellm_params normalizes drop_params once, so a client-body string and
router_settings.default_litellm_params reach the anthropic, bedrock, and
azure_ai gates as a bool. LITELLM_DROP_PARAMS=false now means off. A value
that is neither a flag nor a string logs one warning and counts as unset,
both in the deployment validator and in litellm_settings.
---
 litellm/__init__.py                           |  3 +-
 litellm/litellm_core_utils/core_helpers.py    |  2 +-
 .../litellm_core_utils/get_litellm_params.py  |  5 +--
 litellm/proxy/proxy_server.py                 |  9 ++++-
 litellm/types/router.py                       |  7 +++-
 .../test_get_litellm_params.py                |  8 +++++
 .../proxy/proxy_server/test_proxy_config.py   | 34 +++++++++++++++++++
 .../test_litellm/test_drop_params_env_var.py  | 17 ++++++++++
 tests/test_litellm/types/test_router.py       | 15 ++++++--
 9 files changed, 92 insertions(+), 8 deletions(-)
 create mode 100644 tests/test_litellm/test_drop_params_env_var.py

diff --git a/litellm/__init__.py b/litellm/__init__.py
index 62477dd6264..36f143376ff 100644
--- a/litellm/__init__.py
+++ b/litellm/__init__.py
@@ -47,6 +47,7 @@ from typing import (
 )
 from litellm.types.integrations.datadog import DatadogInitParams
 from litellm.types.integrations.newrelic import NewRelicInitParams
+from litellm.litellm_core_utils.core_helpers import normalize_drop_params
 from litellm._logging import (
     set_verbose,
     _turn_on_debug,
@@ -238,7 +239,7 @@ token: Optional[str] = (
 )
 telemetry = True
 max_tokens: int = DEFAULT_MAX_TOKENS  # OpenAI Defaults
-drop_params = bool(os.getenv("LITELLM_DROP_PARAMS", False))
+drop_params = bool(normalize_drop_params(os.getenv("LITELLM_DROP_PARAMS")))
 modify_params = bool(os.getenv("LITELLM_MODIFY_PARAMS", False))
 use_chat_completions_url_for_anthropic_messages: bool = bool(
     os.getenv("LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES", False)
diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py
index bd7a1b8f384..c1f1076c710 100644
--- a/litellm/litellm_core_utils/core_helpers.py
+++ b/litellm/litellm_core_utils/core_helpers.py
@@ -42,7 +42,7 @@ _DROP_PARAMS_BOOL: Final = TypeAdapter(bool)
 
 
 def normalize_drop_params(value: object) -> bool | None:
-    if isinstance(value, bool):
+    if value is None or isinstance(value, bool):
         return value
     try:
         return _DROP_PARAMS_BOOL.validate_python(value.strip() if isinstance(value, str) else value)
diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py
index 1fd79db15a6..92b32d32dc0 100644
--- a/litellm/litellm_core_utils/get_litellm_params.py
+++ b/litellm/litellm_core_utils/get_litellm_params.py
@@ -2,6 +2,7 @@ from collections.abc import Mapping, MutableMapping
 from types import MappingProxyType
 from typing import Final
 
+from litellm.litellm_core_utils.core_helpers import normalize_drop_params
 from litellm.llms.openai.data_residency import infer_openai_data_residency
 
 AWS_CREDENTIAL_KWARGS_KEYS: Final = frozenset(
@@ -113,7 +114,7 @@ def get_litellm_params(
     custom_prompt_dict: dict | None = None,
     litellm_metadata: dict | None = None,
     disable_add_transform_inline_image_block: bool | None = None,
-    drop_params: bool | None = None,
+    drop_params: bool | str | None = None,
     prompt_id: str | None = None,
     prompt_variables: dict | None = None,
     async_call: bool | None = None,
@@ -175,7 +176,7 @@ def get_litellm_params(
         "custom_prompt_dict": custom_prompt_dict,
         "litellm_metadata": litellm_metadata,
         "disable_add_transform_inline_image_block": disable_add_transform_inline_image_block,
-        "drop_params": drop_params,
+        "drop_params": normalize_drop_params(drop_params),
         "prompt_id": prompt_id,
         "prompt_variables": prompt_variables,
         "async_call": async_call,
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 76acd414976..8a30274cc9f 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -5510,7 +5510,7 @@ class ProxyConfig:
                     parse_budget_reset_time(value)
                     setattr(litellm, key, value)
                 elif key == "drop_params":
-                    litellm.drop_params = bool(normalize_drop_params(value))
+                    litellm.drop_params = _drop_params_from_litellm_settings(value)
                 else:
                     verbose_proxy_logger.debug(
                         "%s setting litellm.%s=%s%s",
@@ -16915,6 +16915,13 @@ def _redact_config_param_value_for_logging(param_name: str | None, param_value:
     return param_value
 
 
+def _drop_params_from_litellm_settings(value: object) -> bool:
+    normalized: Final = normalize_drop_params(value)
+    if normalized is None and value is not None:
+        verbose_proxy_logger.warning("litellm_settings.drop_params=%r is not a flag value, treating it as off", value)
+    return bool(normalized)
+
+
 def _redact_general_setting_value(field_name: str, value: JsonValue, is_full_admin: bool) -> JsonValue:
     if is_full_admin:
         return value
diff --git a/litellm/types/router.py b/litellm/types/router.py
index e8aec027a5e..5c9eab30f3d 100644
--- a/litellm/types/router.py
+++ b/litellm/types/router.py
@@ -12,6 +12,7 @@ import httpx
 from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
 from typing_extensions import Protocol, ReadOnly, Required, TypedDict, runtime_checkable
 
+from litellm._logging import verbose_logger
 from litellm._uuid import uuid
 from litellm.litellm_core_utils.core_helpers import normalize_drop_params
 
@@ -412,7 +413,11 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
         normalized: Final = normalize_drop_params(value)
         if normalized is not None:
             return normalized
-        return value if isinstance(value, str) else None
+        if isinstance(value, str):
+            return value
+        if value is not None:
+            verbose_logger.warning("drop_params=%r is not a flag value, treating it as unset", value)
+        return None
 
     def __contains__(self, key) -> bool:
         # Define custom behavior for the 'in' operator
diff --git a/tests/test_litellm/litellm_core_utils/test_get_litellm_params.py b/tests/test_litellm/litellm_core_utils/test_get_litellm_params.py
index fb4cb494bee..f026ff57719 100644
--- a/tests/test_litellm/litellm_core_utils/test_get_litellm_params.py
+++ b/tests/test_litellm/litellm_core_utils/test_get_litellm_params.py
@@ -215,3 +215,11 @@ class TestMetadataFallsBackToLitellmMetadata:
         assert result["metadata"] is not litellm_metadata
         result["metadata"].pop("trace_id")
         assert litellm_metadata == {"trace_id": "trace-1"}
+
+
+@pytest.mark.parametrize(
+    "value, expected",
+    [("true", True), ("false", False), (" TRUE ", True), (True, True), (None, None), ("os.environ/DROP_PARAMS", None)],
+)
+def test_drop_params_strings_reach_litellm_params_as_flags(value, expected):
+    assert get_litellm_params(drop_params=value)["drop_params"] is expected
diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py
index c57cd387d13..774c63d754a 100644
--- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py
+++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py
@@ -9,6 +9,7 @@ Pins covered:
 from __future__ import annotations
 
 import json
+import logging
 import os
 import re
 from types import SimpleNamespace
@@ -2474,6 +2475,39 @@ async def test_ProxyConfig_load_config_turns_litellm_settings_drop_params_string
     assert litellm.drop_params is expected
 
 
+@pytest.mark.asyncio
+async def test_ProxyConfig_load_config_resolves_a_litellm_settings_drop_params_env_ref(tmp_path, monkeypatch):
+    f = tmp_path / "c.yaml"
+    f.write_text("model_list: []\nlitellm_settings:\n  drop_params: os.environ/DROP_PARAMS_FROM_ENV\n")
+    monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
+    monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False)
+    monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False)
+    monkeypatch.setenv("DROP_PARAMS_FROM_ENV", "true")
+    monkeypatch.setattr(litellm, "drop_params", False)
+
+    await ProxyConfig().load_config(router=None, config_file_path=str(f))
+
+    assert litellm.drop_params is True
+
+
+@pytest.mark.asyncio
+async def test_ProxyConfig_load_config_warns_and_turns_off_a_non_flag_litellm_settings_drop_params(
+    tmp_path, monkeypatch, caplog
+):
+    f = tmp_path / "c.yaml"
+    f.write_text("model_list: []\nlitellm_settings:\n  drop_params: ture\n")
+    monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
+    monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False)
+    monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False)
+    monkeypatch.setattr(litellm, "drop_params", True)
+
+    with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
+        await ProxyConfig().load_config(router=None, config_file_path=str(f))
+
+    assert litellm.drop_params is False
+    assert "litellm_settings.drop_params='ture' is not a flag value, treating it as off" in caplog.text
+
+
 # ---------------------------------------------------------------------------
 # ProxyConfig.decrypt_model_list_from_db
 # ---------------------------------------------------------------------------
diff --git a/tests/test_litellm/test_drop_params_env_var.py b/tests/test_litellm/test_drop_params_env_var.py
new file mode 100644
index 00000000000..a1ef3648f95
--- /dev/null
+++ b/tests/test_litellm/test_drop_params_env_var.py
@@ -0,0 +1,17 @@
+import os
+import subprocess
+import sys
+
+import pytest
+
+
+@pytest.mark.parametrize("configured, expected", [("false", "False"), ("true", "True")])
+def test_litellm_drop_params_env_var_is_parsed_as_a_flag(configured, expected):
+    result = subprocess.run(
+        [sys.executable, "-c", "import litellm; print(litellm.drop_params)"],
+        env={**os.environ, "LITELLM_DROP_PARAMS": configured},
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    assert result.stdout.strip() == expected
diff --git a/tests/test_litellm/types/test_router.py b/tests/test_litellm/types/test_router.py
index 47fc08167e3..fd933a9d993 100644
--- a/tests/test_litellm/types/test_router.py
+++ b/tests/test_litellm/types/test_router.py
@@ -1,3 +1,5 @@
+import logging
+
 import pytest
 
 from litellm.types.router import (
@@ -109,5 +111,14 @@ def test_drop_params_coerces_flags_and_keeps_unresolved_strings(value, expected)
 
 
 @pytest.mark.parametrize("value", [2, 2.5, [], {}])
-def test_drop_params_ignores_non_flag_non_string_values(value):
-    assert GenericLiteLLMParams(drop_params=value).drop_params is None
+def test_drop_params_ignores_non_flag_non_string_values_with_a_warning(value, caplog):
+    with caplog.at_level(logging.WARNING, logger="LiteLLM"):
+        assert GenericLiteLLMParams(drop_params=value).drop_params is None
+    assert f"drop_params={value!r} is not a flag value" in caplog.text
+
+
+@pytest.mark.parametrize("value", [True, "true", None, "os.environ/DROP_PARAMS", "v2:gcm:ciphertext-from-a-pre-fix-row"])
+def test_drop_params_flags_and_strings_log_nothing(value, caplog):
+    with caplog.at_level(logging.WARNING, logger="LiteLLM"):
+        GenericLiteLLMParams(drop_params=value)
+    assert caplog.text == ""

From d594b9385eb8eae760c809e43ac9174bceadfd46 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 7 Sep 2026 21:05:01 -0700
Subject: [PATCH 49/85] fix(drop_params): warn when a deployment or env
 drop_params value is not a flag

A deployment drop_params string that is not a flag value (a typo like ture) stayed silently off. The router now logs one warning per deployment. LITELLM_DROP_PARAMS and litellm_settings.drop_params share the same helper, so a non-flag value there warns as well instead of flipping silently from on to off
---
 litellm/__init__.py                           |  4 +--
 litellm/litellm_core_utils/core_helpers.py    |  8 +++++
 litellm/proxy/proxy_server.py                 | 11 ++-----
 litellm/router.py                             |  6 ++++
 .../litellm_core_utils/test_core_helpers.py   | 17 ++++++++++
 .../test_litellm/test_drop_params_env_var.py  | 19 +++++++++--
 tests/test_litellm/test_router.py             | 33 +++++++++++++++++++
 7 files changed, 84 insertions(+), 14 deletions(-)

diff --git a/litellm/__init__.py b/litellm/__init__.py
index 36f143376ff..f7d4dce87d2 100644
--- a/litellm/__init__.py
+++ b/litellm/__init__.py
@@ -47,7 +47,7 @@ from typing import (
 )
 from litellm.types.integrations.datadog import DatadogInitParams
 from litellm.types.integrations.newrelic import NewRelicInitParams
-from litellm.litellm_core_utils.core_helpers import normalize_drop_params
+from litellm.litellm_core_utils.core_helpers import drop_params_flag
 from litellm._logging import (
     set_verbose,
     _turn_on_debug,
@@ -239,7 +239,7 @@ token: Optional[str] = (
 )
 telemetry = True
 max_tokens: int = DEFAULT_MAX_TOKENS  # OpenAI Defaults
-drop_params = bool(normalize_drop_params(os.getenv("LITELLM_DROP_PARAMS")))
+drop_params = drop_params_flag(os.getenv("LITELLM_DROP_PARAMS"), "LITELLM_DROP_PARAMS", verbose_logger)
 modify_params = bool(os.getenv("LITELLM_MODIFY_PARAMS", False))
 use_chat_completions_url_for_anthropic_messages: bool = bool(
     os.getenv("LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES", False)
diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py
index c1f1076c710..a3dfac81cc1 100644
--- a/litellm/litellm_core_utils/core_helpers.py
+++ b/litellm/litellm_core_utils/core_helpers.py
@@ -1,6 +1,7 @@
 # What is this?
 ## Helper utilities
 import copy
+import logging
 from collections.abc import Iterable, Mapping
 from typing import TYPE_CHECKING, Any, Final, Literal
 
@@ -50,6 +51,13 @@ def normalize_drop_params(value: object) -> bool | None:
         return None
 
 
+def drop_params_flag(value: object, source: str, logger: logging.Logger) -> bool:
+    normalized: Final = normalize_drop_params(value)
+    if normalized is None and value is not None:
+        logger.warning("%s=%r is not a flag value, treating it as off", source, value)
+    return bool(normalized)
+
+
 def safe_divide(
     numerator: float,
     denominator: float,
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 8a30274cc9f..83b99822b4c 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -278,8 +278,8 @@ from litellm.litellm_core_utils.asyncify import asyncify
 from litellm.litellm_core_utils.audio_utils.utils import resolve_speech_media_type
 from litellm.litellm_core_utils.core_helpers import (
     _get_parent_otel_span_from_kwargs,
+    drop_params_flag,
     get_litellm_metadata_from_kwargs,
-    normalize_drop_params,
 )
 from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@@ -5510,7 +5510,7 @@ class ProxyConfig:
                     parse_budget_reset_time(value)
                     setattr(litellm, key, value)
                 elif key == "drop_params":
-                    litellm.drop_params = _drop_params_from_litellm_settings(value)
+                    litellm.drop_params = drop_params_flag(value, "litellm_settings.drop_params", verbose_proxy_logger)
                 else:
                     verbose_proxy_logger.debug(
                         "%s setting litellm.%s=%s%s",
@@ -16915,13 +16915,6 @@ def _redact_config_param_value_for_logging(param_name: str | None, param_value:
     return param_value
 
 
-def _drop_params_from_litellm_settings(value: object) -> bool:
-    normalized: Final = normalize_drop_params(value)
-    if normalized is None and value is not None:
-        verbose_proxy_logger.warning("litellm_settings.drop_params=%r is not a flag value, treating it as off", value)
-    return bool(normalized)
-
-
 def _redact_general_setting_value(field_name: str, value: JsonValue, is_full_admin: bool) -> JsonValue:
     if is_full_admin:
         return value
diff --git a/litellm/router.py b/litellm/router.py
index 95cabfad4bd..4bbdba08e13 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -9366,6 +9366,12 @@ class Router:
         #### VALIDATE MODEL ########
         # Check if this is a prompt management model before validating as LLM provider
         litellm_model: Final = deployment.litellm_params.model
+        if isinstance(deployment.litellm_params.drop_params, str):
+            verbose_router_logger.warning(
+                "model=%s drop_params=%r is not a flag value, treating it as unset",
+                deployment.model_name,
+                deployment.litellm_params.drop_params,
+            )
         is_prompt_management_model = False
 
         if "/" in litellm_model:
diff --git a/tests/test_litellm/litellm_core_utils/test_core_helpers.py b/tests/test_litellm/litellm_core_utils/test_core_helpers.py
index 8797f7c3591..e3880175759 100644
--- a/tests/test_litellm/litellm_core_utils/test_core_helpers.py
+++ b/tests/test_litellm/litellm_core_utils/test_core_helpers.py
@@ -1,9 +1,12 @@
 """Tests for litellm_core_utils.core_helpers module."""
 
+import logging
+
 import pytest
 
 from litellm.litellm_core_utils.core_helpers import (
     _FINISH_REASON_MAP,
+    drop_params_flag,
     get_or_create_metadata_bucket,
     map_finish_reason,
     normalize_drop_params,
@@ -284,6 +287,20 @@ def test_normalize_drop_params(value, expected):
     assert normalize_drop_params(value) is expected
 
 
+@pytest.mark.parametrize("value, expected", [("true", True), ("off", False), (None, False)])
+def test_drop_params_flag_returns_a_bool_without_a_warning(value, expected, caplog):
+    with caplog.at_level(logging.WARNING, logger="drop-params-test"):
+        assert drop_params_flag(value, "LITELLM_DROP_PARAMS", logging.getLogger("drop-params-test")) is expected
+    assert caplog.text == ""
+
+
+@pytest.mark.parametrize("value", ["temperature", "ture", 2])
+def test_drop_params_flag_treats_non_flag_values_as_off_with_a_warning(value, caplog):
+    with caplog.at_level(logging.WARNING, logger="drop-params-test"):
+        assert drop_params_flag(value, "LITELLM_DROP_PARAMS", logging.getLogger("drop-params-test")) is False
+    assert f"LITELLM_DROP_PARAMS={value!r} is not a flag value, treating it as off" in caplog.text
+
+
 class TestIsExpectedClientError:
     def test_status_ranges(self):
         from litellm.litellm_core_utils.core_helpers import is_expected_client_error
diff --git a/tests/test_litellm/test_drop_params_env_var.py b/tests/test_litellm/test_drop_params_env_var.py
index a1ef3648f95..339298df3d1 100644
--- a/tests/test_litellm/test_drop_params_env_var.py
+++ b/tests/test_litellm/test_drop_params_env_var.py
@@ -5,13 +5,26 @@ import sys
 import pytest
 
 
-@pytest.mark.parametrize("configured, expected", [("false", "False"), ("true", "True")])
-def test_litellm_drop_params_env_var_is_parsed_as_a_flag(configured, expected):
-    result = subprocess.run(
+def _import_litellm_with(configured: str) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
         [sys.executable, "-c", "import litellm; print(litellm.drop_params)"],
         env={**os.environ, "LITELLM_DROP_PARAMS": configured},
         capture_output=True,
         text=True,
         check=True,
     )
+
+
+@pytest.mark.parametrize("configured, expected", [("false", "False"), ("true", "True")])
+def test_litellm_drop_params_env_var_is_parsed_as_a_flag(configured, expected):
+    result = _import_litellm_with(configured)
+
     assert result.stdout.strip() == expected
+    assert "is not a flag value" not in result.stderr
+
+
+def test_litellm_drop_params_env_var_non_flag_value_is_off_with_a_warning():
+    result = _import_litellm_with("temperature")
+
+    assert result.stdout.strip() == "False"
+    assert "LITELLM_DROP_PARAMS='temperature' is not a flag value, treating it as off" in result.stderr
diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py
index 201a4d84ebe..0c0bad8080d 100644
--- a/tests/test_litellm/test_router.py
+++ b/tests/test_litellm/test_router.py
@@ -14424,3 +14424,36 @@ async def test_router_deployment_drop_params_string_true_is_honored(monkeypatch)
         temperature=0.1,
     )
     assert response.choices[0].message.content == "Hello, world!"
+
+
+@pytest.mark.parametrize("value", ["ture", "enabled"])
+def test_router_warns_when_a_deployment_drop_params_string_is_not_a_flag(value, caplog):
+    with caplog.at_level(logging.WARNING, logger="LiteLLM Router"):
+        router = Router(
+            model_list=[
+                {
+                    "model_name": "gpt-5-nano",
+                    "litellm_params": {"model": "openai/gpt-5-nano", "api_key": "sk-fake", "drop_params": value},
+                }
+            ]
+        )
+
+    deployment = router.get_deployment_by_model_group_name(model_group_name="gpt-5-nano")
+    assert deployment is not None
+    assert deployment.litellm_params.drop_params == value
+    assert f"model=gpt-5-nano drop_params={value!r} is not a flag value, treating it as unset" in caplog.text
+
+
+@pytest.mark.parametrize("value", [True, "true", "off", None])
+def test_router_stays_quiet_when_a_deployment_drop_params_is_a_flag(value, caplog):
+    with caplog.at_level(logging.WARNING, logger="LiteLLM Router"):
+        Router(
+            model_list=[
+                {
+                    "model_name": "gpt-5-nano",
+                    "litellm_params": {"model": "openai/gpt-5-nano", "api_key": "sk-fake", "drop_params": value},
+                }
+            ]
+        )
+
+    assert "is not a flag value" not in caplog.text

From 69d2ac1edb83336723d4c9ce5024b93612230e5d Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 7 Sep 2026 21:46:00 -0700
Subject: [PATCH 50/85] fix(policy_engine): run iterator-hook guardrails whose
 post_call pipeline cannot stream

The streaming loop skipped every guardrail stepped by a post_call pipeline, even when the pipeline was dropped from the stream for lacking the unified apply_guardrail interface, so a default_on guardrail that only implements async_post_call_streaming_iterator_hook stopped governing streams it governed on the merge base. The skip set now comes from the pipelines that will gate the stream
---
 litellm/proxy/utils.py                        | 26 +++++++-------
 .../proxy_logging/test_guardrail_pipeline.py  | 36 +++++++++++++++++++
 2 files changed, 50 insertions(+), 12 deletions(-)

diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index d7bf832bca4..8b36061c6be 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -447,14 +447,15 @@ def _policy_pipelines(data: Mapping[str, object]) -> tuple[tuple[str, "Guardrail
     )
 
 
+def _pipeline_step_guardrail_names(pipelines: Sequence[tuple[str, "GuardrailPipeline"]]) -> frozenset[str]:
+    return frozenset(step.guardrail for _policy_name, pipeline in pipelines for step in pipeline.steps)
+
+
 def _pipeline_managed_guardrail_names(
     data: Mapping[str, object], mode: Literal["pre_call", "post_call"]
 ) -> frozenset[str]:
-    return frozenset(
-        step.guardrail
-        for _policy_name, pipeline in _policy_pipelines(data)
-        if pipeline.mode == mode
-        for step in pipeline.steps
+    return _pipeline_step_guardrail_names(
+        tuple((policy_name, pipeline) for policy_name, pipeline in _policy_pipelines(data) if pipeline.mode == mode)
     )
 
 
@@ -547,7 +548,7 @@ def _pipeline_is_streamable(policy_name: str, pipeline: "GuardrailPipeline") ->
         return True
     verbose_proxy_logger.warning(
         "Policy '%s' has post_call pipeline guardrails without the unified apply_guardrail interface, "
-        "which streaming pipelines need; the stream is released ungoverned by it: %s",
+        "which streaming pipelines need; the stream skips the pipeline and its guardrails run on their own: %s",
         policy_name,
         ", ".join(unsupported),
     )
@@ -563,9 +564,9 @@ def _streamable_post_call_pipelines(
     Streaming pipelines scan the buffered stream through the endpoint guardrail
     translation of the request route, so every step's guardrail needs the
     unified apply_guardrail interface and the route needs a translation. A
-    pipeline that cannot be run that way yet is left out and the stream is
-    released the way it was before pipelines ran on streams at all, with a
-    warning naming what went ungoverned.
+    pipeline that cannot be run that way yet is left out and its guardrails
+    run on the stream on their own, the way they did before pipelines ran on
+    streams at all, with a warning naming the pipeline.
     """
     post_call_pipelines: Final = _post_call_pipelines(request_data)
     if not post_call_pipelines:
@@ -574,7 +575,8 @@ def _streamable_post_call_pipelines(
     if route and resolve_endpoint_translation(user_api_key_dict, None) is None:
         verbose_proxy_logger.warning(
             "Policies with post_call guardrail pipelines cannot scan streaming responses on route %s yet "
-            "(no endpoint guardrail translation); the stream is released ungoverned by them: %s",
+            "(no endpoint guardrail translation); the stream skips the pipelines and their guardrails run "
+            "on their own: %s",
             route,
             ", ".join(policy_name for policy_name, _pipeline in post_call_pipelines),
         )
@@ -3361,10 +3363,10 @@ class ProxyLogging:
         current_response = response
         stream_needs_translation: Final = ProxyLogging._stream_requires_guardrail_translation(user_api_key_dict)
 
-        pipeline_managed_names: Final = _pipeline_managed_guardrail_names(request_data, "post_call")
+        pipeline_gated_names: Final = _pipeline_step_guardrail_names(post_call_pipelines)
         for resolved_callback, kind in caps.iterator_overrides:
             if isinstance(resolved_callback, CustomGuardrail):
-                if resolved_callback.guardrail_name in pipeline_managed_names:
+                if resolved_callback.guardrail_name in pipeline_gated_names:
                     continue
                 if (
                     resolved_callback.should_run_guardrail(data=request_data, event_type=GuardrailEventHooks.post_call)
diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
index ad2b4a0efbb..73dd6746b29 100644
--- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
+++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
@@ -1620,6 +1620,42 @@ async def test_streaming_iterator_hook_releases_stream_when_pipeline_guardrail_l
     assert any("'response-governance'" in message and "gr-post" in message for message in _warnings(caplog))
 
 
+@pytest.mark.asyncio
+async def test_streaming_iterator_hook_runs_iterator_hook_guardrail_whose_pipeline_cannot_stream(
+    proxy_logging, make_user_api_key_auth, monkeypatch, caplog
+):
+    seen: Dict[str, Any] = {}
+
+    class IteratorHookGuardrail(CustomGuardrail):
+        async def async_post_call_streaming_iterator_hook(self, user_api_key_dict, response, request_data):
+            seen["count"] = seen.get("count", 0) + 1
+            async for item in response:
+                item.choices[0].delta.content = f"[governed] {item.choices[0].delta.content}"
+                yield item
+
+    monkeypatch.setattr(
+        litellm,
+        "callbacks",
+        [IteratorHookGuardrail(guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=True)],
+    )
+    monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
+    data = _post_call_pipeline_data(stream=True)
+
+    with caplog.at_level(logging.WARNING, logger="LiteLLM Proxy"):
+        delivered = [
+            item
+            async for item in proxy_logging.async_post_call_streaming_iterator_hook(
+                user_api_key_dict=make_user_api_key_auth(request_route="/v1/chat/completions"),
+                response=_async_chunk_iter(_stream_chunks()),
+                request_data=data,
+            )
+        ]
+
+    assert seen["count"] == 1
+    assert [item.choices[0].delta.content for item in delivered] == ["[governed] hello ", "[governed] world"]
+    assert any("'response-governance'" in message and "gr-post" in message for message in _warnings(caplog))
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
     "rewrite_attribute, value",

From 91fc1b201027b3d21a7ced292ac19a51661cba0e Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 7 Sep 2026 21:48:19 -0700
Subject: [PATCH 51/85] fix(policy_engine): record a streaming pipeline step
 once and in the applied guardrails header

CustomGuardrail.__init_subclass__ wrapped _StreamRewriteObserver.apply_guardrail in log_guardrail_information, so every streaming step recorded a second standard_logging_guardrail_information entry and span next to the inner guardrail's own. The observer's method now carries the marker that skips the wrapper. The step also adds the guardrail to the applied guardrails header the way the non-streaming unified path does, so streamed spend rows name the guardrail that scanned them
---
 .../proxy/policy_engine/pipeline_executor.py  | 27 +++++++++---
 .../policy_engine/test_pipeline_executor.py   | 44 +++++++++++++++++++
 2 files changed, 65 insertions(+), 6 deletions(-)

diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py
index 970ac20487c..9bc10949e9f 100644
--- a/litellm/proxy/policy_engine/pipeline_executor.py
+++ b/litellm/proxy/policy_engine/pipeline_executor.py
@@ -7,19 +7,21 @@ pass/fail actions (allow, block, next, modify_response) and data forwarding.
 
 import copy
 import time
-from collections.abc import Mapping, Sequence
-from typing import TYPE_CHECKING, Any, Final, Literal
+from collections.abc import Callable, Mapping, Sequence
+from typing import TYPE_CHECKING, Any, Final, Literal, TypeVar
 
 from pydantic import BaseModel
 
 import litellm
 from litellm._logging import verbose_proxy_logger
+from litellm.constants import LOGS_GUARDRAIL_INFORMATION_MARKER
 from litellm.integrations.custom_guardrail import (
     CustomGuardrail,
     ModifyResponseException,
 )
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.core_helpers import independent_snapshot
+from litellm.proxy.common_utils.callback_utils import add_guardrail_to_applied_guardrails_header
 from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import (
     UnifiedLLMGuardrails,
 )
@@ -72,13 +74,23 @@ def _rewrote(sent: tuple[object, ...] | None, returned: tuple[object, ...] | Non
     return sent is not None and returned is not None and returned != sent
 
 
+_GuardrailMethodT = TypeVar("_GuardrailMethodT", bound=Callable[..., object])
+
+
+def _logged_by_inner_guardrail(method: _GuardrailMethodT) -> _GuardrailMethodT:
+    vars(method)[LOGS_GUARDRAIL_INFORMATION_MARKER] = True  # rebind-ok: stamps the method the class body just defined
+    return method
+
+
 class _StreamRewriteObserver(CustomGuardrail):
     """Stand-in handed to the endpoint translation in place of a streaming pipeline step's
     guardrail. It records whether the guardrail returned different output than it was given,
     which for guardrails like Bedrock's ANONYMIZED action is only known at runtime. Text
     rewrites are deliverable on translations that write them back across the buffered chunks
     (``delivers_ended_stream_text_rewrites``); tool-call rewrites and text rewrites on any
-    other translation are discarded by the executor, which releases the original chunks."""
+    other translation are discarded by the executor, which releases the original chunks.
+    The inner guardrail's ``apply_guardrail`` already records the guardrail information
+    and span, so the observer's stays out of ``log_guardrail_information``."""
 
     def __init__(self, inner: CustomGuardrail) -> None:
         super().__init__(guardrail_name=inner.guardrail_name)
@@ -89,6 +101,7 @@ class _StreamRewriteObserver(CustomGuardrail):
     def structured_messages_cover_full_request(self) -> bool:
         return self.inner.structured_messages_cover_full_request()
 
+    @_logged_by_inner_guardrail
     async def apply_guardrail(
         self,
         inputs: GenericGuardrailAPIInputs,
@@ -305,9 +318,11 @@ class PipelineExecutor:
                 )
         except UndeliverableStreamRewrite:
             _release_original_chunks(step.guardrail, streaming_chunks, originals)
-            return
-        if observer.rewrote_tool_calls or (observer.rewrote_texts and not deliver_rewrites):
-            _release_original_chunks(step.guardrail, streaming_chunks, originals)
+        else:
+            if observer.rewrote_tool_calls or (observer.rewrote_texts and not deliver_rewrites):
+                _release_original_chunks(step.guardrail, streaming_chunks, originals)
+        if not callback.records_own_guardrail_information:
+            add_guardrail_to_applied_guardrails_header(request_data=hook_input, guardrail_name=step.guardrail)
 
     @staticmethod
     async def _run_step(
diff --git a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py
index 0685bc6aa1e..54ef1f79f4d 100644
--- a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py
+++ b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py
@@ -1099,6 +1099,50 @@ async def test_streaming_step_discards_tool_call_rewrite_and_restores_written_te
     assert chunks == [_chunk()]
 
 
+class _BlockingStreamGuardrail(CustomGuardrail):
+    def __init__(self):
+        super().__init__(guardrail_name="masker", event_hook="post_call", default_on=True)
+
+    async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None):
+        raise HTTPException(status_code=400, detail={"error": "output blocked"})
+
+
+def _recorded_guardrail_statuses(result):
+    return [
+        entry["guardrail_status"]
+        for entry in result.modified_data["metadata"]["standard_logging_guardrail_information"]
+    ]
+
+
+@pytest.mark.asyncio
+async def test_streaming_step_records_guardrail_information_once_on_mask(monkeypatch):
+    monkeypatch.setattr(litellm, "callbacks", [_TextReturningGuardrail(["hello [MASKED]"])])
+
+    result = await _run_streaming_step(_WritingTranslation(), [_chunk()])
+
+    assert result.terminal_action == "allow"
+    assert _recorded_guardrail_statuses(result) == ["success"]
+
+
+@pytest.mark.asyncio
+async def test_streaming_step_records_the_guardrail_in_the_applied_guardrails_header(monkeypatch):
+    monkeypatch.setattr(litellm, "callbacks", [_TextReturningGuardrail(["hello [MASKED]"])])
+
+    result = await _run_streaming_step(_WritingTranslation(), [_chunk()])
+
+    assert result.modified_data["metadata"]["applied_guardrails"] == ["masker"]
+
+
+@pytest.mark.asyncio
+async def test_streaming_step_records_guardrail_information_once_on_block(monkeypatch):
+    monkeypatch.setattr(litellm, "callbacks", [_BlockingStreamGuardrail()])
+
+    result = await _run_streaming_step(_WritingTranslation(), [_chunk()])
+
+    assert [step.outcome for step in result.step_results] == ["fail"]
+    assert _recorded_guardrail_statuses(result) == ["guardrail_intervened"]
+
+
 @pytest.mark.asyncio
 async def test_streaming_step_restores_chunks_when_translation_refuses_the_rewrite(monkeypatch, caplog):
     monkeypatch.setattr(litellm, "callbacks", [_TextReturningGuardrail(["hello [MASKED]"])])

From d08a177bc75b45769e5e22efa2bbb4b0ea27ee86 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 7 Sep 2026 21:51:34 -0700
Subject: [PATCH 52/85] fix(policy_engine): keep a policy-added guardrail's
 other stages when a pipeline steps it

A policy that both adds a guardrail and steps it in a post_call pipeline
used to drop the guardrail from the request's guardrail list outright, so
its pre_call stage never ran. The per-hook loops already skip guardrails
by pipeline mode, so the mode-agnostic subtraction only lost coverage
---
 litellm/proxy/litellm_pre_call_utils.py       |  7 +---
 .../proxy/test_litellm_pre_call_utils.py      | 42 +++++++++++++++++++
 2 files changed, 44 insertions(+), 5 deletions(-)

diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py
index 56512570448..e4bce4378f0 100644
--- a/litellm/proxy/litellm_pre_call_utils.py
+++ b/litellm/proxy/litellm_pre_call_utils.py
@@ -3079,10 +3079,9 @@ def _apply_resolved_guardrails_to_metadata(
     if metadata_variable_name not in data:
         data[metadata_variable_name] = {}
 
-    # Track pipeline-managed guardrails to exclude from independent execution
-    pipeline_managed_guardrails: set = set()
+    # Record the pipelines and the guardrails they step; the hook loops skip those per pipeline mode
     if pipelines:
-        pipeline_managed_guardrails = PolicyResolver.get_pipeline_managed_guardrails(pipelines)
+        pipeline_managed_guardrails: Final = PolicyResolver.get_pipeline_managed_guardrails(pipelines)
         data[metadata_variable_name]["_guardrail_pipelines"] = pipelines
         data[metadata_variable_name]["_pipeline_managed_guardrails"] = pipeline_managed_guardrails
         verbose_proxy_logger.debug(
@@ -3099,10 +3098,8 @@ def _apply_resolved_guardrails_to_metadata(
         existing_guardrails = []
 
     # Combine existing guardrails with policy-resolved guardrails (no duplicates)
-    # Exclude pipeline-managed guardrails from the flat list
     combined = set(existing_guardrails)
     combined.update(resolved_guardrails)
-    combined -= pipeline_managed_guardrails
     data[metadata_variable_name]["guardrails"] = list(combined)
 
     verbose_proxy_logger.debug("Policy engine: added guardrails to request metadata: %s", list(combined))
diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py
index 7070617ce3e..78fe2e6df88 100644
--- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py
+++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py
@@ -4148,6 +4148,48 @@ async def test_add_guardrails_from_policy_engine():
     attachment_registry._initialized = False
 
 
+@pytest.mark.asyncio
+async def test_add_guardrails_from_policy_engine_keeps_a_policy_added_guardrail_its_pipeline_also_steps():
+    from litellm.proxy.policy_engine.attachment_registry import get_attachment_registry
+    from litellm.proxy.policy_engine.policy_registry import get_policy_registry
+    from litellm.types.proxy.policy_engine import (
+        GuardrailPipeline,
+        PipelineStep,
+        Policy,
+        PolicyAttachment,
+        PolicyGuardrails,
+    )
+
+    data = {"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}], "metadata": {}}
+    policy_registry = get_policy_registry()
+    policy_registry._policies = {
+        "response-governance": Policy(
+            guardrails=PolicyGuardrails(add=["pii_blocker"]),
+            pipeline=GuardrailPipeline(mode="post_call", steps=[PipelineStep(guardrail="pii_blocker")]),
+        ),
+    }
+    policy_registry._initialized = True
+    attachment_registry = get_attachment_registry()
+    attachment_registry._attachments = [PolicyAttachment(policy="response-governance", scope="*")]
+    attachment_registry._initialized = True
+
+    try:
+        await add_guardrails_from_policy_engine(
+            data=data,
+            metadata_variable_name="metadata",
+            user_api_key_dict=UserAPIKeyAuth(api_key="test-key"),
+        )
+    finally:
+        policy_registry._policies = {}
+        policy_registry._initialized = False
+        attachment_registry._attachments = []
+        attachment_registry._initialized = False
+
+    assert data["metadata"]["guardrails"] == ["pii_blocker"]
+    assert data["metadata"]["_pipeline_managed_guardrails"] == {"pii_blocker"}
+    assert [pipeline.mode for _policy_name, pipeline in data["metadata"]["_guardrail_pipelines"]] == ["post_call"]
+
+
 @pytest.mark.asyncio
 async def test_add_guardrails_from_policy_engine_accepts_dynamic_policies_and_pops_from_data():
     """

From 08b60c409a0b556efb6bbe6470402a4530666870 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 7 Sep 2026 21:53:26 -0700
Subject: [PATCH 53/85] refactor(guardrails): drop the unused
 rewrites_streamed_output hook

Nothing calls it since the streaming pipeline detects rewrites at run time
through the stream observer, so the base method and the content filter's
override were dead code with dead tests
---
 litellm/integrations/custom_guardrail.py      |  3 -
 .../litellm_content_filter/content_filter.py  |  9 ---
 .../content_filter/test_content_filter.py     | 56 -------------------
 3 files changed, 68 deletions(-)

diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py
index 883329c9fa8..2d66a280663 100644
--- a/litellm/integrations/custom_guardrail.py
+++ b/litellm/integrations/custom_guardrail.py
@@ -773,9 +773,6 @@ class CustomGuardrail(CustomLogger):
     def uses_apply_guardrail_interface(self) -> bool:
         return type(self).apply_guardrail is not CustomGuardrail.apply_guardrail
 
-    def rewrites_streamed_output(self) -> bool:
-        return self.mask_response_content
-
     def _deployment_pre_call_target(self) -> "CustomLogger":
         if not self.uses_apply_guardrail_interface() or self.use_native_lifecycle_hooks:
             return self
diff --git a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py
index 85eb50c78e7..722f96ef814 100644
--- a/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py
+++ b/litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py
@@ -1947,15 +1947,6 @@ class ContentFilterGuardrail(CustomGuardrail):
                 exception_str=exception_str,
             )
 
-    def rewrites_streamed_output(self) -> bool:
-        return (
-            super().rewrites_streamed_output()
-            or any(entry["action"] == ContentFilterAction.MASK for entry in self.compiled_patterns)
-            or any(action == ContentFilterAction.MASK for action, _ in self.blocked_words.values())
-            or any(action == ContentFilterAction.MASK for _, _, action in self.category_keywords.values())
-            or any(action == ContentFilterAction.MASK for _, _, action in self.always_block_category_keywords.values())
-        )
-
     async def async_post_call_streaming_iterator_hook(
         self,
         user_api_key_dict: UserAPIKeyAuth,
diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py
index 73020fe3e6f..be55ac47bde 100644
--- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py
+++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py
@@ -3068,59 +3068,3 @@ class TestContentFilterToolCallArguments:
                 request_data={},
                 input_type="response",
             )
-
-
-class TestRewritesStreamedOutput:
-    def test_block_only_rules_do_not_rewrite(self):
-        guardrail = ContentFilterGuardrail(
-            guardrail_name="cf",
-            patterns=[ContentFilterPattern(pattern_type="prebuilt", pattern_name="us_ssn", action=ContentFilterAction.BLOCK)],
-            blocked_words=[BlockedWord(keyword="kumquat", action=ContentFilterAction.BLOCK)],
-        )
-
-        assert guardrail.rewrites_streamed_output() is False
-
-    def test_mask_blocked_word_rewrites(self):
-        guardrail = ContentFilterGuardrail(
-            guardrail_name="cf",
-            blocked_words=[BlockedWord(keyword="persimmon", action=ContentFilterAction.MASK)],
-        )
-
-        assert guardrail.rewrites_streamed_output() is True
-
-    def test_mask_pattern_rewrites(self):
-        guardrail = ContentFilterGuardrail(
-            guardrail_name="cf",
-            patterns=[ContentFilterPattern(pattern_type="prebuilt", pattern_name="us_ssn", action=ContentFilterAction.MASK)],
-        )
-
-        assert guardrail.rewrites_streamed_output() is True
-
-    def test_mask_response_content_rewrites(self):
-        guardrail = ContentFilterGuardrail(
-            guardrail_name="cf",
-            blocked_words=[BlockedWord(keyword="kumquat", action=ContentFilterAction.BLOCK)],
-            mask_response_content=True,
-        )
-
-        assert guardrail.rewrites_streamed_output() is True
-
-    @pytest.mark.parametrize("action, expected", [("MASK", True), ("BLOCK", False)])
-    def test_category_keywords_follow_the_category_action(self, action, expected):
-        guardrail = ContentFilterGuardrail(
-            guardrail_name="cf",
-            categories=[{"category": "bias_gender", "enabled": True, "action": action}],
-        )
-
-        assert guardrail.category_keywords and not guardrail.always_block_category_keywords
-        assert guardrail.rewrites_streamed_output() is expected
-
-    @pytest.mark.parametrize("action, expected", [("MASK", True), ("BLOCK", False)])
-    def test_always_block_category_keywords_follow_the_category_action(self, action, expected):
-        guardrail = ContentFilterGuardrail(
-            guardrail_name="cf",
-            categories=[{"category": "age_discrimination", "enabled": True, "action": action}],
-        )
-
-        assert guardrail.always_block_category_keywords and not guardrail.category_keywords
-        assert guardrail.rewrites_streamed_output() is expected

From 704013dbb6f83962e4cac97c79ac0dd2109cbabf Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Mon, 7 Sep 2026 22:13:23 -0700
Subject: [PATCH 54/85] fix(init): keep non-flag LITELLM_DROP_PARAMS values on
 with a warning

The merge base read the variable by truthiness, so any non-empty value
turned the global flag on. Parsing it as a flag made a value such as
temperature or enabled silently turn it off, and the only docs for the
variable describe it as a list of parameter names, so keep those values
on and log a warning that asks for true or false. A blank value stays off
without a warning
---
 litellm/__init__.py                           |  4 +--
 litellm/litellm_core_utils/core_helpers.py    | 16 +++++++++++
 .../litellm_core_utils/test_core_helpers.py   | 28 +++++++++++++++++++
 .../test_litellm/test_drop_params_env_var.py  | 11 +++++---
 4 files changed, 53 insertions(+), 6 deletions(-)

diff --git a/litellm/__init__.py b/litellm/__init__.py
index f7d4dce87d2..fc6dc35fe55 100644
--- a/litellm/__init__.py
+++ b/litellm/__init__.py
@@ -47,7 +47,7 @@ from typing import (
 )
 from litellm.types.integrations.datadog import DatadogInitParams
 from litellm.types.integrations.newrelic import NewRelicInitParams
-from litellm.litellm_core_utils.core_helpers import drop_params_flag
+from litellm.litellm_core_utils.core_helpers import drop_params_env_flag
 from litellm._logging import (
     set_verbose,
     _turn_on_debug,
@@ -239,7 +239,7 @@ token: Optional[str] = (
 )
 telemetry = True
 max_tokens: int = DEFAULT_MAX_TOKENS  # OpenAI Defaults
-drop_params = drop_params_flag(os.getenv("LITELLM_DROP_PARAMS"), "LITELLM_DROP_PARAMS", verbose_logger)
+drop_params = drop_params_env_flag(os.environ, verbose_logger)
 modify_params = bool(os.getenv("LITELLM_MODIFY_PARAMS", False))
 use_chat_completions_url_for_anthropic_messages: bool = bool(
     os.getenv("LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES", False)
diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py
index a3dfac81cc1..eacc3e4860a 100644
--- a/litellm/litellm_core_utils/core_helpers.py
+++ b/litellm/litellm_core_utils/core_helpers.py
@@ -58,6 +58,22 @@ def drop_params_flag(value: object, source: str, logger: logging.Logger) -> bool
     return bool(normalized)
 
 
+DROP_PARAMS_ENV_VAR: Final = "LITELLM_DROP_PARAMS"
+
+
+def drop_params_env_flag(environ: Mapping[str, str], logger: logging.Logger) -> bool:
+    configured: Final = environ.get(DROP_PARAMS_ENV_VAR, "").strip()
+    if configured == "":
+        return False
+    normalized: Final = normalize_drop_params(configured)
+    if normalized is None:
+        logger.warning(
+            "%s=%r is not a flag value, treating it as on. Set it to true or false", DROP_PARAMS_ENV_VAR, configured
+        )
+        return True
+    return normalized
+
+
 def safe_divide(
     numerator: float,
     denominator: float,
diff --git a/tests/test_litellm/litellm_core_utils/test_core_helpers.py b/tests/test_litellm/litellm_core_utils/test_core_helpers.py
index e3880175759..e937be47441 100644
--- a/tests/test_litellm/litellm_core_utils/test_core_helpers.py
+++ b/tests/test_litellm/litellm_core_utils/test_core_helpers.py
@@ -6,6 +6,7 @@ import pytest
 
 from litellm.litellm_core_utils.core_helpers import (
     _FINISH_REASON_MAP,
+    drop_params_env_flag,
     drop_params_flag,
     get_or_create_metadata_bucket,
     map_finish_reason,
@@ -301,6 +302,33 @@ def test_drop_params_flag_treats_non_flag_values_as_off_with_a_warning(value, ca
     assert f"LITELLM_DROP_PARAMS={value!r} is not a flag value, treating it as off" in caplog.text
 
 
+@pytest.mark.parametrize(
+    "environ, expected",
+    [
+        ({}, False),
+        ({"LITELLM_DROP_PARAMS": ""}, False),
+        ({"LITELLM_DROP_PARAMS": "   "}, False),
+        ({"LITELLM_DROP_PARAMS": "true"}, True),
+        ({"LITELLM_DROP_PARAMS": " False "}, False),
+        ({"LITELLM_DROP_PARAMS": "0"}, False),
+    ],
+)
+def test_drop_params_env_flag_reads_a_flag_without_a_warning(environ, expected, caplog):
+    with caplog.at_level(logging.WARNING, logger="drop-params-test"):
+        assert drop_params_env_flag(environ, logging.getLogger("drop-params-test")) is expected
+    assert caplog.text == ""
+
+
+@pytest.mark.parametrize("configured", ["temperature", "temperature,top_p", "enabled"])
+def test_drop_params_env_flag_keeps_a_non_flag_value_on_with_a_warning(configured, caplog):
+    with caplog.at_level(logging.WARNING, logger="drop-params-test"):
+        assert drop_params_env_flag({"LITELLM_DROP_PARAMS": configured}, logging.getLogger("drop-params-test")) is True
+    assert (
+        f"LITELLM_DROP_PARAMS={configured!r} is not a flag value, treating it as on. Set it to true or false"
+        in caplog.text
+    )
+
+
 class TestIsExpectedClientError:
     def test_status_ranges(self):
         from litellm.litellm_core_utils.core_helpers import is_expected_client_error
diff --git a/tests/test_litellm/test_drop_params_env_var.py b/tests/test_litellm/test_drop_params_env_var.py
index 339298df3d1..1e0b7801ef1 100644
--- a/tests/test_litellm/test_drop_params_env_var.py
+++ b/tests/test_litellm/test_drop_params_env_var.py
@@ -15,7 +15,7 @@ def _import_litellm_with(configured: str) -> subprocess.CompletedProcess[str]:
     )
 
 
-@pytest.mark.parametrize("configured, expected", [("false", "False"), ("true", "True")])
+@pytest.mark.parametrize("configured, expected", [("false", "False"), ("true", "True"), ("", "False")])
 def test_litellm_drop_params_env_var_is_parsed_as_a_flag(configured, expected):
     result = _import_litellm_with(configured)
 
@@ -23,8 +23,11 @@ def test_litellm_drop_params_env_var_is_parsed_as_a_flag(configured, expected):
     assert "is not a flag value" not in result.stderr
 
 
-def test_litellm_drop_params_env_var_non_flag_value_is_off_with_a_warning():
+def test_litellm_drop_params_env_var_non_flag_value_stays_on_with_a_warning():
     result = _import_litellm_with("temperature")
 
-    assert result.stdout.strip() == "False"
-    assert "LITELLM_DROP_PARAMS='temperature' is not a flag value, treating it as off" in result.stderr
+    assert result.stdout.strip() == "True"
+    assert (
+        "LITELLM_DROP_PARAMS='temperature' is not a flag value, treating it as on. Set it to true or false"
+        in result.stderr
+    )

From 5c037299f413c38609cbb7f5a582662e44b197d4 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Mon, 7 Sep 2026 23:02:27 -0700
Subject: [PATCH 55/85] feat: move MongoDB vector search to an optional sidecar

---
 .github/workflows/_test-unit-base.yml         |    2 +-
 Dockerfile                                    |    2 -
 docker/Dockerfile.database                    |    2 -
 docker/Dockerfile.non_root                    |    3 -
 gateway/Dockerfile                            |    2 -
 .../base_llm/vector_store/transformation.py   |    3 +
 litellm/llms/custom_httpx/llm_http_handler.py |   17 +-
 litellm/llms/mongodb/common_utils.py          |  303 ---
 .../mongodb/vector_stores/transformation.py   |  449 ++---
 pyproject.toml                                |    1 -
 .../test_mongodb_transformation.py            | 1691 ++---------------
 .../_components/VectorStoreForm.test.tsx      |   12 +-
 .../_components/VectorStoreForm.tsx           |    4 +-
 .../vector_store_providers.test.tsx           |   11 +-
 .../src/components/vector_store_providers.tsx |   27 +-
 uv.lock                                       |   79 +-
 16 files changed, 412 insertions(+), 2196 deletions(-)
 delete mode 100644 litellm/llms/mongodb/common_utils.py

diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml
index 75b0f93fd77..62790e23143 100644
--- a/.github/workflows/_test-unit-base.yml
+++ b/.github/workflows/_test-unit-base.yml
@@ -113,7 +113,7 @@ jobs:
         if: steps.changes.outputs.decision != 'skip'
         timeout-minutes: 8
         run: |
-          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra mongodb
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml
           uv run --no-sync python -c 'import os, sys; print(sys.version); assert f"{sys.version_info.major}.{sys.version_info.minor}" == os.environ["UV_PYTHON"]'
 
       - name: Cache Prisma binaries
diff --git a/Dockerfile b/Dockerfile
index 1648ec69d13..0a92aa9a68c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -67,7 +67,6 @@ RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-gr
     --extra semantic-router \
     --extra saml \
     --extra bedrock-realtime \
-    --extra mongodb \
     --python python3.13
 
 # Copy full source tree
@@ -90,7 +89,6 @@ RUN uv sync --frozen --no-default-groups --no-editable \
     --extra semantic-router \
     --extra saml \
     --extra bedrock-realtime \
-    --extra mongodb \
     --python python3.13
 
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database
index cc81ad6b3d3..e9ad2849bb2 100644
--- a/docker/Dockerfile.database
+++ b/docker/Dockerfile.database
@@ -65,7 +65,6 @@ RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-gr
     --extra semantic-router \
     --extra saml \
     --extra bedrock-realtime \
-    --extra mongodb \
     --python python3.13
 
 # Copy full source tree
@@ -88,7 +87,6 @@ RUN uv sync --frozen --no-default-groups --no-editable \
     --extra semantic-router \
     --extra saml \
     --extra bedrock-realtime \
-    --extra mongodb \
     --python python3.13
 
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root
index 358425af901..edf20e8bbff 100644
--- a/docker/Dockerfile.non_root
+++ b/docker/Dockerfile.non_root
@@ -71,7 +71,6 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
     --extra semantic-router \
     --extra saml \
     --extra bedrock-realtime \
-    --extra mongodb \
     --python python3.13
 
 # Copy full source tree
@@ -100,7 +99,6 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
         --extra semantic-router \
         --extra saml \
         --extra bedrock-realtime \
-        --extra mongodb \
         --python python3.13 \
         --no-sources-package litellm-proxy-extras; \
     else \
@@ -111,7 +109,6 @@ RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
         --extra semantic-router \
         --extra saml \
         --extra bedrock-realtime \
-        --extra mongodb \
         --python python3.13; \
     fi
 
diff --git a/gateway/Dockerfile b/gateway/Dockerfile
index e42e488d57f..308d70a6b26 100644
--- a/gateway/Dockerfile
+++ b/gateway/Dockerfile
@@ -47,7 +47,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra extra_proxy \
         --extra semantic-router \
         --extra bedrock-realtime \
-        --extra mongodb \
         --python python3.13
 
 # Stage 2 — copy source and install the project + workspace members.
@@ -60,7 +59,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \
         --extra extra_proxy \
         --extra semantic-router \
         --extra bedrock-realtime \
-        --extra mongodb \
         --python python3.13
 
 RUN HOME=/opt/prisma XDG_CACHE_HOME=/opt/prisma/.cache PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries \
diff --git a/litellm/llms/base_llm/vector_store/transformation.py b/litellm/llms/base_llm/vector_store/transformation.py
index c8d2b7fe522..07b60cb4b72 100644
--- a/litellm/llms/base_llm/vector_store/transformation.py
+++ b/litellm/llms/base_llm/vector_store/transformation.py
@@ -121,6 +121,9 @@ class RouterVectorStoreEmbeddingExecutor:
 
 
 class BaseVectorStoreConfig:
+    def validate_create_vector_store(self) -> None:
+        return None
+
     def get_supported_openai_params(self, model: str) -> list[VECTOR_STORE_OPENAI_PARAMS]:
         return []
 
diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py
index 2f561809940..1dc4b198890 100644
--- a/litellm/llms/custom_httpx/llm_http_handler.py
+++ b/litellm/llms/custom_httpx/llm_http_handler.py
@@ -9814,7 +9814,7 @@ class BaseLLMHTTPHandler:
                 vector_store_search_optional_params=vector_store_search_optional_params,
                 api_base=api_base,
                 litellm_logging_obj=logging_obj,
-                litellm_params=dict(litellm_params),
+                litellm_params={**dict(litellm_params), "timeout": timeout},
                 extra_body=extra_body,
                 embedding_executor=embedding_executor,
             )
@@ -9859,6 +9859,10 @@ class BaseLLMHTTPHandler:
                 data=request_data,
                 timeout=timeout,
             )
+        except httpx.TimeoutException:
+            raise vector_store_provider_config.get_error_class(
+                error_message="Vector store search exceeded the caller timeout.", status_code=408, headers={}
+            ) from None
         except Exception as e:
             raise self._handle_error(e=e, provider_config=vector_store_provider_config)
 
@@ -9943,7 +9947,7 @@ class BaseLLMHTTPHandler:
                 vector_store_search_optional_params=vector_store_search_optional_params,
                 api_base=api_base,
                 litellm_logging_obj=logging_obj,
-                litellm_params=dict(litellm_params),
+                litellm_params={**dict(litellm_params), "timeout": timeout},
                 extra_body=extra_body,
                 embedding_executor=embedding_executor,
             )
@@ -9988,7 +9992,12 @@ class BaseLLMHTTPHandler:
                 url=url,
                 headers=headers,
                 data=request_data,
+                timeout=timeout,
             )
+        except httpx.TimeoutException:
+            raise vector_store_provider_config.get_error_class(
+                error_message="Vector store search exceeded the caller timeout.", status_code=408, headers={}
+            ) from None
         except Exception as e:
             raise self._handle_error(e=e, provider_config=vector_store_provider_config)
 
@@ -10018,6 +10027,8 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
+        vector_store_provider_config.validate_create_vector_store()
+
         headers: Final = vector_store_provider_config.validate_environment(
             headers=extra_headers or {}, litellm_params=litellm_params
         )
@@ -10088,6 +10099,8 @@ class BaseLLMHTTPHandler:
         else:
             sync_httpx_client = client
 
+        vector_store_provider_config.validate_create_vector_store()
+
         headers: Final = vector_store_provider_config.validate_environment(
             headers=extra_headers or {}, litellm_params=litellm_params
         )
diff --git a/litellm/llms/mongodb/common_utils.py b/litellm/llms/mongodb/common_utils.py
deleted file mode 100644
index 02c0b359407..00000000000
--- a/litellm/llms/mongodb/common_utils.py
+++ /dev/null
@@ -1,303 +0,0 @@
-"""Shared helpers for the MongoDB integrations. pymongo lives in the optional ``mongodb`` extra,
-so every import of it is deferred to call time."""
-
-import asyncio
-import threading
-import weakref
-from asyncio import AbstractEventLoop
-from collections import OrderedDict
-from collections.abc import Callable, Mapping
-from dataclasses import dataclass
-from types import MappingProxyType
-from typing import TYPE_CHECKING, Final, TypeAlias, TypeVar
-
-from litellm.exceptions import BadRequestError, ServiceUnavailableError, Timeout
-
-if TYPE_CHECKING:
-    from pymongo import AsyncMongoClient, MongoClient
-
-PYMONGO_INSTALL_HINT: Final = (
-    "The MongoDB vector store requires the 'pymongo' package. "
-    "Run 'pip install litellm[mongodb]' (or 'pip install pymongo') to install it."
-)
-
-MONGODB_PROVIDER: Final = "mongodb"
-
-
-def config_error(message: str) -> BadRequestError:
-    """400 rather than the 500 a bare ValueError becomes once litellm.exception_type wraps it."""
-    return BadRequestError(message=message, model=None, llm_provider=MONGODB_PROVIDER)
-
-
-def timeout_error(message: str) -> Timeout:
-    return Timeout(message=message, model=None, llm_provider=MONGODB_PROVIDER)
-
-
-def unavailable_error(message: str) -> ServiceUnavailableError:
-    """litellm only retries 408, 409, 429 and 5xx, so a 400 here would make a failover permanent."""
-    return ServiceUnavailableError(message=message, model=None, llm_provider=MONGODB_PROVIDER)
-
-
-DEFAULT_CONNECT_TIMEOUT_MS: Final = 10_000
-DEFAULT_SOCKET_TIMEOUT_MS: Final = 30_000
-DEFAULT_SERVER_SELECTION_TIMEOUT_MS: Final = 10_000
-
-_MAX_CACHED_CLIENTS: Final = 32
-
-_APP_NAME: Final = "litellm"
-
-
-@dataclass(frozen=True, slots=True)
-class MongoClientKey:
-    connection_string: str
-    connect_timeout_ms: int
-    socket_timeout_ms: int
-    server_selection_timeout_ms: int
-
-
-SyncClientFactory: TypeAlias = Callable[..., "MongoClient"]
-AsyncClientFactory: TypeAlias = Callable[..., "AsyncMongoClient"]
-
-_K = TypeVar("_K")
-_V = TypeVar("_V")
-
-_AsyncClientCacheKey: TypeAlias = tuple[MongoClientKey, int]
-# CPython recycles id() aggressively, so the id alone would hand a new loop a closed loop's client
-_AsyncClientEntry: TypeAlias = tuple["weakref.ref[AbstractEventLoop]", "AsyncMongoClient"]
-
-_SyncClientCache: TypeAlias = "OrderedDict[MongoClientKey, MongoClient]"
-_AsyncClientCache: TypeAlias = "OrderedDict[_AsyncClientCacheKey, _AsyncClientEntry]"
-
-_sync_clients: Final[_SyncClientCache] = OrderedDict()  # mutable-ok: process-level client cache
-_async_clients: Final[_AsyncClientCache] = OrderedDict()  # mutable-ok: same cache, per loop
-# async searches reach the sync client through executor threads, so both caches are shared state
-_cache_lock: Final = threading.Lock()
-
-
-def _store_bounded(cache: "OrderedDict[_K, _V]", cache_key: "_K", value: "_V") -> None:
-    """Eviction only drops this cache's reference; an in-flight search keeps its client alive."""
-    with _cache_lock:
-        cache[cache_key] = value  # mutable-ok: an LRU cache is mutable state by definition
-        cache.move_to_end(cache_key)
-        while len(cache) > _MAX_CACHED_CLIENTS:
-            cache.popitem(last=False)
-
-
-def _mark_used(cache: "OrderedDict[_K, _V]", cache_key: "_K") -> None:
-    with _cache_lock:
-        if cache_key in cache:
-            cache.move_to_end(cache_key)
-
-
-def import_sync_mongo_client() -> "type[MongoClient]":
-    try:
-        from pymongo import MongoClient as SyncMongoClient
-    except ImportError as e:
-        raise config_error(PYMONGO_INSTALL_HINT) from e
-    return SyncMongoClient
-
-
-def import_async_mongo_client() -> "type[AsyncMongoClient]":
-    try:
-        from pymongo import AsyncMongoClient as AsyncMongoClientClass
-    except ImportError as e:
-        raise config_error(PYMONGO_INSTALL_HINT) from e
-    return AsyncMongoClientClass
-
-
-def _client_kwargs(key: MongoClientKey) -> Mapping[str, object]:
-    return MappingProxyType(
-        {
-            "connectTimeoutMS": key.connect_timeout_ms,
-            "socketTimeoutMS": key.socket_timeout_ms,
-            "serverSelectionTimeoutMS": key.server_selection_timeout_ms,
-            "appname": _APP_NAME,
-        }
-    )
-
-
-def get_sync_client(key: MongoClientKey, client_class: SyncClientFactory | None = None) -> "MongoClient":
-    cached: Final = _sync_clients.get(key)
-    if cached is not None:
-        _mark_used(_sync_clients, key)
-        return cached
-    build: Final = client_class if client_class is not None else import_sync_mongo_client()
-    client: Final = build(key.connection_string, **_client_kwargs(key))
-    _store_bounded(_sync_clients, key, client)
-    return client
-
-
-def _purge_dead_loops() -> None:
-    """A cached client holds its loop alive, so a closed loop's entry would pin that client and its
-    sockets for the life of the process."""
-    with _cache_lock:
-        for stale in tuple(
-            cache_key
-            for cache_key, (loop_ref, _) in _async_clients.items()
-            if (cached_loop := loop_ref()) is None or cached_loop.is_closed()
-        ):
-            del _async_clients[stale]
-
-
-def get_async_client(key: MongoClientKey, client_class: AsyncClientFactory | None = None) -> "AsyncMongoClient":
-    """Async clients bind to the loop that created them, so the cache is keyed per loop."""
-    loop: Final = asyncio.get_running_loop()
-    loop_key: Final = (key, id(loop))
-    cached: Final = _async_clients.get(loop_key)
-    if cached is not None and cached[0]() is loop:
-        _mark_used(_async_clients, loop_key)
-        return cached[1]
-    _purge_dead_loops()
-    build: Final = client_class if client_class is not None else import_async_mongo_client()
-    client: Final = build(key.connection_string, **_client_kwargs(key))
-    _store_bounded(_async_clients, loop_key, (weakref.ref(loop), client))
-    return client
-
-
-def reset_client_cache() -> None:
-    with _cache_lock:
-        _sync_clients.clear()
-        _async_clients.clear()
-
-
-_AUTHENTICATION_FAILED_CODE: Final = 18
-_UNAUTHORIZED_CODE: Final = 13
-# Atlas reports a rejected user as code 8000 "AtlasError" where a self-managed mongod reports 18
-_AUTHENTICATION_MESSAGE_MARKERS: Final = ("bad auth", "authentication failed", "not authorized")
-_RESOLUTION_TIMEOUT_MARKERS: Final = ("resolution lifetime expired", "dns operation timed out")
-_UNKNOWN_HOSTNAME_MARKERS: Final = ("dns query name does not exist", "name or service not known")
-_CREDENTIAL_ESCAPING_MARKERS: Final = ("must be escaped according to rfc 3986", "bad database name")
-
-
-def _index_hint(index_name: str, database: str, collection: str) -> str:
-    return (
-        f"No queryable MongoDB Vector Search index named '{index_name}' was found on "
-        f"'{database}.{collection}'. Confirm the index exists on that exact collection, that its "
-        "status is READY rather than still building, and that the vector store id matches the index name."
-    )
-
-
-def missing_index_error(index_name: str, database: str, collection: str) -> BadRequestError:
-    """$vectorSearch against a missing index, database or collection returns zero documents rather
-    than failing, so an empty result set is checked against the catalogue and reported as this."""
-    return config_error(
-        f"{_index_hint(index_name, database, collection)} A vector search against a database, "
-        "collection or index that does not exist returns no results rather than an error, so this "
-        "was reported as an empty result set by MongoDB."
-    )
-
-
-def index_not_ready_error(index_name: str, database: str, collection: str, status: str) -> BadRequestError:
-    return config_error(
-        f"The MongoDB Vector Search index '{index_name}' on '{database}.{collection}' is not queryable "
-        f"yet; its status is {status}. Searches against it return no results until the build finishes."
-    )
-
-
-def translate_mongo_error(error: Exception, index_name: str, database: str, collection: str) -> Exception:
-    """Returns the exception to raise, so callers keep the driver error as ``__cause__``."""
-    try:
-        from pymongo.errors import (
-            ConfigurationError,
-            ConnectionFailure,
-            ExecutionTimeout,
-            InvalidOperation,
-            NetworkTimeout,
-            OperationFailure,
-            ServerSelectionTimeoutError,
-        )
-    except ImportError:
-        return error
-
-    if isinstance(error, ServerSelectionTimeoutError):
-        return timeout_error(
-            "Could not reach the MongoDB deployment before the timeout. On Atlas this is usually the "
-            "project's IP access list not containing this host, or a paused cluster. On a self-managed "
-            "deployment it is usually the host or port in the URI, or a firewall between this process "
-            f"and mongod. Either way it can also be an unresolvable hostname. Driver detail: {error}"
-        )
-    # ExecutionTimeout subclasses OperationFailure, so it has to be matched before it
-    if isinstance(error, (NetworkTimeout, ExecutionTimeout)):
-        return timeout_error(
-            f"The MongoDB vector search against '{database}.{collection}' timed out before returning. "
-            f"Driver detail: {error}"
-        )
-    # ServerSelectionTimeoutError and NetworkTimeout also subclass ConnectionFailure, so this only
-    # sees what those branches left
-    if isinstance(error, ConnectionFailure):
-        return unavailable_error(
-            f"The connection to '{database}.{collection}' was dropped or refused. That is usually a "
-            "replica set failover or a restarted node, so the search is worth retrying. If it keeps "
-            "happening: on Atlas the usual cause is a connection string with no username and password, "
-            "or a TLS failure, so confirm the URI is the one Atlas shows under Connect, Drivers; on a "
-            "self-managed deployment, check that mongod is listening on the host and port in the URI. "
-            f"Driver detail: {error}"
-        )
-    if isinstance(error, OperationFailure):
-        code: Final = error.code
-        detail: Final = str(error).lower()
-        if code in (_AUTHENTICATION_FAILED_CODE, _UNAUTHORIZED_CODE) or any(
-            marker in detail for marker in _AUTHENTICATION_MESSAGE_MARKERS
-        ):
-            return config_error(
-                "MongoDB rejected the credentials in mongodb_connection_string, or the database user "
-                f"lacks read access to '{database}.{collection}'. Driver detail: {error.details}"
-            )
-        if "dimension" in detail:
-            return config_error(
-                "The query embedding does not match the vector dimensions the index was built for. "
-                "litellm_embedding_model must be the same model that produced the stored vectors. "
-                f"Driver detail: {error}"
-            )
-        if "is not indexed as vector" in detail:
-            return config_error(
-                "mongodb_embedding_field names a field the MongoDB Vector Search index does not cover. "
-                f"It must match the 'path' the index '{index_name}' was created on. Driver detail: {error}"
-            )
-        if "index" in detail and ("not found" in detail or "does not exist" in detail or "unknown" in detail):
-            return config_error(f"{_index_hint(index_name, database, collection)} Driver detail: {error}")
-        return config_error(
-            f"MongoDB rejected the vector search against '{database}.{collection}' using index "
-            f"'{index_name}'. Driver detail: {error}"
-        )
-    if isinstance(error, ConfigurationError):
-        configuration_detail: Final = str(error).lower()
-        if any(marker in configuration_detail for marker in _RESOLUTION_TIMEOUT_MARKERS):
-            return timeout_error(
-                "The DNS lookup for the cluster in mongodb_connection_string did not finish in time. "
-                "A mongodb+srv:// URI needs an SRV lookup before any connection is attempted, so this "
-                f"is DNS or the configured timeout, not MongoDB. Driver detail: {error}"
-            )
-        if any(marker in configuration_detail for marker in _UNKNOWN_HOSTNAME_MARKERS):
-            return config_error(
-                "The hostname in mongodb_connection_string does not exist in DNS. On Atlas, check the "
-                "cluster name against the URI shown under Connect, Drivers. On a self-managed deployment, "
-                f"check that the hostname resolves from this process. Driver detail: {error}"
-            )
-        if any(marker in configuration_detail for marker in _CREDENTIAL_ESCAPING_MARKERS):
-            return config_error(
-                "mongodb_connection_string could not be parsed. A username or password containing "
-                "'@', '/', ':' or '%' has to be percent-encoded per RFC 3986, so 'p@ss/word' becomes "
-                "'p%40ss%2Fword'. If the credentials are already encoded, check the database name in "
-                f"the URI path instead. Driver detail: {error}"
-            )
-        return config_error(
-            f"mongodb_connection_string is not a usable MongoDB connection string. Driver detail: {error}"
-        )
-    if isinstance(error, InvalidOperation):
-        return config_error(f"The MongoDB client was already closed or is unusable. Driver detail: {error}")
-    # An unreadable tlsCAFile or tlsCertificateKeyFile raises OSError, not a PyMongoError
-    if isinstance(error, OSError) and error.filename:
-        return config_error(
-            f"'{error.filename}', named by a TLS option in mongodb_connection_string, could not be read. "
-            "Check that tlsCAFile and tlsCertificateKeyFile point at files this process can open; inside "
-            f"a container that is the path in the container, not on the host. Driver detail: {error}"
-        )
-    # pymongo raises a plain ValueError, not a PyMongoError, for an unusable port
-    if isinstance(error, ValueError):
-        return config_error(
-            "The host and port in mongodb_connection_string could not be parsed. If the port is a "
-            "number between 0 and 65535, the cause is usually an unescaped ':' in the password, which "
-            f"has to be percent-encoded per RFC 3986 as '%3A'. Driver detail: {error}"
-        )
-    return error
diff --git a/litellm/llms/mongodb/vector_stores/transformation.py b/litellm/llms/mongodb/vector_stores/transformation.py
index 3382c931c96..92965ab745b 100644
--- a/litellm/llms/mongodb/vector_stores/transformation.py
+++ b/litellm/llms/mongodb/vector_stores/transformation.py
@@ -1,37 +1,28 @@
-"""MongoDB Vector Search has no HTTP query API, so this is a direct provider that runs the
-``$vectorSearch`` aggregation through pymongo. ``vector_store_id`` is the search index name."""
-
-from collections.abc import Callable, Mapping, Sequence
+from collections.abc import Mapping, Sequence
+from math import isfinite
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Final, NoReturn
+from typing import TYPE_CHECKING, Final, Literal, NoReturn
+from urllib.parse import quote, urlsplit
 
 import httpx
-from pydantic import BaseModel, ConfigDict
+from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
 
+from litellm.exceptions import AuthenticationError, BadRequestError, ServiceUnavailableError, Timeout
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.base_llm.vector_store.transformation import (
-    BaseDirectVectorStoreConfig,
+    BaseQueryEmbeddingVectorStoreConfig,
     LiteLLMVectorStoreEmbeddingExecutor,
     VectorStoreEmbeddingExecutor,
 )
-from litellm.llms.mongodb.common_utils import (
-    DEFAULT_CONNECT_TIMEOUT_MS,
-    DEFAULT_SERVER_SELECTION_TIMEOUT_MS,
-    DEFAULT_SOCKET_TIMEOUT_MS,
-    MongoClientKey,
-    config_error,
-    get_async_client,
-    get_sync_client,
-    index_not_ready_error,
-    missing_index_error,
-    translate_mongo_error,
-)
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import EmbeddingResponse
 from litellm.types.vector_stores import (
+    BaseVectorStoreAuthCredentials,
     VectorStoreCreateOptionalRequestParams,
-    VectorStoreResultContent,
+    VectorStoreIndexEndpoints,
     VectorStoreSearchOptionalRequestParams,
     VectorStoreSearchResponse,
-    VectorStoreSearchResult,
 )
 
 if TYPE_CHECKING:
@@ -39,26 +30,45 @@ if TYPE_CHECKING:
 
 DEFAULT_EMBEDDING_FIELD_NAME: Final = "embedding"
 DEFAULT_TEXT_FIELD_NAME: Final = "text"
-SCORE_FIELD_NAME: Final = "score"
-
 DEFAULT_MAX_NUM_RESULTS: Final = 10
 MIN_MAX_NUM_RESULTS: Final = 1
 MAX_MAX_NUM_RESULTS: Final = 50
-
 NUM_CANDIDATES_MULTIPLIER: Final = 10
 MIN_NUM_CANDIDATES: Final = 100
 MAX_NUM_CANDIDATES: Final = 10_000
-
 MAX_QUERY_CHARACTERS: Final = 32_000
-
 _EMPTY_EMBEDDING_CONFIG: Final = MappingProxyType({})
-
 _SEARCH_ONLY_MESSAGE: Final = (
     "MongoDB vector store is search-only. Create the collection and its MongoDB Vector Search "
     "index in MongoDB directly, then register it here by index name."
 )
 
 
+def config_error(message: str) -> BadRequestError:
+    return BadRequestError(message=message, model=None, llm_provider="mongodb")
+
+
+class _Content(BaseModel):
+    model_config = ConfigDict(frozen=True, strict=True)
+    type: Literal["text"]
+    text: str
+
+
+class _Result(BaseModel):
+    model_config = ConfigDict(frozen=True, strict=True, allow_inf_nan=False)
+    score: float | None
+    content: list[_Content]
+    file_id: str | None
+    filename: str | None
+
+
+class _SearchResponse(BaseModel):
+    model_config = ConfigDict(frozen=True, strict=True)
+    object: Literal["vector_store.search_results.page"]
+    search_query: str
+    data: list[_Result]
+
+
 class _MongoDBSearchParams(BaseModel):
     """Typed view over the vector store's litellm_params; unrelated keys are ignored."""
 
@@ -66,7 +76,6 @@ class _MongoDBSearchParams(BaseModel):
 
     litellm_embedding_model: str | None = None
     litellm_embedding_config: Mapping[str, object] | None = None
-    mongodb_connection_string: str | None = None
     mongodb_database: str | None = None
     mongodb_collection: str | None = None
     mongodb_text_field: str | None = None
@@ -91,21 +100,6 @@ class _MongoDBSearchParams(BaseModel):
             )
         return self.litellm_embedding_model
 
-    def require_connection_string(self) -> str:
-        if not self.mongodb_connection_string:
-            raise config_error(
-                "mongodb_connection_string is required in litellm_params for the MongoDB vector store. "
-                "Example: mongodb+srv://<user>:<password>@<cluster>.mongodb.net for Atlas, or "
-                "mongodb://<user>:<password>@<host>:27017 for a self-managed deployment"
-            )
-        scheme: Final = self.mongodb_connection_string.split("://", 1)[0].lower()
-        if scheme not in ("mongodb", "mongodb+srv"):
-            raise config_error(
-                "mongodb_connection_string must start with 'mongodb://' or 'mongodb+srv://', "
-                f"got '{self.mongodb_connection_string.split('://', 1)[0]}://'"
-            )
-        return self.mongodb_connection_string
-
     def require_database(self) -> str:
         if not self.mongodb_database:
             raise config_error(
@@ -127,30 +121,28 @@ _MONGODB_PARAM_PREFIX: Final = "mongodb_"
 _KNOWN_MONGODB_PARAMS: Final = frozenset(
     name for name in _MongoDBSearchParams.model_fields if name.startswith(_MONGODB_PARAM_PREFIX)
 )
+_RESPONSE_ADAPTER: Final = TypeAdapter(VectorStoreSearchResponse)
 
 
-class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig):
-    def __init__(
-        self,
-        embedding_executor: VectorStoreEmbeddingExecutor | None = None,
-        sync_client_factory: Callable[[MongoClientKey], object] | None = None,
-        async_client_factory: Callable[[MongoClientKey], object] | None = None,
-    ) -> None:
-        super().__init__()
-        self.embedding_executor: Final[VectorStoreEmbeddingExecutor] = (
-            embedding_executor if embedding_executor is not None else LiteLLMVectorStoreEmbeddingExecutor()
-        )
-        self.sync_client_factory: Final[Callable[[MongoClientKey], object]] = (
-            sync_client_factory if sync_client_factory is not None else get_sync_client
-        )
-        self.async_client_factory: Final[Callable[[MongoClientKey], object]] = (
-            async_client_factory if async_client_factory is not None else get_async_client
-        )
+class MongoDBVectorStoreConfig(BaseQueryEmbeddingVectorStoreConfig):
+    def __init__(self, embedding_executor: VectorStoreEmbeddingExecutor | None = None) -> None:
+        self.embedding_executor: Final = embedding_executor or LiteLLMVectorStoreEmbeddingExecutor()
+
+    def get_auth_credentials(self, litellm_params: Mapping[str, object]) -> BaseVectorStoreAuthCredentials:
+        return BaseVectorStoreAuthCredentials()
+
+    def get_vector_store_endpoints_by_type(self) -> VectorStoreIndexEndpoints:
+        return VectorStoreIndexEndpoints(read=[], write=[])  # mutable-ok: the TypedDict declares list fields
 
     @staticmethod
     def _reject_unknown_params(litellm_params: Mapping[str, object]) -> None:
         """Without this a mistyped mongodb_collection reads as 'mongodb_collection is required',
         naming a key the reader can see they have set."""
+        if litellm_params.get("mongodb_connection_string") is not None:
+            raise config_error(
+                "MongoDB vector stores now use the BETA sidecar. Move mongodb_connection_string to "
+                "MONGODB_CONNECTION_STRING in the sidecar, remove it from LiteLLM, and configure api_base and api_key."
+            )
         unknown: Final = sorted(
             key for key in litellm_params if key.startswith(_MONGODB_PARAM_PREFIX) and key not in _KNOWN_MONGODB_PARAMS
         )
@@ -191,239 +183,182 @@ class MongoDBVectorStoreConfig(BaseDirectVectorStoreConfig):
             return configured
         return min(max(limit * NUM_CANDIDATES_MULTIPLIER, MIN_NUM_CANDIDATES), MAX_NUM_CANDIDATES)
 
-    @staticmethod
-    def _timeout_ms(timeout: float | httpx.Timeout | None) -> tuple[int, int]:
-        """The connect and socket budgets pymongo is built with, in that order."""
-        if isinstance(timeout, httpx.Timeout):
-            return (
-                int((timeout.connect or DEFAULT_CONNECT_TIMEOUT_MS / 1000) * 1000),
-                int((timeout.read or DEFAULT_SOCKET_TIMEOUT_MS / 1000) * 1000),
+    def validate_environment(
+        self, headers: dict[str, object], litellm_params: GenericLiteLLMParams | None
+    ) -> dict[str, object]:
+        if litellm_params is None:
+            raise config_error("Configure api_base and api_key for the MongoDB BETA sidecar.")
+        self._reject_unknown_params(dict(litellm_params))
+        api_key: Final = litellm_params.api_key or get_secret_str("MONGODB_SIDECAR_API_KEY")
+        if not api_key:
+            raise config_error("MongoDB sidecar api_key is required. Set api_key or MONGODB_SIDECAR_API_KEY.")
+        return {**headers, "Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}
+
+    def get_complete_url(self, api_base: str | None, litellm_params: dict[str, object]) -> str:
+        if not api_base:
+            raise config_error("MongoDB sidecar api_base is required, for example http://mongodb-sidecar:8080.")
+        try:
+            parsed: Final = urlsplit(api_base)
+            valid: Final = parsed.scheme in ("http", "https") and bool(parsed.hostname) and parsed.port != 0
+        except ValueError:
+            raise config_error("MongoDB sidecar api_base must be a valid HTTP or HTTPS URL.") from None
+        if not valid or parsed.username or parsed.password or parsed.query or parsed.fragment:
+            raise config_error(
+                "MongoDB sidecar api_base must be an HTTP or HTTPS URL without credentials, query, or fragment."
             )
-        if timeout is None:
-            return DEFAULT_CONNECT_TIMEOUT_MS, DEFAULT_SOCKET_TIMEOUT_MS
-        return min(int(float(timeout) * 1000), DEFAULT_CONNECT_TIMEOUT_MS), int(float(timeout) * 1000)
+        return api_base.rstrip("/")
+
+    @staticmethod
+    def _timeout_ms(value: object) -> int:
+        seconds: Final = value.read if isinstance(value, httpx.Timeout) else value
+        if seconds is None:
+            return 30_000
+        if not isinstance(seconds, (int, float)) or not isfinite(seconds) or seconds <= 0:
+            raise config_error("MongoDB search timeout must be a positive finite number.")
+        return max(1, min(int(seconds * 1000), 30_000))
 
     @classmethod
-    def _client_key(cls, params: _MongoDBSearchParams, timeout: float | httpx.Timeout | None) -> MongoClientKey:
-        connect_ms, socket_ms = cls._timeout_ms(timeout)
-        return MongoClientKey(
-            connection_string=params.require_connection_string(),
-            connect_timeout_ms=connect_ms,
-            socket_timeout_ms=socket_ms,
-            server_selection_timeout_ms=min(connect_ms, DEFAULT_SERVER_SELECTION_TIMEOUT_MS),
-        )
+    def _params(
+        cls,
+        litellm_params: Mapping[str, object],
+        optional_params: VectorStoreSearchOptionalRequestParams,
+        extra_body: Mapping[str, object] | None,
+    ) -> _MongoDBSearchParams:
+        cls._reject_unknown_params(litellm_params)
+        if extra_body:
+            raise config_error("MongoDB vector store does not support extra_body overrides.")
+        for unsupported in ("filters", "ranking_options", "rewrite_query"):
+            if optional_params.get(unsupported) is not None:
+                raise config_error(f"MongoDB vector store does not support the {unsupported} parameter.")
+        try:
+            params: Final = _MongoDBSearchParams.model_validate(litellm_params)
+        except ValidationError:
+            raise config_error(
+                "Invalid MongoDB vector-store configuration. Check the database, collection, fields, and candidate count."
+            ) from None
+        params.require_database()
+        params.require_collection()
+        params.require_embedding_model()
+        cls._num_candidates(cls._limit(optional_params), params.mongodb_num_candidates)
+        cls._timeout_ms(litellm_params.get("timeout"))
+        return params
 
     @classmethod
-    def _pipeline(
+    def _request(
         cls,
         vector_store_id: str,
-        query_vector: Sequence[float],
+        query_text: str,
         params: _MongoDBSearchParams,
-        vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams,
-    ) -> Sequence[Mapping[str, object]]:
-        if vector_store_search_optional_params.get("filters") is not None:
+        optional_params: VectorStoreSearchOptionalRequestParams,
+        api_base: str,
+        embedding_response: EmbeddingResponse,
+        timeout: object,
+    ) -> tuple[str, dict[str, object]]:
+        if not embedding_response.data:
             raise config_error(
-                "MongoDB vector store does not support the filters parameter yet. "
-                "Restrict the collection or the MongoDB Vector Search index definition instead."
+                "The embedding model returned no embedding for the search query. Check litellm_embedding_model."
             )
-        if vector_store_search_optional_params.get("ranking_options") is not None:
-            raise config_error(
-                "MongoDB vector store does not support the ranking_options parameter yet. "
-                "Every result already carries the vectorSearchScore, so filter or re-rank "
-                "on that rather than having the threshold silently ignored."
-            )
-        if vector_store_search_optional_params.get("rewrite_query") is not None:
-            raise config_error(
-                "MongoDB vector store does not support the rewrite_query parameter. The query is "
-                "embedded exactly as sent; rewrite it before calling if you need that."
-            )
-        limit: Final = cls._limit(vector_store_search_optional_params)
-        search: Final = MappingProxyType(
-            {
-                "index": vector_store_id,
-                "path": params.embedding_field,
-                "queryVector": tuple(query_vector),
-                "numCandidates": cls._num_candidates(limit, params.mongodb_num_candidates),
-                "limit": limit,
-            }
-        )
-        projection: Final = MappingProxyType(
-            {params.text_field: 1, SCORE_FIELD_NAME: MappingProxyType({"$meta": "vectorSearchScore"})}
-        )
-        return [  # mutable-ok: pymongo rejects any non-list pipeline in common.validate_list
-            MappingProxyType({"$vectorSearch": search}),
-            MappingProxyType({"$project": projection}),
-        ]
+        vector: Final = embedding_response.data[0]["embedding"]
+        if not vector or any(not isinstance(value, (float, int)) or not isfinite(value) for value in vector):
+            raise config_error("The embedding model must return a non-empty, finite query vector.")
+        limit: Final = cls._limit(optional_params)
+        return f"{api_base}/v1/vector_stores/{quote(vector_store_id, safe='')}/search", {
+            "query": query_text,
+            "query_vector": tuple(vector),
+            "mongodb_database": params.require_database(),
+            "mongodb_collection": params.require_collection(),
+            "mongodb_embedding_field": params.embedding_field,
+            "mongodb_text_field": params.text_field,
+            "mongodb_num_candidates": cls._num_candidates(limit, params.mongodb_num_candidates),
+            "max_num_results": limit,
+            "timeout_ms": cls._timeout_ms(timeout),
+        }
 
-    @classmethod
-    def _field_value(cls, document: Mapping[str, object], dotted_path: str) -> str | None:
-        """None means absent, which is what separates a mistyped field from genuinely empty text."""
-        head, _, rest = dotted_path.partition(".")
-        if head not in document:
-            return None
-        value: Final = document[head]
-        if not rest:
-            return None if value is None else str(value)
-        return cls._field_value(value, rest) if isinstance(value, Mapping) else None
-
-    @classmethod
-    def _to_result(cls, document: Mapping[str, object], text_field: str) -> VectorStoreSearchResult:
-        document_id: Final = document.get("_id")
-        identifier: Final = None if document_id is None else str(document_id)
-        content: Final = [  # mutable-ok: VectorStoreSearchResult declares a list of content parts
-            VectorStoreResultContent(text=cls._field_value(document, text_field) or "", type="text")
-        ]
-        raw_score: Final = document.get(SCORE_FIELD_NAME)
-        return VectorStoreSearchResult(
-            score=float(raw_score) if isinstance(raw_score, (int, float)) else None,
-            content=content,
-            file_id=identifier,
-            filename=identifier,
-        )
-
-    @classmethod
-    def _raise_for_missing_text_field(
-        cls, documents: Sequence[Mapping[str, object]], text_field: str, database: str, collection: str
-    ) -> None:
-        """$vectorSearch matches documents carrying no text, so a mistyped mongodb_text_field
-        returns well-scored results with empty content instead of failing."""
-        if documents and all(cls._field_value(document, text_field) is None for document in documents):
-            raise config_error(
-                f"None of the {len(documents)} matched documents in '{database}.{collection}' has a "
-                f"'{text_field}' field, so every result would carry empty text. Set mongodb_text_field "
-                "to the field holding the readable text; it accepts a dotted path such as metadata.body."
-            )
-
-    @classmethod
-    def _to_response(
-        cls, documents: Sequence[Mapping[str, object]], query_text: str, text_field: str
-    ) -> VectorStoreSearchResponse:
-        return VectorStoreSearchResponse(
-            object="vector_store.search_results.page",
-            search_query=query_text,
-            data=[  # mutable-ok: VectorStoreSearchResponse declares data as a list
-                cls._to_result(document, text_field) for document in documents
-            ],
-        )
-
-    @staticmethod
-    def _raise_for_unusable_index(
-        catalogue: Sequence[Mapping[str, object]], index_name: str, database: str, collection: str
-    ) -> None:
-        """mongod returns zero documents both for a query that matched nothing and for a missing
-        database, collection or index, so the catalogue decides which one happened."""
-        if not catalogue:
-            raise missing_index_error(index_name, database, collection)
-        entry: Final = catalogue[0]
-        if not entry.get("queryable"):
-            raise index_not_ready_error(index_name, database, collection, str(entry.get("status") or "unknown"))
-
-    @staticmethod
-    def _embedding_vector(embedding_response: EmbeddingResponse) -> Sequence[float]:
-        data: Final = embedding_response.data
-        if not data:
-            raise config_error(
-                "The embedding model returned no embedding for the search query, so there is nothing "
-                "to search MongoDB with. Check the embedding deployment named by litellm_embedding_model."
-            )
-        return data[0]["embedding"]
-
-    def execute_search_vector_store_request(
+    def transform_search_vector_store_request(
         self,
         vector_store_id: str,
         query: str | Sequence[str],
         vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams,
+        api_base: str,
         litellm_logging_obj: "LiteLLMLoggingObj",
         litellm_params: Mapping[str, object],
+        extra_body: Mapping[str, object] | None = None,
         embedding_executor: VectorStoreEmbeddingExecutor | None = None,
-        timeout: float | httpx.Timeout | None = None,
-    ) -> VectorStoreSearchResponse:
-        self._reject_unknown_params(litellm_params)
-        params: Final = _MongoDBSearchParams.model_validate(litellm_params)
+    ) -> tuple[str, dict[str, object]]:
+        params: Final = self._params(litellm_params, vector_store_search_optional_params, extra_body)
         query_text: Final = self._query_text(query)
-        key: Final = self._client_key(params, timeout)
-        database: Final = params.require_database()
-        collection: Final = params.require_collection()
-
-        embedding_response: Final = (embedding_executor or self.embedding_executor).embed(
-            params.require_embedding_model(),
+        response: Final = (embedding_executor or self.embedding_executor).embed(
+            params.require_embedding_model(), query_text, params.litellm_embedding_config or _EMPTY_EMBEDDING_CONFIG
+        )
+        return self._request(
+            vector_store_id,
             query_text,
-            params.litellm_embedding_config or _EMPTY_EMBEDDING_CONFIG,
-        )
-        pipeline: Final = self._pipeline(
-            vector_store_id, self._embedding_vector(embedding_response), params, vector_store_search_optional_params
+            params,
+            vector_store_search_optional_params,
+            api_base,
+            response,
+            litellm_params.get("timeout"),
         )
 
-        try:
-            client: Final = self.sync_client_factory(key)
-            target: Final = client[database][collection]  # pyright: ignore[reportIndexIssue]  # factory is typed as returning object so injected doubles are accepted
-            documents: Final = tuple(target.aggregate(pipeline))
-        except Exception as e:
-            raise translate_mongo_error(e, index_name=vector_store_id, database=database, collection=collection) from e
-        if not documents:
-            try:
-                catalogue: Final = tuple(target.list_search_indexes(vector_store_id))
-            except Exception as e:
-                raise translate_mongo_error(
-                    e, index_name=vector_store_id, database=database, collection=collection
-                ) from e
-            self._raise_for_unusable_index(catalogue, vector_store_id, database, collection)
-        self._raise_for_missing_text_field(documents, params.text_field, database, collection)
-        return self._to_response(documents, query_text, params.text_field)
-
-    async def aexecute_search_vector_store_request(
+    async def atransform_search_vector_store_request(
         self,
         vector_store_id: str,
         query: str | Sequence[str],
         vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams,
+        api_base: str,
         litellm_logging_obj: "LiteLLMLoggingObj",
         litellm_params: Mapping[str, object],
+        extra_body: Mapping[str, object] | None = None,
         embedding_executor: VectorStoreEmbeddingExecutor | None = None,
-        timeout: float | httpx.Timeout | None = None,
-    ) -> VectorStoreSearchResponse:
-        self._reject_unknown_params(litellm_params)
-        params: Final = _MongoDBSearchParams.model_validate(litellm_params)
+    ) -> tuple[str, dict[str, object]]:
+        params: Final = self._params(litellm_params, vector_store_search_optional_params, extra_body)
         query_text: Final = self._query_text(query)
-        key: Final = self._client_key(params, timeout)
-        database: Final = params.require_database()
-        collection: Final = params.require_collection()
-
-        embedding_response: Final = await (embedding_executor or self.embedding_executor).aembed(
-            params.require_embedding_model(),
+        response: Final = await (embedding_executor or self.embedding_executor).aembed(
+            params.require_embedding_model(), query_text, params.litellm_embedding_config or _EMPTY_EMBEDDING_CONFIG
+        )
+        return self._request(
+            vector_store_id,
             query_text,
-            params.litellm_embedding_config or _EMPTY_EMBEDDING_CONFIG,
-        )
-        pipeline: Final = self._pipeline(
-            vector_store_id, self._embedding_vector(embedding_response), params, vector_store_search_optional_params
+            params,
+            vector_store_search_optional_params,
+            api_base,
+            response,
+            litellm_params.get("timeout"),
         )
 
+    def transform_search_vector_store_response(
+        self, response: httpx.Response, litellm_logging_obj: "LiteLLMLoggingObj"
+    ) -> VectorStoreSearchResponse:
         try:
-            client: Final = self.async_client_factory(key)
-            target: Final = client[database][collection]  # pyright: ignore[reportIndexIssue]  # factory is typed as returning object so injected doubles are accepted
-            cursor: Final = await target.aggregate(pipeline)
-            documents: Final = [  # mutable-ok: an async comprehension cannot build a tuple directly
-                document async for document in cursor
-            ]
-        except Exception as e:
-            raise translate_mongo_error(e, index_name=vector_store_id, database=database, collection=collection) from e
-        if not documents:
-            try:
-                index_cursor: Final = await target.list_search_indexes(vector_store_id)
-                catalogue: Final = [  # mutable-ok: an async comprehension cannot build a tuple directly
-                    entry async for entry in index_cursor
-                ]
-            except Exception as e:
-                raise translate_mongo_error(
-                    e, index_name=vector_store_id, database=database, collection=collection
-                ) from e
-            self._raise_for_unusable_index(catalogue, vector_store_id, database, collection)
-        self._raise_for_missing_text_field(documents, params.text_field, database, collection)
-        return self._to_response(documents, query_text, params.text_field)
+            validated: Final = _SearchResponse.model_validate_json(response.content)
+            return _RESPONSE_ADAPTER.validate_python(validated.model_dump())
+        except ValidationError:
+            raise ServiceUnavailableError(
+                message="MongoDB sidecar returned an invalid search response. Check the sidecar version and deployment.",
+                model=None,
+                llm_provider="mongodb",
+            ) from None
+
+    def get_error_class(
+        self, error_message: str, status_code: int, headers: dict[str, object] | httpx.Headers
+    ) -> BaseLLMException:
+        if status_code == 400:
+            raise config_error(error_message)
+        if status_code == 401:
+            raise AuthenticationError(message="MongoDB sidecar rejected api_key.", model=None, llm_provider="mongodb")
+        if status_code == 408:
+            raise Timeout(message=error_message, model=None, llm_provider="mongodb")
+        raise ServiceUnavailableError(
+            message="MongoDB sidecar is unavailable. Check its address, health, and logs.",
+            model=None,
+            llm_provider="mongodb",
+        )
+
+    def validate_create_vector_store(self) -> NoReturn:
+        raise config_error(_SEARCH_ONLY_MESSAGE)
 
     def transform_create_vector_store_request(
-        self,
-        vector_store_create_optional_params: VectorStoreCreateOptionalRequestParams,
-        api_base: str,
+        self, vector_store_create_optional_params: VectorStoreCreateOptionalRequestParams, api_base: str
     ) -> NoReturn:
         raise config_error(_SEARCH_ONLY_MESSAGE)
 
diff --git a/pyproject.toml b/pyproject.toml
index af35c77d259..b7eecfb2109 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -114,7 +114,6 @@ caching = ["diskcache>=5.6.3,<6.0"]
 mcp = ["mcp>=1.28.1,<2.0"]
 # Driver for the MongoDB Atlas vector store; Atlas Vector Search has no HTTP query API.
 # The floor is 4.9 because that is the release AsyncMongoClient landed in.
-mongodb = ["pymongo>=4.9,<5.0"]
 # SAML SSO for the admin UI. python3-saml pulls in xmlsec/lxml, whose wheels
 # bundle the native libxmlsec1/libxml2 libraries, so no system packages are
 # required. Kept out of the base `proxy` extra so it stays optional.
diff --git a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py
index f5d31c0da54..bca2b544673 100644
--- a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py
+++ b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py
@@ -1,1537 +1,182 @@
-import asyncio
-import gc
-import sys
-import threading
-import weakref
-from types import SimpleNamespace
-from unittest.mock import MagicMock, patch
+import json
+from collections.abc import Mapping
+from typing import Final
+from unittest.mock import MagicMock
 
 import httpx
 import pytest
 
 import litellm
-from litellm.exceptions import BadRequestError, ServiceUnavailableError, Timeout
-from litellm.llms.mongodb.common_utils import (
-    _MAX_CACHED_CLIENTS,
-    _async_clients,
-    _sync_clients,
-    MongoClientKey,
-    index_not_ready_error,
-    missing_index_error,
-    get_async_client,
-    get_sync_client,
-    reset_client_cache,
-    translate_mongo_error,
-)
-from litellm.llms.mongodb.vector_stores.transformation import (
-    MongoDBVectorStoreConfig,
-    _MongoDBSearchParams,
-)
-from litellm.types.utils import LlmProviders
-from litellm.utils import ProviderConfigManager
+from litellm.llms.custom_httpx.http_handler import HTTPHandler
+from litellm.llms.mongodb.vector_stores.transformation import MongoDBVectorStoreConfig
+from litellm.types.utils import EmbeddingResponse
+from litellm.types.vector_stores import VectorStoreSearchOptionalRequestParams
 
-CONNECTION_STRING = "mongodb+srv://user:pw@cluster.example.mongodb.net"
-INDEX = "movies_vector_index"
-
-BASE_PARAMS = {
-    "litellm_embedding_model": "openai/text-embedding-ada-002",
-    "mongodb_connection_string": CONNECTION_STRING,
-    "mongodb_database": "sample_mflix",
-    "mongodb_collection": "embedded_movies",
+BASE_PARAMS: Final = {
+    "api_base": "https://sidecar.example/prefix",
+    "api_key": "test-sidecar-key",
+    "litellm_embedding_model": "embedding-alias",
+    "mongodb_database": "policies",
+    "mongodb_collection": "documents",
+}
+RESULT: Final = {
+    "object": "vector_store.search_results.page",
+    "search_query": "travel policy",
+    "data": [
+        {"score": 0.9, "file_id": "123", "filename": "123", "content": [{"type": "text", "text": "Use code BLUE-42"}]}
+    ],
 }
 
 
-READY_INDEX = [{"name": INDEX, "status": "READY", "queryable": True}]
+class RecordingEmbeddingExecutor:
+    def __init__(self) -> None:
+        self.call: Final = MagicMock(return_value=EmbeddingResponse(data=[{"embedding": [0.1, 0.2, 0.3]}]))
+
+    def embed(self, model: str, query: str, configuration: Mapping[str, object]) -> EmbeddingResponse:
+        return self.call(model, query, configuration)
+
+    async def aembed(self, model: str, query: str, configuration: Mapping[str, object]) -> EmbeddingResponse:
+        return self.call(model, query, configuration)
 
 
-class RecordingClient:
-    """Stands in for pymongo's client class so the cache tests inject a fake rather than
-    patching the importer, and so they can assert what the client was actually built with."""
-
-    def __init__(self, connection_string, **kwargs):
-        self.connection_string = connection_string
-        self.kwargs = kwargs
-
-
-class FakeCollection:
-    def __init__(self, documents, error=None, search_indexes=None):
-        self.documents = documents
-        self.error = error
-        self.search_indexes = READY_INDEX if search_indexes is None else search_indexes
-        self.pipeline = None
-        self.listed_indexes = []
-
-    def aggregate(self, pipeline):
-        self.pipeline = pipeline
-        if self.error is not None:
-            raise self.error
-        return iter(self.documents)
-
-    def list_search_indexes(self, name):
-        self.listed_indexes.append(name)
-        return iter(self.search_indexes)
-
-
-class FakeAsyncCollection(FakeCollection):
-    async def aggregate(self, pipeline):
-        self.pipeline = pipeline
-        if self.error is not None:
-            raise self.error
-
-        async def cursor():
-            for document in self.documents:
-                yield document
-
-        return cursor()
-
-    async def list_search_indexes(self, name):
-        self.listed_indexes.append(name)
-
-        async def cursor():
-            for entry in self.search_indexes:
-                yield entry
-
-        return cursor()
-
-
-class FakeDatabase:
-    def __init__(self, collection):
-        self.collection = collection
-        self.requested_collection = None
-
-    def __getitem__(self, name):
-        self.requested_collection = name
-        return self.collection
-
-
-class FakeClient:
-    def __init__(self, collection):
-        self.database = FakeDatabase(collection)
-        self.requested_database = None
-
-    def __getitem__(self, name):
-        self.requested_database = name
-        return self.database
-
-
-class FakeEmbeddingExecutor:
-    def __init__(self, embedding):
-        self.embedding = embedding
-        self.captured = None
-
-    def _respond(self, model, query, configuration):
-        self.captured = SimpleNamespace(model=model, query=query, configuration=configuration)
-        return SimpleNamespace(data=[{"embedding": self.embedding}] if self.embedding is not None else [])
-
-    def embed(self, model, query, configuration):
-        return self._respond(model, query, configuration)
-
-    async def aembed(self, model, query, configuration):
-        return self._respond(model, query, configuration)
-
-
-def _config(documents=(), embedding=(0.1, 0.2, 0.3), error=None, search_indexes=None):
-    collection = FakeCollection(list(documents), error, search_indexes)
-    client = FakeClient(collection)
-    config = MongoDBVectorStoreConfig(
-        embedding_executor=FakeEmbeddingExecutor(list(embedding) if embedding is not None else None),
-        sync_client_factory=lambda key: client,
-    )
-    return config, client, collection
-
-
-def _async_config(documents=(), embedding=(0.1, 0.2, 0.3), error=None, search_indexes=None):
-    collection = FakeAsyncCollection(list(documents), error, search_indexes)
-    client = FakeClient(collection)
-    config = MongoDBVectorStoreConfig(
-        embedding_executor=FakeEmbeddingExecutor(list(embedding) if embedding is not None else None),
-        async_client_factory=lambda key: client,
-    )
-    return config, client, collection
-
-
-def _search(config, query="a lone astronaut", optional_params=None, litellm_params=None, timeout=None):
-    return config.execute_search_vector_store_request(
-        vector_store_id=INDEX,
-        query=query,
-        vector_store_search_optional_params=optional_params or {},
-        litellm_logging_obj=MagicMock(),
-        litellm_params={**BASE_PARAMS, **(litellm_params or {})},
-        timeout=timeout,
-    )
-
-
-async def _asearch(config, query="a lone astronaut", optional_params=None, litellm_params=None):
-    return await config.aexecute_search_vector_store_request(
-        vector_store_id=INDEX,
-        query=query,
-        vector_store_search_optional_params=optional_params or {},
-        litellm_logging_obj=MagicMock(),
-        litellm_params={**BASE_PARAMS, **(litellm_params or {})},
-    )
-
-
-def _stage(collection, name):
-    return next(stage[name] for stage in collection.pipeline if name in stage)
-
-
-def test_search_builds_vector_search_stage_against_the_named_index():
-    config, client, collection = _config()
-
-    _search(config, optional_params={"max_num_results": 5})
-
-    assert client.requested_database == "sample_mflix"
-    assert client.database.requested_collection == "embedded_movies"
-    assert _stage(collection, "$vectorSearch") == {
-        "index": INDEX,
-        "path": "embedding",
-        "queryVector": (0.1, 0.2, 0.3),
-        "numCandidates": 100,
-        "limit": 5,
+@pytest.mark.parametrize("asynchronous", [False, True])
+@pytest.mark.parametrize("limit,candidates", [(None, 100), (1, 100), (50, 500)])
+@pytest.mark.asyncio
+async def test_search_preserves_embedding_and_http_contract(
+    asynchronous: bool, limit: int | None, candidates: int
+) -> None:
+    executor: Final = RecordingEmbeddingExecutor()
+    config: Final = MongoDBVectorStoreConfig(executor)
+    params: Final = {
+        **BASE_PARAMS,
+        "mongodb_text_field": "metadata.body",
+        "mongodb_embedding_field": "stored_vector",
+        "litellm_embedding_config": {"dimensions": 3},
+        "timeout": 0.75,
     }
-
-
-def test_the_pipeline_reaches_pymongo_as_a_list():
-    """pymongo's common.validate_list rejects any other sequence with
-    'pipeline must be a list, not <class ...>', so the outer container is part of the contract."""
-    config, _, collection = _config()
-
-    _search(config)
-
-    assert isinstance(collection.pipeline, list)
-
-
-def test_search_projects_the_text_field_and_the_similarity_score():
-    config, _, collection = _config()
-
-    _search(config)
-
-    assert _stage(collection, "$project") == {"text": 1, "score": {"$meta": "vectorSearchScore"}}
-
-
-def test_search_defaults_to_ten_results():
-    config, _, collection = _config()
-
-    _search(config)
-
-    assert _stage(collection, "$vectorSearch")["limit"] == 10
-
-
-def test_search_honors_custom_field_names():
-    config, _, collection = _config()
-
-    _search(
-        config,
-        litellm_params={"mongodb_embedding_field": "plot_embedding", "mongodb_text_field": "plot"},
-    )
-
-    assert _stage(collection, "$vectorSearch")["path"] == "plot_embedding"
-    assert _stage(collection, "$project") == {"plot": 1, "score": {"$meta": "vectorSearchScore"}}
-
-
-def test_num_candidates_scales_with_the_requested_limit():
-    config, _, collection = _config()
-
-    _search(config, optional_params={"max_num_results": 40})
-
-    assert _stage(collection, "$vectorSearch")["numCandidates"] == 400
-
-
-def test_num_candidates_can_be_overridden():
-    config, _, collection = _config()
-
-    _search(config, optional_params={"max_num_results": 5}, litellm_params={"mongodb_num_candidates": 250})
-
-    assert _stage(collection, "$vectorSearch")["numCandidates"] == 250
-
-
-@pytest.mark.parametrize("configured", [4, 10_001])
-def test_num_candidates_below_the_limit_or_above_the_ceiling_is_rejected(configured):
-    config, _, _ = _config()
-
-    with pytest.raises(BadRequestError, match="mongodb_num_candidates"):
-        _search(config, optional_params={"max_num_results": 5}, litellm_params={"mongodb_num_candidates": configured})
-
-
-def test_list_query_is_joined_into_one_embedding_input():
-    config, _, _ = _config()
-
-    _search(config, query=["deep", "space", "rescue"])
-
-    assert config.embedding_executor.captured.query == "deep space rescue"
-
-
-def test_embedding_config_is_expanded_into_the_embedding_call():
-    config, _, _ = _config()
-
-    _search(config, litellm_params={"litellm_embedding_config": {"api_base": "https://example.test", "timeout": 7}})
-
-    captured = config.embedding_executor.captured
-    assert captured.configuration == {"api_base": "https://example.test", "timeout": 7}
-    assert captured.model == "openai/text-embedding-ada-002"
-
-
-def test_response_maps_documents_to_openai_shaped_results():
-    documents = [
-        {"_id": "abc123", "text": "an astronaut adrift", "score": 0.94},
-        {"_id": "def456", "text": "a robot dog", "score": 0.81},
-    ]
-    config, _, _ = _config(documents=documents)
-
-    response = _search(config)
-
-    assert response["object"] == "vector_store.search_results.page"
-    assert response["search_query"] == "a lone astronaut"
-    assert [result["score"] for result in response["data"]] == [0.94, 0.81]
-    assert [result["content"][0]["text"] for result in response["data"]] == ["an astronaut adrift", "a robot dog"]
-    assert [result["file_id"] for result in response["data"]] == ["abc123", "def456"]
-    assert [result["filename"] for result in response["data"]] == ["abc123", "def456"]
-    assert response["data"][0]["content"][0]["type"] == "text"
-
-
-def test_response_reads_a_dotted_text_field_path():
-    config, _, _ = _config(documents=[{"_id": 1, "metadata": {"body": "nested text"}, "score": 0.5}])
-
-    response = _search(config, litellm_params={"mongodb_text_field": "metadata.body"})
-
-    assert response["data"][0]["content"][0]["text"] == "nested text"
-
-
-def test_a_dotted_path_resolves_three_levels_deep():
-    config, _, _ = _config(documents=[{"_id": 1, "a": {"b": {"c": "deep text"}}, "score": 0.5}])
-
-    response = _search(config, litellm_params={"mongodb_text_field": "a.b.c"})
-
-    assert response["data"][0]["content"][0]["text"] == "deep text"
-
-
-def test_a_dotted_path_that_runs_through_a_scalar_counts_as_absent():
-    """Walking 'plot.nope' when plot is a string must report the misconfiguration, not
-    stringify the scalar and hand the model text from the wrong field."""
-    config, _, _ = _config(documents=[{"_id": 1, "plot": "a plain string", "score": 0.5}])
-
-    with pytest.raises(BadRequestError, match=r"has a 'plot\.nope' field"):
-        _search(config, litellm_params={"mongodb_text_field": "plot.nope"})
-
-
-def test_a_non_string_text_field_is_stringified():
-    config, _, _ = _config(documents=[{"_id": 1, "year": 1979, "score": 0.5}])
-
-    response = _search(config, litellm_params={"mongodb_text_field": "year"})
-
-    assert response["data"][0]["content"][0]["text"] == "1979"
-
-
-def test_a_null_text_field_counts_as_absent():
-    config, _, _ = _config(documents=[{"_id": 1, "text": None, "score": 0.5}])
-
-    with pytest.raises(BadRequestError, match="has a 'text' field"):
-        _search(config)
-
-
-def test_response_tolerates_a_sparse_document_missing_the_text_field():
-    config, _, _ = _config(documents=[{"_id": 1, "score": 0.5}, {"_id": 2, "text": "has text", "score": 0.4}])
-
-    response = _search(config)
-
-    assert response["data"][0]["content"][0]["text"] == ""
-    assert response["data"][1]["content"][0]["text"] == "has text"
-
-
-def test_a_present_but_empty_text_field_is_not_treated_as_a_misconfiguration():
-    config, _, _ = _config(documents=[{"_id": 1, "text": "", "score": 0.5}])
-
-    response = _search(config)
-
-    assert response["data"][0]["content"][0]["text"] == ""
-
-
-def test_matches_that_all_lack_the_text_field_name_the_setting_to_fix():
-    """Atlas matches on the vector, so a mistyped mongodb_text_field returns confidently
-    scored results whose content is empty and hands the model an empty context."""
-    config, _, _ = _config(documents=[{"_id": 1, "score": 0.9}, {"_id": 2, "score": 0.8}])
-
-    with pytest.raises(BadRequestError, match="mongodb_text_field"):
-        _search(config)
-
-
-def test_response_tolerates_a_document_missing_a_score():
-    config, _, _ = _config(documents=[{"_id": 1, "text": "no score"}])
-
-    response = _search(config)
-
-    assert response["data"][0]["score"] is None
-
-
-def test_response_stringifies_a_non_string_document_id():
-    config, _, _ = _config(documents=[{"_id": 12345, "text": "numeric id", "score": 0.5}])
-
-    response = _search(config)
-
-    assert response["data"][0]["file_id"] == "12345"
-
-
-def test_search_requires_an_embedding_model():
-    config, _, _ = _config()
-
-    with pytest.raises(BadRequestError, match="litellm_embedding_model is required"):
-        config.execute_search_vector_store_request(
-            vector_store_id=INDEX,
-            query="q",
-            vector_store_search_optional_params={},
+    kwargs: Final = {
+        "vector_store_id": "exact index",
+        "query": ["travel", "policy"],
+        "vector_store_search_optional_params": {"max_num_results": limit},
+        "api_base": BASE_PARAMS["api_base"],
+        "litellm_logging_obj": MagicMock(),
+        "litellm_params": params,
+    }
+    if asynchronous:
+        url, body = await config.atransform_search_vector_store_request(**kwargs)
+    else:
+        url, body = config.transform_search_vector_store_request(**kwargs)
+    assert url == "https://sidecar.example/prefix/v1/vector_stores/exact%20index/search"
+    assert body == {
+        "query": "travel policy",
+        "query_vector": (0.1, 0.2, 0.3),
+        "mongodb_database": "policies",
+        "mongodb_collection": "documents",
+        "mongodb_text_field": "metadata.body",
+        "mongodb_embedding_field": "stored_vector",
+        "mongodb_num_candidates": candidates,
+        "max_num_results": limit or 10,
+        "timeout_ms": 750,
+    }
+    executor.call.assert_called_once_with("embedding-alias", "travel policy", {"dimensions": 3})
+    assert config.transform_search_vector_store_response(httpx.Response(200, json=RESULT), MagicMock()) == RESULT
+
+
+@pytest.mark.parametrize(
+    "query,overrides,options",
+    [
+        ("", {}, {}),
+        ("  ", {}, {}),
+        ("x" * 32_001, {}, {}),
+        ("travel", {"litellm_embedding_model": None}, {}),
+        ("travel", {"mongodb_database": None}, {}),
+        ("travel", {"mongodb_collection": None}, {}),
+        ("travel", {"mongodb_connection_string": "mongodb://obsolete-secret"}, {}),
+        ("travel", {"mongodb_filter": {"private": True}}, {}),
+        ("travel", {"mongodb_num_candidates": 9}, {}),
+        ("travel", {"mongodb_num_candidates": 10_001}, {}),
+        ("travel", {}, {"max_num_results": 0}),
+        ("travel", {}, {"max_num_results": 51}),
+        ("travel", {}, {"filters": {}}),
+        ("travel", {}, {"ranking_options": {}}),
+        ("travel", {}, {"rewrite_query": False}),
+    ],
+)
+def test_invalid_search_is_rejected_before_embedding(
+    query: str, overrides: Mapping[str, object], options: VectorStoreSearchOptionalRequestParams
+) -> None:
+    executor: Final = RecordingEmbeddingExecutor()
+    config: Final = MongoDBVectorStoreConfig(executor)
+    with pytest.raises(litellm.BadRequestError) as error:
+        config.transform_search_vector_store_request(
+            vector_store_id="policy_index",
+            query=query,
+            vector_store_search_optional_params=options,
+            api_base=BASE_PARAMS["api_base"],
             litellm_logging_obj=MagicMock(),
-            litellm_params={k: v for k, v in BASE_PARAMS.items() if k != "litellm_embedding_model"},
+            litellm_params={**BASE_PARAMS, **overrides},
         )
-
-
-def test_missing_embedding_model_message_names_the_field_being_searched():
-    config, _, _ = _config()
-
-    with pytest.raises(BadRequestError, match=r"embedded_movies\.embedding"):
-        config.execute_search_vector_store_request(
-            vector_store_id=INDEX,
-            query="q",
-            vector_store_search_optional_params={},
-            litellm_logging_obj=MagicMock(),
-            litellm_params={k: v for k, v in BASE_PARAMS.items() if k != "litellm_embedding_model"},
-        )
-
-
-def test_search_requires_a_connection_string():
-    config, _, _ = _config()
-
-    with pytest.raises(BadRequestError, match="mongodb_connection_string is required"):
-        _search(config, litellm_params={"mongodb_connection_string": None})
-
-
-@pytest.mark.parametrize("connection_string", ["postgres://host/db", "https://cluster.mongodb.net", "redis://host"])
-def test_search_rejects_a_non_mongodb_connection_scheme(connection_string):
-    config, _, _ = _config()
-
-    with pytest.raises(BadRequestError, match="must start with 'mongodb://' or 'mongodb\\+srv://'"):
-        _search(config, litellm_params={"mongodb_connection_string": connection_string})
-
-
-def test_search_accepts_the_plain_mongodb_scheme():
-    config, _, collection = _config()
-
-    _search(config, litellm_params={"mongodb_connection_string": "mongodb://localhost:27017"})
-
-    assert collection.pipeline is not None
-
-
-def test_search_requires_a_database():
-    config, _, _ = _config()
-
-    with pytest.raises(BadRequestError, match="mongodb_database is required"):
-        _search(config, litellm_params={"mongodb_database": None})
-
-
-def test_search_requires_a_collection():
-    config, _, _ = _config()
-
-    with pytest.raises(BadRequestError, match="mongodb_collection is required"):
-        _search(config, litellm_params={"mongodb_collection": None})
-
-
-def test_search_rejects_filters_rather_than_silently_ignoring_them():
-    config, _, _ = _config()
-
-    with pytest.raises(BadRequestError, match="does not support the filters parameter"):
-        _search(config, optional_params={"filters": {"genre": "sci-fi"}})
-
-
-@pytest.mark.asyncio
-async def test_async_search_rejects_filters_rather_than_silently_ignoring_them():
-    config, _, _ = _async_config()
-
-    with pytest.raises(BadRequestError, match="does not support the filters parameter"):
-        await _asearch(config, optional_params={"filters": {"genre": "sci-fi"}})
-
-
-def test_search_rejects_ranking_options_rather_than_silently_ignoring_them():
-    """A score_threshold that is quietly dropped is worse than an error: the caller asked for
-    results above 0.9, gets results scoring 0.5, and nothing says the threshold never ran."""
-    config, _, _ = _config()
-
-    with pytest.raises(BadRequestError, match="does not support the ranking_options parameter"):
-        _search(config, optional_params={"ranking_options": {"score_threshold": 0.9}})
-
-
-def test_search_rejects_rewrite_query_rather_than_silently_ignoring_it():
-    config, _, _ = _config()
-
-    with pytest.raises(BadRequestError, match="does not support the rewrite_query parameter"):
-        _search(config, optional_params={"rewrite_query": True})
-
-
-@pytest.mark.asyncio
-async def test_async_search_rejects_ranking_options_rather_than_silently_ignoring_them():
-    config, _, _ = _async_config()
-
-    with pytest.raises(BadRequestError, match="does not support the ranking_options parameter"):
-        await _asearch(config, optional_params={"ranking_options": {"score_threshold": 0.9}})
-
-
-@pytest.mark.parametrize("query", ["", "   ", "\n\t", []])
-def test_search_rejects_an_empty_query(query):
-    config, _, _ = _config()
-
-    with pytest.raises(BadRequestError, match="query must not be empty"):
-        _search(config, query=query)
-
-
-def test_search_rejects_an_oversized_query():
-    config, _, _ = _config()
-
-    with pytest.raises(BadRequestError, match="at most 32000 characters"):
-        _search(config, query="x" * 32_001)
-
-
-def test_search_accepts_a_query_at_the_size_ceiling():
-    config, _, collection = _config()
-
-    _search(config, query="x" * 32_000)
-
-    assert collection.pipeline is not None
-
-
-@pytest.mark.parametrize("max_num_results", [0, -1, 51, 1000])
-def test_search_rejects_out_of_range_max_num_results(max_num_results):
-    config, _, _ = _config()
-
-    with pytest.raises(BadRequestError, match="max_num_results must be between 1 and 50"):
-        _search(config, optional_params={"max_num_results": max_num_results})
-
-
-@pytest.mark.parametrize("max_num_results", [1, 50])
-def test_search_allows_max_num_results_at_the_bounds(max_num_results):
-    config, _, collection = _config()
-
-    _search(config, optional_params={"max_num_results": max_num_results})
-
-    assert _stage(collection, "$vectorSearch")["limit"] == max_num_results
-
-
-def test_search_treats_an_explicit_null_max_num_results_as_the_default():
-    config, _, collection = _config()
-
-    _search(config, optional_params={"max_num_results": None})
-
-    assert _stage(collection, "$vectorSearch")["limit"] == 10
-
-
-def test_search_fails_when_the_embedding_model_returns_nothing():
-    config, _, _ = _config(embedding=None)
-
-    with pytest.raises(BadRequestError, match="returned no embedding"):
-        _search(config)
-
-
-def test_validation_runs_before_any_connection_is_opened():
-    opened = []
-    config = MongoDBVectorStoreConfig(
-        embedding_executor=FakeEmbeddingExecutor([0.1]),
-        sync_client_factory=lambda key: opened.append(key) or FakeClient(FakeCollection([])),
-    )
-
-    with pytest.raises(BadRequestError, match="query must not be empty"):
-        _search(config, query="")
-
-    assert opened == []
-
-
-def test_create_vector_store_is_not_supported_and_says_why():
-    """litellm.exception_type only passes its own exception types through untouched, so a
-    NotImplementedError here reaches the caller as APIConnectionError, which the proxy serves
-    as a 500 with a traceback. Refusing an unsupported operation is a client error."""
-    config = MongoDBVectorStoreConfig()
-
-    with pytest.raises(BadRequestError, match="search-only"):
-        config.transform_create_vector_store_request({}, "https://example.test")
-
-    with pytest.raises(BadRequestError, match="search-only"):
-        config.transform_create_vector_store_response(httpx.Response(200))
-
-
-def test_the_create_refusal_survives_the_public_sdk_error_wrapper():
-    import litellm
-
-    with pytest.raises(BadRequestError) as raised:
-        litellm.vector_stores.create(custom_llm_provider="mongodb", name="anything")
-
-    assert "search-only" in str(raised.value)
-
-
-def test_provider_config_manager_returns_the_mongodb_config():
-    config = ProviderConfigManager.get_provider_vector_stores_config(LlmProviders.MONGODB)
-
-    assert isinstance(config, MongoDBVectorStoreConfig)
-
-
-@pytest.mark.asyncio
-async def test_async_search_builds_the_same_pipeline_and_maps_the_response():
-    documents = [{"_id": "abc123", "text": "an astronaut adrift", "score": 0.94}]
-    config, client, collection = _async_config(documents=documents)
-
-    response = await _asearch(config, optional_params={"max_num_results": 3})
-
-    assert client.requested_database == "sample_mflix"
-    assert client.database.requested_collection == "embedded_movies"
-    assert _stage(collection, "$vectorSearch")["limit"] == 3
-    assert _stage(collection, "$vectorSearch")["queryVector"] == (0.1, 0.2, 0.3)
-    assert response["data"][0]["content"][0]["text"] == "an astronaut adrift"
-    assert response["data"][0]["score"] == 0.94
-
-
-@pytest.mark.asyncio
-async def test_async_search_requires_an_embedding_model():
-    config, _, _ = _async_config()
-
-    with pytest.raises(BadRequestError, match="litellm_embedding_model is required"):
-        await config.aexecute_search_vector_store_request(
-            vector_store_id=INDEX,
-            query="q",
-            vector_store_search_optional_params={},
-            litellm_logging_obj=MagicMock(),
-            litellm_params={k: v for k, v in BASE_PARAMS.items() if k != "litellm_embedding_model"},
-        )
-
-
-class TestClientCache:
-    def setup_method(self):
-        reset_client_cache()
-
-    def teardown_method(self):
-        reset_client_cache()
-
-    def _key(self, connection_string=CONNECTION_STRING, socket_timeout_ms=30_000):
-        return MongoClientKey(
-            connection_string=connection_string,
-            connect_timeout_ms=10_000,
-            socket_timeout_ms=socket_timeout_ms,
-            server_selection_timeout_ms=10_000,
-        )
-
-    def test_the_same_connection_reuses_one_client(self):
-        first = get_sync_client(self._key(), RecordingClient)
-        second = get_sync_client(self._key(), RecordingClient)
-
-        assert first is second
-        assert first.connection_string == CONNECTION_STRING
-        assert first.kwargs["socketTimeoutMS"] == 30_000
-        assert first.kwargs["connectTimeoutMS"] == 10_000
-        assert first.kwargs["appname"] == "litellm"
-
-    def test_a_different_connection_gets_its_own_client(self):
-        first = get_sync_client(self._key(), RecordingClient)
-        second = get_sync_client(self._key(connection_string="mongodb://other.example.test"), RecordingClient)
-
-        assert first is not second
-        assert second.connection_string == "mongodb://other.example.test"
-
-    def test_a_different_timeout_gets_its_own_client(self):
-        first = get_sync_client(self._key(), RecordingClient)
-        second = get_sync_client(self._key(socket_timeout_ms=5_000), RecordingClient)
-
-        assert first is not second
-        assert second.kwargs["socketTimeoutMS"] == 5_000
-
-    @pytest.mark.asyncio
-    async def test_async_clients_are_cached_per_event_loop(self):
-        first = get_async_client(self._key(), RecordingClient)
-        second = get_async_client(self._key(), RecordingClient)
-
-        assert first is second
-        assert first.connection_string == CONNECTION_STRING
-
-
-    def _fill_cache(self):
-        for slot in range(_MAX_CACHED_CLIENTS):
-            get_sync_client(self._key(f"mongodb://cold-{slot}:27017"), RecordingClient)
-
-    def test_a_store_added_after_the_cache_filled_is_still_cached(self):
-        """Rebuilding a client costs an SRV lookup, a TLS handshake and topology discovery, so a
-        store that misses the cache on every single search pays that on every search."""
-        self._fill_cache()
-        latecomer = self._key("mongodb://latecomer:27017")
-
-        first = get_sync_client(latecomer, RecordingClient)
-
-        assert get_sync_client(latecomer, RecordingClient) is first
-
-    def test_the_cache_evicts_the_least_recently_used_client(self):
-        self._fill_cache()
-        oldest = self._key("mongodb://cold-0:27017")
-        newest = self._key(f"mongodb://cold-{_MAX_CACHED_CLIENTS - 1}:27017")
-        kept = get_sync_client(newest, RecordingClient)
-
-        get_sync_client(self._key("mongodb://latecomer:27017"), RecordingClient)
-
-        assert get_sync_client(newest, RecordingClient) is kept
-        assert oldest not in _sync_clients
-
-    def test_concurrent_searches_never_trip_over_an_eviction(self):
-        """Async searches run the sync client through executor threads, so a key can be evicted
-        between the lookup and the reordering that follows it."""
-        errors = []
-        churn = _MAX_CACHED_CLIENTS + 2
-
-        def hammer(offset):
-            try:
-                for step in range(3_000):
-                    get_sync_client(self._key(f"mongodb://h-{(step + offset) % churn}:27017"), RecordingClient)
-            except Exception as e:
-                errors.append(repr(e))
-
-        previous = sys.getswitchinterval()
-        sys.setswitchinterval(1e-9)
-        try:
-            threads = [threading.Thread(target=hammer, args=(offset,)) for offset in range(16)]
-            for thread in threads:
-                thread.start()
-            for thread in threads:
-                thread.join()
-        finally:
-            sys.setswitchinterval(previous)
-
-        assert errors == []
-
-    def test_the_cache_never_grows_past_its_cap(self):
-        for slot in range(_MAX_CACHED_CLIENTS * 3):
-            get_sync_client(self._key(f"mongodb://host-{slot}:27017"), RecordingClient)
-
-        assert len(_sync_clients) == _MAX_CACHED_CLIENTS
-
-    def test_a_new_loop_never_inherits_a_closed_loop_client(self):
-        """CPython recycles id() so aggressively that a fresh event loop almost always lands on
-        the id of one already collected: measured at 37 of 40 rounds. Keying the cache on the id
-        alone therefore hands the new loop an AsyncMongoClient bound to a closed loop, and every
-        operation on it raises "Event loop is closed"."""
-
-        class LoopAgnosticClient:
-            """Holds no reference to the loop, unlike pymongo's, whose own reference happens to
-            keep ids from being recycled and hides the bug until the cache fills."""
-
-            def __init__(self, *args, **kwargs):
-                self.built_on = None
-
-        key = self._key()
-        clients_handed_out = []
-
-        async def fetch():
-            return get_async_client(key, LoopAgnosticClient)
-
-        for _ in range(20):
-            loop = asyncio.new_event_loop()
-            client = loop.run_until_complete(fetch())
-            clients_handed_out.append((client, client.built_on, loop.is_closed()))
-            client.built_on = weakref.ref(loop)
-            loop.close()
-            del loop
-            gc.collect()
-
-        stale = [
-            handed_out
-            for client, built_on, _ in clients_handed_out
-            if built_on is not None and (built_on() is None or built_on().is_closed())
-            for handed_out in (client,)
-        ]
-        assert stale == [], f"{len(stale)} of 20 loops were handed a client built on a closed loop"
-
-    def test_the_cache_releases_clients_built_on_closed_loops(self):
-        """pymongo's AsyncMongoClient keeps a reference to the loop it was built on, so an entry
-        for a closed loop holds that client, and its sockets, for the life of the process. A
-        script calling asyncio.run per search fills the cache to its cap that way: measured live
-        against Atlas at 32 pinned clients and 212 open descriptors after 40 loops."""
-
-        class LoopHoldingClient:
-            def __init__(self, *args, **kwargs):
-                self.loop = asyncio.get_running_loop()
-
-        key = self._key()
-
-        async def fetch():
-            return get_async_client(key, LoopHoldingClient)
-
-        for _ in range(_MAX_CACHED_CLIENTS + 8):
-            loop = asyncio.new_event_loop()
-            loop.run_until_complete(fetch())
-            loop.close()
-
-        assert len(_async_clients) == 1, f"{len(_async_clients)} closed-loop clients are still cached"
-
-
-class TestClientKeyDerivation:
-    def test_no_timeout_uses_the_bounded_defaults(self):
-        key = MongoDBVectorStoreConfig._client_key(_MongoDBSearchParams.model_validate(BASE_PARAMS), None)
-
-        assert key.connect_timeout_ms == 10_000
-        assert key.socket_timeout_ms == 30_000
-        assert key.server_selection_timeout_ms == 10_000
-
-    def test_a_numeric_timeout_bounds_the_connect_phase(self):
-        key = MongoDBVectorStoreConfig._client_key(_MongoDBSearchParams.model_validate(BASE_PARAMS), 3.0)
-
-        assert key.socket_timeout_ms == 3_000
-        assert key.connect_timeout_ms == 3_000
-
-    def test_a_short_timeout_also_shortens_server_selection(self):
-        """Server selection runs before the connect attempt, so leaving it at the 10s default
-        would let a caller asking for a 3s budget block for 10s before anything is tried."""
-        key = MongoDBVectorStoreConfig._client_key(_MongoDBSearchParams.model_validate(BASE_PARAMS), 3.0)
-
-        assert key.server_selection_timeout_ms == 3_000
-
-    def test_a_generous_timeout_does_not_raise_server_selection_above_the_default(self):
-        key = MongoDBVectorStoreConfig._client_key(_MongoDBSearchParams.model_validate(BASE_PARAMS), 120.0)
-
-        assert key.socket_timeout_ms == 120_000
-        assert key.server_selection_timeout_ms == 10_000
-
-    def test_an_httpx_timeout_maps_connect_and_read_separately(self):
-        key = MongoDBVectorStoreConfig._client_key(
-            _MongoDBSearchParams.model_validate(BASE_PARAMS), httpx.Timeout(connect=2.0, read=45.0, write=5.0, pool=5.0)
-        )
-
-        assert key.connect_timeout_ms == 2_000
-        assert key.socket_timeout_ms == 45_000
-
-
-class TestErrorTranslation:
-    def _translate(self, error):
-        return translate_mongo_error(error, index_name=INDEX, database="sample_mflix", collection="embedded_movies")
-
-    def test_server_selection_timeout_points_at_the_atlas_access_list(self):
-        from pymongo.errors import ServerSelectionTimeoutError
-
-        translated = self._translate(ServerSelectionTimeoutError("no servers"))
-
-        assert "IP access list" in str(translated)
-        assert "paused cluster" in str(translated)
-
-    def test_authentication_failure_points_at_the_connection_string_credentials(self):
-        from pymongo.errors import OperationFailure
-
-        translated = self._translate(OperationFailure("auth failed", code=18))
-
-        assert "rejected the credentials" in str(translated)
-
-    def test_a_dropped_connection_stays_retryable(self):
-        """A replica set failover reaches the driver as AutoReconnect. litellm only retries 408,
-        409, 429 and 5xx, so classifying it as a client error would turn one failover into a
-        permanently failed search."""
-        from pymongo.errors import AutoReconnect
-
-        translated = self._translate(AutoReconnect("connection closed"))
-
-        assert litellm._should_retry(translated.status_code)
-        assert "dropped or refused" in str(translated)
-
-    def test_a_dropped_connection_still_names_the_misconfigurations_behind_it(self):
-        """Atlas answers a URI with no credentials by closing the connection rather than failing
-        auth, so the retryable message still has to name that."""
-        from pymongo.errors import AutoReconnect
-
-        translated = self._translate(AutoReconnect("connection closed"))
-
-        assert "no username and password" in str(translated)
-        assert "mongod is listening" in str(translated)
-
-    def test_the_retryable_classification_survives_the_public_sdk_error_wrapper(self):
-        """litellm.exception_type only passes its own exception types through; anything else becomes
-        an APIConnectionError and a 500, which would drop the retryable classification."""
-        from pymongo.errors import AutoReconnect
-
-        translated = self._translate(AutoReconnect("connection closed"))
-
-        wrapped = litellm.exception_type(
-            model=None,
-            original_exception=translated,
-            custom_llm_provider="mongodb",
-            completion_kwargs={},
-            extra_kwargs={},
-        )
-
-        assert isinstance(wrapped, ServiceUnavailableError)
-        assert litellm._should_retry(wrapped.status_code)
-
-    def test_a_pool_wait_queue_timeout_stays_retryable(self):
-        from pymongo.errors import WaitQueueTimeoutError
-
-        translated = self._translate(WaitQueueTimeoutError("timed out waiting for a connection"))
-
-        assert litellm._should_retry(translated.status_code)
-
-    def test_server_selection_timeout_still_wins_over_the_connection_branch(self):
-        from pymongo.errors import ServerSelectionTimeoutError
-
-        translated = self._translate(ServerSelectionTimeoutError("no servers"))
-
-        assert isinstance(translated, Timeout)
-        assert "dropped or refused" not in str(translated)
-
-    def test_network_timeout_still_wins_over_the_connection_branch(self):
-        from pymongo.errors import NetworkTimeout
-
-        translated = self._translate(NetworkTimeout("socket timed out"))
-
-        assert isinstance(translated, Timeout)
-        assert "dropped or refused" not in str(translated)
-
-    def test_an_unescaped_password_character_is_a_400_not_a_500(self):
-        """pymongo's URI parser raises a plain ValueError, not a PyMongoError, for an unusable port,
-        which is also what an unescaped ':' in a password produces. It must not be a 500."""
-        translated = self._translate(ValueError("Port contains non-digit characters"))
-
-        assert isinstance(translated, BadRequestError)
-        assert "percent-encoded" in str(translated)
-
-    def test_unauthorized_points_at_the_database_user_permissions(self):
-        from pymongo.errors import OperationFailure
-
-        translated = self._translate(OperationFailure("not authorized", code=13))
-
-        assert "sample_mflix.embedded_movies" in str(translated)
-
-    def test_code_13_alone_is_enough_without_a_recognisable_message(self):
-        """The other unauthorized case carries "not authorized", which the message markers also
-        match, so it cannot tell whether the code is still being checked at all."""
-        from pymongo.errors import OperationFailure
-
-        translated = self._translate(OperationFailure("user lacks privileges on this namespace", code=13))
-
-        assert "rejected the credentials" in str(translated)
-        assert "sample_mflix.embedded_movies" in str(translated)
-
-    def test_a_missing_index_names_the_index_and_the_collection(self):
-        from pymongo.errors import OperationFailure
-
-        translated = self._translate(OperationFailure("Index not found for name movies_vector_index", code=27))
-
-        assert INDEX in str(translated)
-        assert "READY" in str(translated)
-
-    def test_a_dimension_mismatch_points_at_the_embedding_model(self):
-        from pymongo.errors import OperationFailure
-
-        translated = self._translate(OperationFailure("queryVector has 1536 dimensions, index expects 2048"))
-
-        assert "litellm_embedding_model must be the same model" in str(translated)
-
-    def test_an_unrecognised_operation_failure_still_names_the_target(self):
-        from pymongo.errors import OperationFailure
-
-        translated = self._translate(OperationFailure("something else entirely"))
-
-        assert "sample_mflix.embedded_movies" in str(translated)
-        assert INDEX in str(translated)
-
-    def test_a_configuration_error_points_at_the_connection_string(self):
-        from pymongo.errors import ConfigurationError
-
-        translated = self._translate(ConfigurationError("bad uri"))
-
-        assert "not a usable MongoDB connection string" in str(translated)
-
-    def test_a_non_driver_error_is_returned_unchanged(self):
-        original = RuntimeError("unrelated")
-
-        assert self._translate(original) is original
-
-    def test_search_surfaces_a_translated_driver_error(self):
-        from pymongo.errors import ServerSelectionTimeoutError
-
-        config, _, _ = _config(error=ServerSelectionTimeoutError("no servers"))
-
-        with pytest.raises(Timeout, match="IP access list"):
-            _search(config)
-
-    @pytest.mark.asyncio
-    async def test_async_search_surfaces_a_translated_driver_error(self):
-        from pymongo.errors import OperationFailure
-
-        config, _, _ = _async_config(error=OperationFailure("auth failed", code=18))
-
-        with pytest.raises(BadRequestError, match="rejected the credentials"):
-            await _asearch(config)
-
-
-class TestMissingDriver:
-    def test_the_sync_import_names_the_extra_to_install(self):
-        from litellm.llms.mongodb.common_utils import import_sync_mongo_client
-
-        with patch.dict(sys.modules, {"pymongo": None}):
-            with pytest.raises(BadRequestError, match=r"pip install litellm\[mongodb\]"):
-                import_sync_mongo_client()
-
-    def test_the_async_import_names_the_extra_to_install(self):
-        from litellm.llms.mongodb.common_utils import import_async_mongo_client
-
-        with patch.dict(sys.modules, {"pymongo": None}):
-            with pytest.raises(BadRequestError, match=r"pip install litellm\[mongodb\]"):
-                import_async_mongo_client()
-
-    def test_error_translation_degrades_gracefully_without_the_driver(self):
-        original = RuntimeError("boom")
-
-        with patch.dict(sys.modules, {"pymongo.errors": None}):
-            assert translate_mongo_error(original, INDEX, "db", "col") is original
-
-
-class TestEmptyResultsAreDisambiguated:
-    """$vectorSearch returns zero documents for a missing database, collection or index just as it
-    does for a query that matched nothing, so an empty result set is checked against the index
-    catalogue before it is reported as 'no matches'."""
-
-    def test_a_missing_index_becomes_an_error_rather_than_an_empty_page(self):
-        config, _, collection = _config(documents=[], search_indexes=[])
-
-        with pytest.raises(BadRequestError, match="No queryable MongoDB Vector Search index"):
-            _search(config)
-
-        assert collection.listed_indexes == [INDEX]
-
-    def test_the_missing_index_error_explains_why_mongodb_reported_no_results(self):
-        config, _, _ = _config(documents=[], search_indexes=[])
-
-        with pytest.raises(BadRequestError, match="returns no results rather than an error"):
-            _search(config)
-
-    def test_an_index_still_building_becomes_an_error_naming_its_status(self):
-        config, _, _ = _config(
-            documents=[], search_indexes=[{"name": INDEX, "status": "PENDING", "queryable": False}]
-        )
-
-        with pytest.raises(BadRequestError, match="not queryable yet; its status is PENDING"):
-            _search(config)
-
-    def test_a_genuine_no_match_against_a_ready_index_returns_an_empty_page(self):
-        config, _, collection = _config(documents=[])
-
-        response = _search(config)
-
-        assert response["data"] == []
-        assert response["object"] == "vector_store.search_results.page"
-        assert collection.listed_indexes == [INDEX]
-
-    def test_the_catalogue_is_not_consulted_when_the_search_returned_hits(self):
-        config, _, collection = _config(documents=[{"_id": 1, "text": "hit", "score": 0.9}])
-
-        _search(config)
-
-        assert collection.listed_indexes == []
-
-    @pytest.mark.asyncio
-    async def test_async_missing_index_becomes_an_error_rather_than_an_empty_page(self):
-        config, _, collection = _async_config(documents=[], search_indexes=[])
-
-        with pytest.raises(BadRequestError, match="No queryable MongoDB Vector Search index"):
-            await _asearch(config)
-
-        assert collection.listed_indexes == [INDEX]
-
-    @pytest.mark.asyncio
-    async def test_async_index_still_building_becomes_an_error_naming_its_status(self):
-        config, _, _ = _async_config(
-            documents=[], search_indexes=[{"name": INDEX, "status": "PENDING", "queryable": False}]
-        )
-
-        with pytest.raises(BadRequestError, match="not queryable yet; its status is PENDING"):
-            await _asearch(config)
-
-    @pytest.mark.asyncio
-    async def test_async_genuine_no_match_returns_an_empty_page(self):
-        config, _, _ = _async_config(documents=[])
-
-        response = await _asearch(config)
-
-        assert response["data"] == []
-
-    @pytest.mark.asyncio
-    async def test_async_catalogue_is_not_consulted_when_the_search_returned_hits(self):
-        config, _, collection = _async_config(documents=[{"_id": 1, "text": "hit", "score": 0.9}])
-
-        await _asearch(config)
-
-        assert collection.listed_indexes == []
-
-    def test_a_failure_while_checking_the_catalogue_is_translated_too(self):
-        from pymongo.errors import OperationFailure
-
-        class ExplodingCollection(FakeCollection):
-            def list_search_indexes(self, name):
-                raise OperationFailure("not authorized", code=13)
-
-        collection = ExplodingCollection([], None, [])
-        config = MongoDBVectorStoreConfig(
-            embedding_executor=FakeEmbeddingExecutor([0.1]),
-            sync_client_factory=lambda key: FakeClient(collection),
-        )
-
-        with pytest.raises(BadRequestError, match="lacks read access"):
-            _search(config)
-
-
-class TestAtlasPlanExecutorErrors:
-    """Atlas reports a wrong vector path and a dimension mismatch through the same error code, so
-    each one has to be told apart by its message or both come back as a generic index failure."""
-
-    def _translate(self, message):
-        from pymongo.errors import OperationFailure
-
-        return translate_mongo_error(
-            OperationFailure(message, code=8),
-            index_name=INDEX,
-            database="sample_mflix",
-            collection="embedded_movies",
-        )
-
-    def test_a_wrong_vector_path_points_at_the_embedding_field_setting(self):
-        translated = self._translate(
-            "PlanExecutor error during aggregation :: caused by :: nope is not indexed as vector"
-        )
-
-        assert "mongodb_embedding_field names a field" in str(translated)
-
-    def test_a_dimension_mismatch_is_not_reported_as_a_wrong_path(self):
-        translated = self._translate(
-            "PlanExecutor error during aggregation :: caused by :: vector field is indexed with "
-            "1536 dimensions but queried with 3072"
-        )
-
-        assert "does not match the vector dimensions" in str(translated)
-        assert "mongodb_embedding_field" not in str(translated)
-
-
-class TestErrorsCarryTheRightHttpStatus:
-    """litellm.exception_type passes a litellm exception through untouched but wraps anything
-    else into APIConnectionError, which the proxy serves as a 500 with a Python traceback in the
-    body. A misconfigured connection string is the caller's to fix, so it has to arrive as a 400.
-    """
-
-    @pytest.mark.parametrize(
-        "invoke",
-        [
-            pytest.param(lambda: _search(_config()[0], query="  "), id="empty-query"),
-            pytest.param(
-                lambda: _search(_config()[0], optional_params={"max_num_results": 999}),
-                id="max-num-results-out-of-range",
-            ),
-            pytest.param(
-                lambda: _search(_config()[0], optional_params={"filters": {"genre": "Action"}}),
-                id="unsupported-filters",
-            ),
-            pytest.param(
-                lambda: _search(_config()[0], litellm_params={"mongodb_connection_string": "postgres://host/db"}),
-                id="wrong-uri-scheme",
-            ),
-            pytest.param(
-                lambda: _search(_config()[0], litellm_params={"mongodb_database": None}), id="missing-database"
-            ),
-            pytest.param(
-                lambda: _search(_config()[0], litellm_params={"litellm_embedding_model": None}),
-                id="missing-embedding-model",
-            ),
-        ],
-    )
-    def test_configuration_failures_are_400(self, invoke):
-        with pytest.raises(BadRequestError) as excinfo:
-            invoke()
-        assert excinfo.value.status_code == 400
-        assert excinfo.value.llm_provider == "mongodb"
-
-    def test_missing_index_is_400(self):
-        error = missing_index_error("idx", "db", "coll")
-        assert error.status_code == 400
-        assert error.llm_provider == "mongodb"
-
-    def test_index_still_building_is_400(self):
-        error = index_not_ready_error("idx", "db", "coll", "PENDING")
-        assert error.status_code == 400
-
-    def test_unreachable_deployment_is_a_timeout_not_a_bad_request(self):
-        from pymongo.errors import ServerSelectionTimeoutError
-
-        translated = translate_mongo_error(
-            ServerSelectionTimeoutError("no servers"), index_name="idx", database="db", collection="coll"
-        )
-        assert isinstance(translated, Timeout)
-        assert translated.status_code == 408
-
-    def test_query_execution_timeout_is_a_timeout(self):
-        from pymongo.errors import ExecutionTimeout
-
-        translated = translate_mongo_error(
-            ExecutionTimeout("too slow"), index_name="idx", database="db", collection="coll"
-        )
-        assert isinstance(translated, Timeout)
-        assert translated.status_code == 408
-
-    def test_unrecognised_errors_are_not_relabelled_as_bad_requests(self):
-        original = RuntimeError("something else entirely")
-        assert (
-            translate_mongo_error(original, index_name="idx", database="db", collection="coll")
-            is original
-        )
-
-
-def test_atlas_rejected_credentials_are_named_even_though_the_code_is_8000():
-    """Atlas answers a wrong password with code 8000 "AtlasError", not the 18 that a
-    self-hosted deployment returns, so a code-only check reports it as a generic
-    rejected search and never tells the caller to look at their connection string."""
-    from pymongo.errors import OperationFailure
-
-    error = OperationFailure(
-        "bad auth : authentication failed",
-        code=8000,
-        details={"ok": 0, "errmsg": "bad auth : authentication failed", "code": 8000, "codeName": "AtlasError"},
-    )
-    translated = translate_mongo_error(error, index_name="idx", database="sample_mflix", collection="embedded_movies")
-
-    assert isinstance(translated, BadRequestError)
-    assert "mongodb_connection_string" in str(translated)
-    assert "sample_mflix.embedded_movies" in str(translated)
-
-
-def test_a_rejected_search_that_is_not_an_auth_failure_keeps_the_generic_message():
-    from pymongo.errors import OperationFailure
-
-    error = OperationFailure("PlanExecutor error", code=8, details={"errmsg": "PlanExecutor error"})
-    translated = translate_mongo_error(error, index_name="idx", database="db", collection="coll")
-
-    assert "mongodb_connection_string" not in str(translated)
-
-
-class TestUnrecognisedParameters:
-    """litellm_params carries plenty of keys this provider does not own, so the params model has
-    to ignore extras. That turns a mistyped mongodb_collection into 'mongodb_collection is
-    required', pointing the reader at a key they can see they have set."""
-
-    def test_a_mistyped_parameter_is_named(self):
-        config, _, _ = _config()
-
-        with pytest.raises(BadRequestError, match="mongodb_collectoin"):
-            _search(config, litellm_params={"mongodb_collectoin": "embedded_movies"})
-
-    def test_the_supported_names_are_listed(self):
-        config, _, _ = _config()
-
-        with pytest.raises(BadRequestError, match="mongodb_connection_string"):
-            _search(config, litellm_params={"mongodb_databse": "sample_mflix"})
-
-    def test_unrelated_litellm_params_are_still_ignored(self):
-        config, _, _ = _config(documents=[{"_id": 1, "text": "hit", "score": 0.9}])
-
-        response = _search(
-            config,
-            litellm_params={"use_litellm_proxy": False, "use_in_pass_through": False, "vector_store_id": "x"},
-        )
-
-        assert len(response["data"]) == 1
-
-    @pytest.mark.asyncio
-    async def test_the_async_path_rejects_them_too(self):
-        config, _, _ = _async_config()
-
-        with pytest.raises(BadRequestError, match="mongodb_collectoin"):
-            await _asearch(config, litellm_params={"mongodb_collectoin": "embedded_movies"})
-
-
-class TestClientConstructionFailures:
-    """Building the client parses the URI and, for mongodb+srv://, performs a DNS SRV lookup, so it
-    fails on exactly the inputs a user is most likely to get wrong. Constructing it outside the
-    translation boundary let those escape as raw pymongo errors, which litellm.exception_type then
-    wrapped into a 500 with a traceback in the body."""
-
-    def _config_that_fails_to_connect(self, error):
-        def factory(_key):
-            raise error
-
-        return MongoDBVectorStoreConfig(
-            embedding_executor=FakeEmbeddingExecutor([0.1, 0.2, 0.3]), sync_client_factory=factory
-        )
-
-    def _async_config_that_fails_to_connect(self, error):
-        def factory(_key):
-            raise error
-
-        return MongoDBVectorStoreConfig(
-            embedding_executor=FakeEmbeddingExecutor([0.1, 0.2, 0.3]), async_client_factory=factory
-        )
-
-    def test_a_malformed_uri_is_a_bad_request_not_a_500(self):
-        from pymongo.errors import InvalidURI
-
-        config = self._config_that_fails_to_connect(InvalidURI("Invalid URI scheme"))
-
-        with pytest.raises(BadRequestError, match="not a usable MongoDB connection string"):
-            _search(config)
-
-    def test_an_unresolvable_cluster_name_says_so(self):
-        from pymongo.errors import ConfigurationError
-
-        config = self._config_that_fails_to_connect(ConfigurationError("The DNS query name does not exist"))
-
-        with pytest.raises(BadRequestError, match="does not exist in DNS"):
-            _search(config)
-
-    def test_a_dns_lookup_that_ran_out_of_time_is_a_timeout(self):
-        from pymongo.errors import ConfigurationError
-
-        config = self._config_that_fails_to_connect(
-            ConfigurationError("The resolution lifetime expired after 0.291 seconds")
-        )
-
-        with pytest.raises(Timeout, match="did not finish in time"):
-            _search(config)
-
-    @pytest.mark.asyncio
-    async def test_the_async_path_translates_them_too(self):
-        from pymongo.errors import InvalidURI
-
-        config = self._async_config_that_fails_to_connect(InvalidURI("Invalid URI scheme"))
-
-        with pytest.raises(BadRequestError, match="not a usable MongoDB connection string"):
-            await _asearch(config)
-
-
-class TestSelfManagedDeploymentsAreFirstClass:
-    """mongod serves $vectorSearch identically whether mongot runs under Atlas or beside a
-    self-managed deployment, so an operator without an Atlas account has to be able to act on
-    every message. Guidance that only names Atlas remedies sends them looking for an IP access
-    list and a paused cluster that do not exist in their deployment."""
-
-    def _config_that_fails_to_connect(self, error):
-        def factory(_key):
-            raise error
-
-        return MongoDBVectorStoreConfig(
-            embedding_executor=FakeEmbeddingExecutor([0.1, 0.2, 0.3]), sync_client_factory=factory
-        )
-
-    def test_a_plain_mongodb_uri_without_srv_or_credentials_is_accepted(self):
-        params = _MongoDBSearchParams.model_validate(
-            {**BASE_PARAMS, "mongodb_connection_string": "mongodb://mongod.internal:27017"}
-        )
-
-        assert params.require_connection_string() == "mongodb://mongod.internal:27017"
-
-    def test_an_unreachable_deployment_names_a_self_managed_remedy(self):
-        from pymongo.errors import ServerSelectionTimeoutError
-
-        config = self._config_that_fails_to_connect(ServerSelectionTimeoutError("connection refused"))
-
-        with pytest.raises(Timeout) as excinfo:
-            _search(config)
-
-        assert "self-managed" in str(excinfo.value)
-        assert "host or port" in str(excinfo.value)
-
-    def test_a_refused_connection_names_a_self_managed_remedy(self):
-        from pymongo.errors import ConnectionFailure
-
-        config = self._config_that_fails_to_connect(ConnectionFailure("connection closed"))
-
-        with pytest.raises(ServiceUnavailableError) as excinfo:
-            _search(config)
-
-        assert "self-managed" in str(excinfo.value)
-        assert "mongod is listening" in str(excinfo.value)
-
-    def test_an_unresolvable_hostname_names_a_self_managed_remedy(self):
-        from pymongo.errors import ConfigurationError
-
-        config = self._config_that_fails_to_connect(ConfigurationError("The DNS query name does not exist"))
-
-        with pytest.raises(BadRequestError) as excinfo:
-            _search(config)
-
-        assert "self-managed" in str(excinfo.value)
-
-    def test_the_missing_index_message_does_not_claim_atlas(self):
-        message = str(missing_index_error(INDEX, "sample_mflix", "embedded_movies"))
-
-        assert "MongoDB Vector Search index" in message
-        assert "Atlas" not in message
-
-    def test_the_not_ready_message_does_not_claim_atlas(self):
-        message = str(index_not_ready_error(INDEX, "sample_mflix", "embedded_movies", "PENDING"))
-
-        assert "MongoDB Vector Search index" in message
-        assert "Atlas" not in message
-
-    def test_the_search_only_refusal_does_not_claim_atlas(self):
-        config = MongoDBVectorStoreConfig()
-
-        with pytest.raises(BadRequestError) as excinfo:
-            config.transform_create_vector_store_request({}, api_base="")
-
-        assert "Atlas" not in str(excinfo.value)
-
-    def test_a_dimension_mismatch_does_not_claim_atlas(self):
-        from pymongo.errors import OperationFailure
-
-        error = OperationFailure("vector field is indexed with 128 dimensions but queried with 256")
-        translated = translate_mongo_error(error, index_name=INDEX, database="db", collection="c")
-
-        assert "Atlas" not in str(translated)
-        assert "dimensions the index was built for" in str(translated)
-
-    def test_an_uncovered_embedding_field_does_not_claim_atlas(self):
-        from pymongo.errors import OperationFailure
-
-        error = OperationFailure("embedding is not indexed as vector")
-        translated = translate_mongo_error(error, index_name=INDEX, database="db", collection="c")
-
-        assert "MongoDB Vector Search index does not cover" in str(translated)
-        assert "Atlas" not in str(translated)
-
-    def test_a_self_managed_auth_failure_is_still_recognised_by_code_18(self):
-        from pymongo.errors import OperationFailure
-
-        error = OperationFailure("Authentication failed.", code=18, details={"code": 18})
-        translated = translate_mongo_error(error, index_name=INDEX, database="db", collection="c")
-
-        assert isinstance(translated, BadRequestError)
-        assert "rejected the credentials" in str(translated)
-
-
-class TestUnescapedCredentialsAreDiagnosed:
-    """Self-managed deployments usually carry a generated password, so '@', '/', ':' and '%' in one
-    are routine. pymongo reports those as a port, a database name or an RFC 3986 complaint, none of
-    which points the operator at their password, so each has to be named for what it is. The errors
-    here come from pymongo's real parser rather than a synthetic stand-in."""
-
-    @staticmethod
-    def _real_parse_error(uri):
-        from pymongo import MongoClient
-
-        try:
-            MongoClient(uri, serverSelectionTimeoutMS=1)
-        except Exception as e:
-            return e
-        raise AssertionError(f"expected {uri!r} to fail parsing")
-
-    def _translated(self, uri):
-        return translate_mongo_error(
-            self._real_parse_error(uri), index_name=INDEX, database="db", collection="c"
-        )
-
-    @pytest.mark.parametrize(
-        "uri",
-        [
-            "mongodb://user:pa@ss@host:27017/",
-            "mongodb://user:pa:ss@host:27017/",
-            "mongodb://user:pa%ss@host:27017/",
-            "mongodb://user@x:pw@host:27017/",
-        ],
-    )
-    def test_rfc_3986_complaints_tell_the_operator_to_encode_the_password(self, uri):
-        translated = self._translated(uri)
-
-        assert isinstance(translated, BadRequestError)
-        assert "percent-encoded per RFC 3986" in str(translated)
-
-    @pytest.mark.parametrize(
-        "uri",
-        ["mongodb://user:pa/ss@host:27017/", "mongodb://user/x:pw@host:27017/"],
-    )
-    def test_a_slash_in_the_credentials_is_not_reported_as_a_database_name(self, uri):
-        translated = self._translated(uri)
-
-        assert isinstance(translated, BadRequestError)
-        assert "percent-encoded per RFC 3986" in str(translated)
-
-    def test_an_unusable_port_names_the_host_and_port_not_the_database(self):
-        translated = self._translated("mongodb://host:99999/")
-
-        assert isinstance(translated, BadRequestError)
-        assert "host and port" in str(translated)
-
-    def test_a_genuinely_bad_database_name_still_mentions_the_uri_path(self):
-        translated = self._translated("mongodb://host:27017/has space")
-
-        assert isinstance(translated, BadRequestError)
-        assert "database name in the URI path" in str(translated)
-
-
-class TestUnreadableTlsFilesAreDiagnosed:
-    """A private CA is how self-managed deployments present TLS, so tlsCAFile and
-    tlsCertificateKeyFile are on-prem options in practice. pymongo opens those files itself and
-    lets OSError out, which is not a PyMongoError, so before this they reached the caller as a 500
-    with a traceback. The errors here come from pymongo's real TLS setup."""
-
-    @staticmethod
-    def _real_tls_error(uri):
-        from pymongo import MongoClient
-
-        try:
-            MongoClient(uri, serverSelectionTimeoutMS=1500).admin.command("ping")
-        except Exception as e:
-            return e
-        raise AssertionError(f"expected {uri!r} to fail")
-
-    def _translated(self, uri):
-        return translate_mongo_error(self._real_tls_error(uri), index_name=INDEX, database="db", collection="c")
-
-    @pytest.mark.parametrize(
-        "path",
-        ["/nonexistent-directory-for-tests/ca.pem", "/tmp"],
-    )
-    def test_an_unreadable_ca_file_is_a_400_naming_the_path(self, path):
-        translated = self._translated(f"mongodb://localhost:27717/?tls=true&tlsCAFile={path}")
-
-        assert isinstance(translated, BadRequestError)
-        assert path in str(translated)
-        assert "tlsCAFile" in str(translated)
-
-    def test_an_unreadable_client_certificate_is_a_400_naming_the_path(self):
-        path = "/nonexistent-directory-for-tests/client.pem"
-        translated = self._translated(f"mongodb://localhost:27717/?tls=true&tlsCertificateKeyFile={path}")
-
-        assert isinstance(translated, BadRequestError)
-        assert path in str(translated)
-
-    def test_an_oserror_carrying_no_filename_is_left_for_the_other_branches(self):
-        translated = translate_mongo_error(OSError("socket hung up"), index_name=INDEX, database="db", collection="c")
-
-        assert not isinstance(translated, BadRequestError)
-
-
-class TestTheCallerSuppliedEmbeddingExecutorIsUsed:
-    """litellm.vector_stores.search always hands a direct provider an embedding_executor, so the
-    provider has to accept it and route the query through it rather than its own default."""
-
-    def test_the_supplied_executor_produces_the_query_vector(self):
-        config, _, collection = _config(embedding=(0.9, 0.9, 0.9), search_indexes=READY_INDEX)
-        caller = FakeEmbeddingExecutor([0.4, 0.5, 0.6])
-
-        config.execute_search_vector_store_request(
-            vector_store_id=INDEX,
-            query="a lone astronaut",
-            vector_store_search_optional_params={},
-            litellm_logging_obj=MagicMock(),
-            litellm_params=BASE_PARAMS,
-            embedding_executor=caller,
-        )
-
-        assert caller.captured.query == "a lone astronaut"
-        assert _stage(collection, "$vectorSearch")["queryVector"] == (0.4, 0.5, 0.6)
-
-    @pytest.mark.asyncio
-    async def test_the_supplied_executor_produces_the_query_vector_on_the_async_path(self):
-        config, _, collection = _async_config(embedding=(0.9, 0.9, 0.9), search_indexes=READY_INDEX)
-        caller = FakeEmbeddingExecutor([0.4, 0.5, 0.6])
-
-        await config.aexecute_search_vector_store_request(
-            vector_store_id=INDEX,
-            query="a lone astronaut",
-            vector_store_search_optional_params={},
-            litellm_logging_obj=MagicMock(),
-            litellm_params=BASE_PARAMS,
-            embedding_executor=caller,
-        )
-
-        assert caller.captured.query == "a lone astronaut"
-        assert _stage(collection, "$vectorSearch")["queryVector"] == (0.4, 0.5, 0.6)
+    assert "obsolete-secret" not in str(error.value)
+    executor.call.assert_not_called()
+
+
+@pytest.mark.parametrize(
+    "status,body,error_type",
+    [
+        (400, {"error": {"message": "Index is not queryable"}}, litellm.BadRequestError),
+        (401, {}, litellm.AuthenticationError),
+        (408, {}, litellm.Timeout),
+        (503, {}, litellm.ServiceUnavailableError),
+        (200, {}, litellm.ServiceUnavailableError),
+        (200, {**RESULT, "data": [{"score": "wrong"}]}, litellm.ServiceUnavailableError),
+        (0, {}, litellm.Timeout),
+        (-1, {}, litellm.BadRequestError),
+        (200, RESULT, None),
+    ],
+)
+def test_public_sdk_preserves_http_errors_response_and_timeout(
+    status: int, body: Mapping[str, object], error_type: type[Exception] | None
+) -> None:
+    executor: Final = RecordingEmbeddingExecutor()
+    if status == -1:
+        with pytest.raises(litellm.BadRequestError, match="search-only"):
+            litellm.vector_stores.create(custom_llm_provider="mongodb")
+        executor.call.assert_not_called()
+        return
+
+    def respond(request: httpx.Request) -> httpx.Response:
+        assert request.url == "https://sidecar.example/prefix/v1/vector_stores/policy_index/search"
+        assert request.headers["authorization"] == "Bearer test-sidecar-key"
+        assert request.extensions["timeout"]["read"] == 0.75
+        payload: Final = json.loads(request.content)
+        assert payload["timeout_ms"] == 750
+        assert payload["query_vector"] == [0.1, 0.2, 0.3]
+        if status == 0:
+            raise httpx.ReadTimeout("timed out", request=request)
+        return httpx.Response(status, json=body)
+
+    with httpx.Client(transport=httpx.MockTransport(respond)) as transport:
+        client: Final = HTTPHandler(client=transport)
+        if error_type is not None:
+            with pytest.raises(error_type):
+                litellm.vector_stores.search(
+                    vector_store_id="policy_index",
+                    query="travel policy",
+                    custom_llm_provider="mongodb",
+                    _direct_vector_store_embedding_executor=executor,
+                    client=client,
+                    timeout=0.75,
+                    **BASE_PARAMS,
+                )
+        else:
+            result: Final = litellm.vector_stores.search(
+                vector_store_id="policy_index",
+                query="travel policy",
+                custom_llm_provider="mongodb",
+                _direct_vector_store_embedding_executor=executor,
+                client=client,
+                timeout=0.75,
+                **BASE_PARAMS,
+            )
+            assert result == RESULT
+    executor.call.assert_called_once_with("embedding-alias", "travel policy", {})
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.test.tsx
index 84a9314ecce..8da7098b695 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.test.tsx
@@ -69,10 +69,11 @@ describe("VectorStoreForm", () => {
   });
 });
 
-const MONGODB_URI = "mongodb+srv://user:pass@cluster0.mongodb.net";
+const MONGODB_SIDECAR_URL = "http://mongodb-sidecar:8080";
 
 const MONGODB_REQUIRED_FORM_VALUES = {
-  mongodb_connection_string: MONGODB_URI,
+  api_base: MONGODB_SIDECAR_URL,
+  api_key: "sidecar-test-key",
   mongodb_database: "sample_mflix",
   mongodb_collection: "embedded_movies",
   embedding_model: "text-embedding-ada-002",
@@ -127,7 +128,8 @@ describe("buildVectorStoreLitellmParams", () => {
       mongodb_num_candidates: "200",
     };
     const expected = {
-      mongodb_connection_string: MONGODB_URI,
+      api_base: MONGODB_SIDECAR_URL,
+      api_key: "sidecar-test-key",
       mongodb_database: "sample_mflix",
       mongodb_collection: "embedded_movies",
       mongodb_embedding_field: "plot_embedding",
@@ -142,6 +144,7 @@ describe("buildVectorStoreLitellmParams", () => {
   it("sends only mongodb fields when an earlier provider left values in the form", () => {
     const formValues = {
       ...MONGODB_REQUIRED_FORM_VALUES,
+      mongodb_connection_string: "mongodb://obsolete-credentials",
       valkey_host: "left-over-from-valkey.example.com",
       valkey_port: "6379",
       aws_region_name: "us-west-2",
@@ -152,7 +155,8 @@ describe("buildVectorStoreLitellmParams", () => {
     expect(params).not.toHaveProperty("valkey_host");
     expect(params).not.toHaveProperty("valkey_port");
     expect(params).not.toHaveProperty("aws_region_name");
-    expect(params.mongodb_connection_string).toBe(MONGODB_URI);
+    expect(params.api_base).toBe(MONGODB_SIDECAR_URL);
+    expect(params).not.toHaveProperty("mongodb_connection_string");
   });
 
   it("omits a blank mongodb_num_candidates so litellm picks its own candidate count", () => {
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx
index 61da25874a5..67ef1b795ba 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.tsx
@@ -70,7 +70,6 @@ const PROVIDER_FIELD_NAMES = [
   "vector_bucket_name",
   "index_name",
   "aws_region_name",
-  "mongodb_connection_string",
   "mongodb_database",
   "mongodb_collection",
   "mongodb_embedding_field",
@@ -107,7 +106,6 @@ const vectorStoreShape = {
   vector_bucket_name: optionalText,
   index_name: optionalText,
   aws_region_name: optionalText,
-  mongodb_connection_string: optionalText,
   mongodb_database: optionalText,
   mongodb_collection: optionalText,
   mongodb_embedding_field: optionalText,
@@ -142,7 +140,7 @@ const VECTOR_STORE_ID_PLACEHOLDERS: Record<string, string> = {
   vertex_rag_engine: '6917529027641081856 (corpus ID from Vertex AI / "RAG Engine" console)',
   "vertex_ai/search_api": 'my-datastore_1234567890 (data store ID from Vertex AI / "Agent Search" console)',
   valkey: "my-search-index (FT index name in Valkey)",
-  mongodb: "my-vector-index (Atlas Vector Search index name)",
+  mongodb: "my-vector-index (MongoDB Vector Search index name)",
 };
 
 const VERTEX_SEARCH_API_WITH_ENGINE_PLACEHOLDER = "Any identifier you'll use to reference this in LiteLLM";
diff --git a/ui/litellm-dashboard/src/components/vector_store_providers.test.tsx b/ui/litellm-dashboard/src/components/vector_store_providers.test.tsx
index 8e3a3aa3402..32d0f5dccc0 100644
--- a/ui/litellm-dashboard/src/components/vector_store_providers.test.tsx
+++ b/ui/litellm-dashboard/src/components/vector_store_providers.test.tsx
@@ -35,7 +35,8 @@ describe("getVectorStoreProviderLogoAndName", () => {
     });
     expect(vectorStoreProviderMap.MongoDB).toBe("mongodb");
     expect(getProviderSpecificFields("mongodb").map((field) => field.name)).toEqual([
-      "mongodb_connection_string",
+      "api_base",
+      "api_key",
       "mongodb_database",
       "mongodb_collection",
       "embedding_model",
@@ -45,12 +46,10 @@ describe("getVectorStoreProviderLogoAndName", () => {
     ]);
   });
 
-  it("hides the mongodb connection string, which carries the database password", () => {
-    const connectionString = getProviderSpecificFields("mongodb").find(
-      (field) => field.name === "mongodb_connection_string",
-    );
+  it("hides the mongodb sidecar API key", () => {
+    const apiKey = getProviderSpecificFields("mongodb").find((field) => field.name === "api_key");
 
-    expect(connectionString).toMatchObject({ type: "password", required: true });
+    expect(apiKey).toMatchObject({ type: "password", required: true });
   });
 
   it("picks the mongodb embedding model from the proxy's models rather than a fixed list", () => {
diff --git a/ui/litellm-dashboard/src/components/vector_store_providers.tsx b/ui/litellm-dashboard/src/components/vector_store_providers.tsx
index a75f10771a8..6a8b2f405d2 100644
--- a/ui/litellm-dashboard/src/components/vector_store_providers.tsx
+++ b/ui/litellm-dashboard/src/components/vector_store_providers.tsx
@@ -14,7 +14,7 @@ export enum VectorStoreProviders {
   OpenAI = "OpenAI",
   Azure = "Azure OpenAI",
   Milvus = "Milvus",
-  MongoDB = "MongoDB Atlas",
+  MongoDB = "MongoDB (BETA)",
   Valkey = "Valkey",
 }
 
@@ -175,18 +175,25 @@ export const vectorStoreProviderFields: Record<string, VectorStoreFieldConfig[]>
   ],
   mongodb: [
     {
-      name: "mongodb_connection_string",
-      label: "Connection String",
-      tooltip:
-        "The full MongoDB connection string for your Atlas cluster, including the database user and password. Copy it from Atlas under Connect, Drivers (e.g. mongodb+srv://user:password@cluster.mongodb.net)",
-      placeholder: "mongodb+srv://user:password@cluster.mongodb.net",
+      name: "api_base",
+      label: "Sidecar URL",
+      tooltip: "The URL of your separately deployed MongoDB sidecar. Configure MongoDB credentials in the sidecar",
+      placeholder: "http://mongodb-sidecar:8080",
+      required: true,
+      type: "text",
+    },
+    {
+      name: "api_key",
+      label: "Sidecar API Key",
+      tooltip: "The MONGODB_SIDECAR_API_KEY configured in your MongoDB sidecar",
+      placeholder: "Enter sidecar API key",
       required: true,
       type: "password",
     },
     {
       name: "mongodb_database",
       label: "Database",
-      tooltip: "The Atlas database holding the collection you want to search",
+      tooltip: "The MongoDB database holding the collection you want to search",
       placeholder: "sample_mflix",
       required: true,
       type: "text",
@@ -194,7 +201,7 @@ export const vectorStoreProviderFields: Record<string, VectorStoreFieldConfig[]>
     {
       name: "mongodb_collection",
       label: "Collection",
-      tooltip: "The collection your Atlas Vector Search index was built on",
+      tooltip: "The collection your MongoDB Vector Search index was built on",
       placeholder: "embedded_movies",
       required: true,
       type: "text",
@@ -212,7 +219,7 @@ export const vectorStoreProviderFields: Record<string, VectorStoreFieldConfig[]>
       name: "mongodb_embedding_field",
       label: "Vector Field Name",
       tooltip:
-        "The field in each document that holds its embedding. It must match the path your Atlas Vector Search index was created on (default: embedding)",
+        "The field in each document that holds its embedding. It must match the path your MongoDB Vector Search index was created on (default: embedding)",
       placeholder: "embedding",
       required: false,
       type: "text",
@@ -232,7 +239,7 @@ export const vectorStoreProviderFields: Record<string, VectorStoreFieldConfig[]>
       name: "mongodb_num_candidates",
       label: "Candidates Considered",
       tooltip:
-        "How many nearest neighbours Atlas examines before returning the top results. Higher is more accurate and slower. Leave blank to let LiteLLM scale it with the requested result count",
+        "How many nearest neighbours MongoDB examines before returning the top results. Higher is more accurate and slower. Leave blank to let LiteLLM scale it with the requested result count",
       placeholder: "100",
       required: false,
       type: "text",
diff --git a/uv.lock b/uv.lock
index 89205cd9527..ce5d96f4d9c 100644
--- a/uv.lock
+++ b/uv.lock
@@ -10,7 +10,7 @@ resolution-markers = [
 ]
 
 [options]
-exclude-newer = "2026-09-02T16:58:34.594994Z"
+exclude-newer = "2026-09-05T05:15:35.833796Z"
 exclude-newer-span = "P3D"
 
 [manifest]
@@ -4415,9 +4415,6 @@ mcp = [
 mlflow = [
     { name = "mlflow" },
 ]
-mongodb = [
-    { name = "pymongo" },
-]
 proxy = [
     { name = "apscheduler" },
     { name = "azure-identity" },
@@ -4649,7 +4646,6 @@ requires-dist = [
     { name = "pydantic", specifier = ">=2.10.0,<3.0.0" },
     { name = "pydantic-settings", specifier = ">=2.14.1,<3.0" },
     { name = "pyjwt", marker = "extra == 'proxy'", specifier = ">=2.13.0,<3.0" },
-    { name = "pymongo", marker = "extra == 'mongodb'", specifier = ">=4.9,<5.0" },
     { name = "pynacl", marker = "extra == 'proxy'", specifier = ">=1.6.2,<2.0" },
     { name = "pypdf", marker = "extra == 'proxy-runtime'", specifier = ">=6.16.1,<7.0" },
     { name = "pyroscope-io", marker = "sys_platform != 'win32' and extra == 'proxy'", specifier = ">=0.8.16,<1.0" },
@@ -4676,7 +4672,7 @@ requires-dist = [
     { name = "uvloop", marker = "sys_platform != 'win32' and extra == 'proxy'", specifier = ">=0.22.1,<1.0" },
     { name = "websockets", marker = "extra == 'proxy'", specifier = ">=15.0.1,<16.0" },
 ]
-provides-extras = ["proxy", "cli", "extra-proxy", "utils", "caching", "mcp", "mongodb", "saml", "semantic-router", "mlflow", "grpc", "stt-nvidia-riva", "google", "bedrock-realtime", "proxy-runtime"]
+provides-extras = ["proxy", "cli", "extra-proxy", "utils", "caching", "mcp", "saml", "semantic-router", "mlflow", "grpc", "stt-nvidia-riva", "google", "bedrock-realtime", "proxy-runtime"]
 
 [package.metadata.requires-dev]
 ci = [
@@ -7620,77 +7616,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/d5/6f/9ac2548e290764781f9e7e2aaf0685b086379dabfb29ca38536985471eaf/pylint-4.0.5-py3-none-any.whl", hash = "sha256:00f51c9b14a3b3ae08cff6b2cdd43f28165c78b165b628692e428fb1f8dc2cf2", size = 536694, upload-time = "2026-02-20T09:07:31.028Z" },
 ]
 
-[[package]]
-name = "pymongo"
-version = "4.17.0"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "dnspython" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/ca/64/50be6fbac9c79fe2e4c17401a467da2d8764d82833d83cec325afe5cab32/pymongo-4.17.0.tar.gz", hash = "sha256:70ffa08ba641468cc068cf46c06b34f01a8ce3489f6411309fcb5ceabe6b2fc0", size = 2523370, upload-time = "2026-04-20T16:39:53.524Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/c9/77/28ebbf69772a4341d530831c7a006cdb06877ac23075cb53b0a227df4fe1/pymongo-4.17.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:47b021363cd923ace5edc7a1d63c0ff8a6d9d43859b8a1ba23645f5afae63221", size = 819234, upload-time = "2026-04-20T16:37:20.888Z" },
-    { url = "https://files.pythonhosted.org/packages/88/cf/5a70cee503ff9a2fea20607607f14d189f4d975960ac0945ec306ee7b695/pymongo-4.17.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:422fa50d7d7f5c22ea0953554396c9ef95684a2d775f860bd75a7b510538dfca", size = 819969, upload-time = "2026-04-20T16:37:24.187Z" },
-    { url = "https://files.pythonhosted.org/packages/23/d5/07b7e27e662c58d872efd104a0e8055eb6569aa1b6d4da436f3fdee7f897/pymongo-4.17.0-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:addd0498ebbdc6354227f6ed457ed9fce442d48a3bb30d5b5bad33e104996561", size = 1244510, upload-time = "2026-04-20T16:37:26.069Z" },
-    { url = "https://files.pythonhosted.org/packages/fb/be/7cac5b1e89bd5a8e395067648241390321593a7c29243e36f91343c02a90/pymongo-4.17.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5c8e180cb2cabe37300e1e36c60aa4f2ff956cc579f0142135a5d2cba252243", size = 1263245, upload-time = "2026-04-20T16:37:28.003Z" },
-    { url = "https://files.pythonhosted.org/packages/2e/20/40e8e99824c1fda18261411e65ce3b0cd3d9a6ed3c056cdd0a569adc870b/pymongo-4.17.0-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bd835cdb37a1adec359dd072c24f8bb14809e2644fde86fab4ee2fc9719b9483", size = 1304113, upload-time = "2026-04-20T16:37:30.048Z" },
-    { url = "https://files.pythonhosted.org/packages/3a/94/fb7e25441dd66f2069a9b172380849b0eaa5881c18b3db217bf64a6d393c/pymongo-4.17.0-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c4979e7e8887862bbb44d203f00cc8263a3f27237876fa691b6beba23e40e6d8", size = 1297046, upload-time = "2026-04-20T16:37:32.054Z" },
-    { url = "https://files.pythonhosted.org/packages/4f/c9/7352e0c20fe772541556e4d283c05e07ec48f8b0d2737ad930ac4a1b6655/pymongo-4.17.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:77aa4bc164b4de60d5db193b322f0f5b6ead716e831031bfdef8e8bd92205556", size = 1265708, upload-time = "2026-04-20T16:37:33.934Z" },
-    { url = "https://files.pythonhosted.org/packages/8d/e4/3df15494c2015ed297958517f0e4f6493e21b00990748068a973e66d45e0/pymongo-4.17.0-cp310-cp310-win32.whl", hash = "sha256:48bbc576677b50af043df870d84ded67cc3a9b4aa7553201beef4da5dc050a0a", size = 805533, upload-time = "2026-04-20T16:37:35.744Z" },
-    { url = "https://files.pythonhosted.org/packages/22/fa/b4e71bb8cb82ad7d21bb4e8c476f2d573ba68b20368aac36ef06e4a196b4/pymongo-4.17.0-cp310-cp310-win_amd64.whl", hash = "sha256:e46767f28dea610e02edf6c5d956ce615c3c7790ea396660b9b1efd5c5ead2e0", size = 815677, upload-time = "2026-04-20T16:37:37.808Z" },
-    { url = "https://files.pythonhosted.org/packages/22/e2/0a4bba644f1cda3970ea1012149eeae3594ebfeed3f81fdaf32b61d90c95/pymongo-4.17.0-cp310-cp310-win_arm64.whl", hash = "sha256:757f2a4c0c2c46cab87df0333681ce69e86c9d5b45bc5203ceba5410b3489e59", size = 807293, upload-time = "2026-04-20T16:37:39.707Z" },
-    { url = "https://files.pythonhosted.org/packages/c4/e2/336d86f221cf1b56b2ed9330d4a3b98f9f38f0b37829ae9a9184617d5419/pymongo-4.17.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4141e6c6a339789b2974efa00ecd9409101672d77a0e3ee2cc3839eedf8ec4df", size = 874668, upload-time = "2026-04-20T16:37:41.39Z" },
-    { url = "https://files.pythonhosted.org/packages/34/8e/75d3c6c935d187ab59c61e9c15d9aab3f274b563eaf1706e8cae5f508dec/pymongo-4.17.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e68c76b84e0c132d9dbf9307f12ff8185702328187a87b9aca8c941303873433", size = 875294, upload-time = "2026-04-20T16:37:43.432Z" },
-    { url = "https://files.pythonhosted.org/packages/5f/ec/62e855744489dbcd54fd778aae4d80fa4c4819e8fb228ca0cf6f21a03997/pymongo-4.17.0-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ba2195d4f386f839a52a23ea1cfd60ffaaba78a3d7841db51b7e433001139918", size = 1496233, upload-time = "2026-04-20T16:37:45.518Z" },
-    { url = "https://files.pythonhosted.org/packages/82/e8/93e4e5e5ce8fdf8929dabeefe24aafa5ce046028eed0dfa8eeb936e72c49/pymongo-4.17.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8446ff4bfcb6ec2a2e50998c860986a1e992136f998b7f53e7a717fb8aa5a0b9", size = 1522927, upload-time = "2026-04-20T16:37:47.492Z" },
-    { url = "https://files.pythonhosted.org/packages/f7/ca/425dc1d21e0f17bdea0072fc463f662f7fa06d2852af52975c9eced3c07c/pymongo-4.17.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2a0d5ac205728c86e0a02192f1aa5f865b0d7d51f8df6101c01a69a7fc620d72", size = 1583468, upload-time = "2026-04-20T16:37:49.221Z" },
-    { url = "https://files.pythonhosted.org/packages/b3/9d/f08b07eeffda1a43c1759f0fa625e88ae12360996eb56d42aad832fa7dff/pymongo-4.17.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:485c8a8eaa4c739f00a331fc73757898ee7c092c214a79e63866ff76aaf282ff", size = 1572787, upload-time = "2026-04-20T16:37:51.061Z" },
-    { url = "https://files.pythonhosted.org/packages/e9/c2/6855a07aafa7b894929af23675b6fb9634800ce43122b76a62f6eeb8da2a/pymongo-4.17.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b2dfcc795f5b9fedbe179a11fdf6051581479d196582a3fe819a92a00e9b9969", size = 1526184, upload-time = "2026-04-20T16:37:53.358Z" },
-    { url = "https://files.pythonhosted.org/packages/4e/05/c952bac7db71c1942ea3559fcd308b49754cc5004b455935fb4000d1f37b/pymongo-4.17.0-cp311-cp311-win32.whl", hash = "sha256:c2292144505fb12156b981bd440f3dc994a883da06ac726c0c8692ccdbc1c510", size = 852621, upload-time = "2026-04-20T16:37:55.28Z" },
-    { url = "https://files.pythonhosted.org/packages/11/c0/c04da9f4c0c6252404598f4e394b862a58a9e866822a70ae261c8a018fdf/pymongo-4.17.0-cp311-cp311-win_amd64.whl", hash = "sha256:2e190827834fce70ecdf9d46796c6dbc0ce08ea87dc2ff5bc6f3f5579b605cb9", size = 867852, upload-time = "2026-04-20T16:37:57.233Z" },
-    { url = "https://files.pythonhosted.org/packages/1d/b2/c7b4870fbeef471e947d3e014676f5910d02e0197074d692ebcf24ec049a/pymongo-4.17.0-cp311-cp311-win_arm64.whl", hash = "sha256:a8f9c40a09bb7d4b9fc8b1da65ecf6efa79bda5cb2756f39d9b6940fac1d19ae", size = 855019, upload-time = "2026-04-20T16:37:58.983Z" },
-    { url = "https://files.pythonhosted.org/packages/98/90/60bcb508840135d5ee46b51b1a950f548338aa8145a8366dbe6639ae51ac/pymongo-4.17.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53ffa94b2340dbf6b055e09a0090618c60482c158ecfc9565642fc996bf0944", size = 930529, upload-time = "2026-04-20T16:38:00.936Z" },
-    { url = "https://files.pythonhosted.org/packages/a6/e9/313840f1e52c6dfac47f704428cbfbce59956ebe7633bffc92b03f74f0ad/pymongo-4.17.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6fe0de9d0f6791abce3471230b32b4817bf89d27b1182b6a550e1ec0fa72aa9a", size = 930665, upload-time = "2026-04-20T16:38:02.915Z" },
-    { url = "https://files.pythonhosted.org/packages/78/35/9d3565ea45b1606f635c1e2cd2563c28d66caafdc50f7ad7d979fcd1b363/pymongo-4.17.0-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:e537e95514dae1aaa718f481ec03151a0f0394bcd05f1322896d8fc1330cb729", size = 1762369, upload-time = "2026-04-20T16:38:05.375Z" },
-    { url = "https://files.pythonhosted.org/packages/95/ee/149b0d4b1a11c38bff6f14c23d5814c9b0843fd6dc38ad40596bdb1a62d2/pymongo-4.17.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:37a8385c29881b43eab31f584100fa0eaddedd5607adf010147ba1810118be90", size = 1798044, upload-time = "2026-04-20T16:38:07.195Z" },
-    { url = "https://files.pythonhosted.org/packages/7b/d4/4cee4a7b8d8f6f0550ef6cd2fea42455c5ed619a220cb6ba4fb40d6a5bc8/pymongo-4.17.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f3ee3d241ed77a4fc99ce3cff3b289c3ebce37f61fdd7349d3592c23b82c8784", size = 1878567, upload-time = "2026-04-20T16:38:09.121Z" },
-    { url = "https://files.pythonhosted.org/packages/45/ef/7fe366c84952619ee2f69973566c214775e083dd4df465751912153e4b72/pymongo-4.17.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:9eb5d63a3c518cb0804ed678f5e2b875af032d89a7cf57a57360322cf6a4d222", size = 1864881, upload-time = "2026-04-20T16:38:10.896Z" },
-    { url = "https://files.pythonhosted.org/packages/2f/35/b577d82c6d1be7aee7ac7e249bc86f7847998345042e5f8360de238e177b/pymongo-4.17.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e97e03fa13327c87e3fdc5656acd01e71817f0c1dc3221cd8f30de136bf4ec3", size = 1800349, upload-time = "2026-04-20T16:38:13.589Z" },
-    { url = "https://files.pythonhosted.org/packages/b8/69/dafcf04f66e130ddd91aeb92e7a692480eda46dcd04ec1dbe82c06619e10/pymongo-4.17.0-cp312-cp312-win32.whl", hash = "sha256:6877214bff5f06f6884a9fc8d9016a4a7a5f51f537f5c51ac3a576f93e7dfb32", size = 900518, upload-time = "2026-04-20T16:38:15.541Z" },
-    { url = "https://files.pythonhosted.org/packages/11/35/5c9262a459f988b4eb2605f70815240b77a0d4131136c4326d18f1822b89/pymongo-4.17.0-cp312-cp312-win_amd64.whl", hash = "sha256:9828485f72f63c7d802e0ec41f71906f633c2692621ab3af55ca990186b091b1", size = 920335, upload-time = "2026-04-20T16:38:17.665Z" },
-    { url = "https://files.pythonhosted.org/packages/8d/da/e9c7265ee176faccf4e52c4797837e794d93569a1046f6b19a4acc36e5ad/pymongo-4.17.0-cp312-cp312-win_arm64.whl", hash = "sha256:1195370a77baf003b59b10e91ecc4706297197f0dd9d29c840cc556dc08f7cee", size = 903289, upload-time = "2026-04-20T16:38:19.33Z" },
-    { url = "https://files.pythonhosted.org/packages/2a/6b/c1206879708b94e82fcd8b9653440ec271f79a3674d122192df383047f5a/pymongo-4.17.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:809ec74de3b9148ae43fa8df9faf53470f511c8d384f13b99d6f671f2a379f15", size = 985829, upload-time = "2026-04-20T16:38:21.031Z" },
-    { url = "https://files.pythonhosted.org/packages/cb/cf/bb044ed85160e5c40f568c7c4f4e8ea16f40764ff5d302e5befbe8f6f814/pymongo-4.17.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a431b737816bf4cddd4fa0fcef04e424ad36b7692734a64150f872fb8f3208be", size = 985899, upload-time = "2026-04-20T16:38:23.409Z" },
-    { url = "https://files.pythonhosted.org/packages/74/0a/f6dfd5ea3901e5d6888da8de8ba728971a1d447debab681cfc56f90d1208/pymongo-4.17.0-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:e4fab10f8403169ce92f3cea921609d9ee81107306caae06c08f592d4b8ad2b5", size = 2028569, upload-time = "2026-04-20T16:38:25.343Z" },
-    { url = "https://files.pythonhosted.org/packages/4a/c5/081f59a1c02ae8c0dc73ae58e563838c44eec81aeafa7d0b93a637841c9b/pymongo-4.17.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:20323b0b1c1d33770ad1fc68d429c757734ce9ad3594421c3d6618f10572b1b9", size = 2072916, upload-time = "2026-04-20T16:38:27.291Z" },
-    { url = "https://files.pythonhosted.org/packages/31/42/6e41d434297ffe8b30d9c3717916591a4a7be9075a0dcc2fafdfaaaa62ed/pymongo-4.17.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5a5de048e6da5c18e27cc2437e8c15b3b0cdc8385c15b41178b0caa3322a09c2", size = 2173234, upload-time = "2026-04-20T16:38:29.474Z" },
-    { url = "https://files.pythonhosted.org/packages/3d/cf/1e4a7db352ef9485831c7268dfe8402f0117b32a9ad54b16e810699e3617/pymongo-4.17.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:dff3de1294fbbc1db0ba6b511f77b8e540601d092538a31312e99c8a91a78b1e", size = 2156784, upload-time = "2026-04-20T16:38:32.134Z" },
-    { url = "https://files.pythonhosted.org/packages/12/10/6195be29962a61ebb5f4bd9e4c7519890b172f7968a0a0d880398c6ddb02/pymongo-4.17.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:faf03e4c2aafd6de626dbd30ba246d369ae33f47f10629d1bbe40f72115027a6", size = 2074446, upload-time = "2026-04-20T16:38:34.004Z" },
-    { url = "https://files.pythonhosted.org/packages/37/48/33410b8819837ed370c738587306bdf060b59cef11823be212f4a07703c5/pymongo-4.17.0-cp313-cp313-win32.whl", hash = "sha256:c9786665926a09630c5d420c79762cfadbff35a9438bcbc4c81a9fb5ab9228b7", size = 948435, upload-time = "2026-04-20T16:38:35.922Z" },
-    { url = "https://files.pythonhosted.org/packages/6f/77/c0ed522f798a286b99acaa7914ed8d9c80ab091f97f57c59ffed72906e5e/pymongo-4.17.0-cp313-cp313-win_amd64.whl", hash = "sha256:5960519b4d7168f1ecdd3ea10c81b2aedeb9423651aca953cfbc8e76705d3b38", size = 972847, upload-time = "2026-04-20T16:38:37.888Z" },
-    { url = "https://files.pythonhosted.org/packages/97/f0/c39480a2db385fde23861d0c8acda41cdaf1d43e46579db72c5c013a2e81/pymongo-4.17.0-cp313-cp313-win_arm64.whl", hash = "sha256:0ff6bd2f735ab5356541e3e57d5b7dbfbc3f2ee1ccb10b6b0f82d58af69d1d8e", size = 951575, upload-time = "2026-04-20T16:38:40.544Z" },
-    { url = "https://files.pythonhosted.org/packages/da/49/2b0250762a89737ed6f9cea238331baca061b89a8ddd10dd17fee52c3970/pymongo-4.17.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ff5aa3f1c7e3f08eb0e7a016c91ba468b1850ccfd63d9b1f12f56350f4974cef", size = 1040945, upload-time = "2026-04-20T16:38:42.783Z" },
-    { url = "https://files.pythonhosted.org/packages/89/1c/7a9b5447a08be20e84b6e5b17330917e8d6d9507daa3cd099a9309f11ad7/pymongo-4.17.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e816db649ba5d7de0568cf3a9f287a9dc9aad21cf0ca667ab156a7ef47fca0b0", size = 1041187, upload-time = "2026-04-20T16:38:45.358Z" },
-    { url = "https://files.pythonhosted.org/packages/78/a1/71704f61632dfc90407a5834fe5f6132854937c4a3648f6c05c351d85a45/pymongo-4.17.0-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:12c4fded3a9f1d6a687e36ebd384ac6d00b9b00de1969aa74048e7051ec2a713", size = 2294806, upload-time = "2026-04-20T16:38:47.734Z" },
-    { url = "https://files.pythonhosted.org/packages/ad/b9/aff42be75108b96c2469b1d9329b912c15108f3e7ef32fdc86da8423c330/pymongo-4.17.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2db66aa8dd253a0fc1fad3b0d23d5b3993f7ebde02fbbd7727128debf2853675", size = 2348231, upload-time = "2026-04-20T16:38:50.371Z" },
-    { url = "https://files.pythonhosted.org/packages/f2/30/44c115b8ba1479942c15fd9480eb29a7da0ba68acd56983423ba0deb4a94/pymongo-4.17.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3987e96e7c7be4083d42e8ac2cc6c0d5b78db9973c90fce42ae800b616ca6b20", size = 2467614, upload-time = "2026-04-20T16:38:52.665Z" },
-    { url = "https://files.pythonhosted.org/packages/d2/84/21ee95c8bf0ca7acae7ec7eb365d740bf8fc0156c194baf2c3bdfcb85ec0/pymongo-4.17.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cee36b3c0d0354f880fa7a7fdcdaf2bb5e542c2281e25c1bfadf8cfe21eba7d2", size = 2445970, upload-time = "2026-04-20T16:38:55.175Z" },
-    { url = "https://files.pythonhosted.org/packages/06/89/081d7f1809d5ca09d1e47e49f2111b245f5694de3a7af32cd3a353a6f43f/pymongo-4.17.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:320b34457b20bbcc79997801f95d25ce00472915ca5241167242b42c4359e027", size = 2348605, upload-time = "2026-04-20T16:38:57.557Z" },
-    { url = "https://files.pythonhosted.org/packages/ea/c3/0d949f9d3f2a341c1f635c398c16615e96f89f51ff424ed81e914cf1a4de/pymongo-4.17.0-cp314-cp314-win32.whl", hash = "sha256:df4a644af9ae132d4bfdb2e9516ea51a615fd881caddfbfbd071cf1354844479", size = 1004119, upload-time = "2026-04-20T16:39:00.309Z" },
-    { url = "https://files.pythonhosted.org/packages/f7/55/5c3a3db1048054c695c75c5964cc8bedc2247fdb5a75ef6fab4ec8bb013e/pymongo-4.17.0-cp314-cp314-win_amd64.whl", hash = "sha256:c797f8a80957134f6dd9690367a0f8f5906d672119af2c6aa55f0c527b656bed", size = 1032314, upload-time = "2026-04-20T16:39:02.665Z" },
-    { url = "https://files.pythonhosted.org/packages/e0/19/e235f39906134cb0ffd5574c5a59c355ef5380f0499644ab94994afbb109/pymongo-4.17.0-cp314-cp314-win_arm64.whl", hash = "sha256:68fca71e05ee5da23a8d73cee8379dfb3d26e609a377cae731d742771ed96946", size = 1007627, upload-time = "2026-04-20T16:39:04.678Z" },
-    { url = "https://files.pythonhosted.org/packages/1e/e0/c4c1a86791415b14c684fa0908f9da96de91594a3fd1fa1b8dc689fbb800/pymongo-4.17.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b4384700cffc3f1dd98e088bc0072dedf6d7d68a230bb4b972665cf69c071c1e", size = 1099151, upload-time = "2026-04-20T16:39:06.969Z" },
-    { url = "https://files.pythonhosted.org/packages/81/4b/69c67f3e23fd9b23b9bedc7ebd23754881cc9d5c5d5b2a9811e96b07f475/pymongo-4.17.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:93641192644fa1ee0f34030e774fd31022a27ad11ba22cb1716142231524f8bd", size = 1099346, upload-time = "2026-04-20T16:39:08.996Z" },
-    { url = "https://files.pythonhosted.org/packages/a2/19/a5208f62f9508a26d73acc69bd3821b8c8adae253679a3c26d2f9652f0d5/pymongo-4.17.0-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:75bc3aa5b94fdb7138d357ec6ca61cd97e0c79f4f7f0bd3efe9639b15cc50942", size = 2619034, upload-time = "2026-04-20T16:39:11.049Z" },
-    { url = "https://files.pythonhosted.org/packages/77/27/426cba1ec5973082a56d4150798529bfdf4151c31391ed1fbbecb23ef2ac/pymongo-4.17.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:50e8f8e23c6df7c6d6929f5e734980b227706e73ee847517c9ba5af90f7fc466", size = 2689939, upload-time = "2026-04-20T16:39:13.617Z" },
-    { url = "https://files.pythonhosted.org/packages/ef/2e/f70993d1255e33f6ee59a4ec4371cc65bff7a7e3fda7d55c3386f25287e8/pymongo-4.17.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:15d3f3d732aecac1f8d481bde4029755615639bd3076f258a2147210aec8515a", size = 2824994, upload-time = "2026-04-20T16:39:16.057Z" },
-    { url = "https://files.pythonhosted.org/packages/b3/eb/87b0e988ba889e1fcc3430c2cfc166b251872c813e92b43174298bee17ff/pymongo-4.17.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6c5f62862d0f87be481fa1fe8cb811994486773c94a2b61e509285e3f2890763", size = 2801745, upload-time = "2026-04-20T16:39:18.476Z" },
-    { url = "https://files.pythonhosted.org/packages/67/4c/3f83412d086f682d4d468761d66ddc49cf161e786ea74073045eb4491c60/pymongo-4.17.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64837adbbd72073301af51bb0fc80e3d7707fe5527cea1033ba0320f0b2f881b", size = 2684636, upload-time = "2026-04-20T16:39:20.878Z" },
-    { url = "https://files.pythonhosted.org/packages/9e/d8/b75f6f4ab6c8beb50b0270a4f1e2530b5774f5e116563440e1677ca1820f/pymongo-4.17.0-cp314-cp314t-win32.whl", hash = "sha256:b93b22eedc62598cf5ee9d8c8007a8e9121c50fd88137012d8985500e9dc3151", size = 1056356, upload-time = "2026-04-20T16:39:22.996Z" },
-    { url = "https://files.pythonhosted.org/packages/e4/5e/648c8a238eef18a25ed8a169ea6542d4a860bbec3e95b3d9badac2935c71/pymongo-4.17.0-cp314-cp314t-win_amd64.whl", hash = "sha256:3689ea34f6b647c7d1e7bdc60fcfb214b2789ed1359a7fb96569c69f50e5f18f", size = 1090964, upload-time = "2026-04-20T16:39:24.989Z" },
-    { url = "https://files.pythonhosted.org/packages/dc/cb/d9780b66939c4fc1f024bcc7be23a2abcfe06a9745ca8fa76dc73395482e/pymongo-4.17.0-cp314-cp314t-win_arm64.whl", hash = "sha256:9543d8f84c2e5608565c08ac679774811e6730770d8a645439b073422a4276fb", size = 1058526, upload-time = "2026-04-20T16:39:27.924Z" },
-]
-
 [[package]]
 name = "pynacl"
 version = "1.6.2"

From 0c519b162f2ca44effc7ff5bd14d80e9cb8935ae Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Mon, 7 Sep 2026 23:43:21 -0700
Subject: [PATCH 56/85] fix: preserve MongoDB deadlines and secure remote
 sidecar transport

---
 .../mongodb/vector_stores/transformation.py   |  19 ++-
 .../test_mongodb_transformation.py            | 114 ++++++++++++------
 .../_components/VectorStoreForm.test.tsx      |   2 +-
 .../src/components/vector_store_providers.tsx |   4 +-
 4 files changed, 97 insertions(+), 42 deletions(-)

diff --git a/litellm/llms/mongodb/vector_stores/transformation.py b/litellm/llms/mongodb/vector_stores/transformation.py
index 92965ab745b..606da9cffb4 100644
--- a/litellm/llms/mongodb/vector_stores/transformation.py
+++ b/litellm/llms/mongodb/vector_stores/transformation.py
@@ -1,4 +1,5 @@
 from collections.abc import Mapping, Sequence
+from ipaddress import ip_address
 from math import isfinite
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Final, Literal, NoReturn
@@ -196,7 +197,7 @@ class MongoDBVectorStoreConfig(BaseQueryEmbeddingVectorStoreConfig):
 
     def get_complete_url(self, api_base: str | None, litellm_params: dict[str, object]) -> str:
         if not api_base:
-            raise config_error("MongoDB sidecar api_base is required, for example http://mongodb-sidecar:8080.")
+            raise config_error("MongoDB sidecar api_base is required, for example http://127.0.0.1:8080.")
         try:
             parsed: Final = urlsplit(api_base)
             valid: Final = parsed.scheme in ("http", "https") and bool(parsed.hostname) and parsed.port != 0
@@ -206,6 +207,17 @@ class MongoDBVectorStoreConfig(BaseQueryEmbeddingVectorStoreConfig):
             raise config_error(
                 "MongoDB sidecar api_base must be an HTTP or HTTPS URL without credentials, query, or fragment."
             )
+        if parsed.scheme == "http":
+            try:
+                loopback: Final = ip_address(parsed.hostname or "").is_loopback
+            except ValueError:
+                raise config_error(
+                    "MongoDB sidecar requires HTTPS. HTTP is supported only for a loopback IP such as 127.0.0.1."
+                ) from None
+            if not loopback:
+                raise config_error(
+                    "MongoDB sidecar requires HTTPS. HTTP is supported only for a loopback IP such as 127.0.0.1."
+                )
         return api_base.rstrip("/")
 
     @staticmethod
@@ -215,7 +227,10 @@ class MongoDBVectorStoreConfig(BaseQueryEmbeddingVectorStoreConfig):
             return 30_000
         if not isinstance(seconds, (int, float)) or not isfinite(seconds) or seconds <= 0:
             raise config_error("MongoDB search timeout must be a positive finite number.")
-        return max(1, min(int(seconds * 1000), 30_000))
+        try:
+            return max(1, int(seconds * 1000))
+        except (ValueError, OverflowError):
+            raise config_error("MongoDB search timeout must be a positive finite number.") from None
 
     @classmethod
     def _params(
diff --git a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py
index bca2b544673..9de473fb1f0 100644
--- a/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py
+++ b/tests/test_litellm/llms/mongodb/vector_stores/test_mongodb_transformation.py
@@ -7,10 +7,10 @@ import httpx
 import pytest
 
 import litellm
-from litellm.llms.custom_httpx.http_handler import HTTPHandler
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from litellm.llms.mongodb.vector_stores.transformation import MongoDBVectorStoreConfig
 from litellm.types.utils import EmbeddingResponse
-from litellm.types.vector_stores import VectorStoreSearchOptionalRequestParams
+from litellm.types.vector_stores import VectorStoreSearchOptionalRequestParams, VectorStoreSearchResponse
 
 BASE_PARAMS: Final = {
     "api_base": "https://sidecar.example/prefix",
@@ -131,52 +131,92 @@ def test_invalid_search_is_rejected_before_embedding(
         (200, {**RESULT, "data": [{"score": "wrong"}]}, litellm.ServiceUnavailableError),
         (0, {}, litellm.Timeout),
         (-1, {}, litellm.BadRequestError),
+        (-2, {"api_base": "http://sidecar.example"}, litellm.BadRequestError),
+        (-2, {"api_base": "http://10.0.0.10:8080"}, litellm.BadRequestError),
+        (-2, {"api_base": "http://localhost:8080"}, litellm.BadRequestError),
         (200, RESULT, None),
     ],
 )
-def test_public_sdk_preserves_http_errors_response_and_timeout(
-    status: int, body: Mapping[str, object], error_type: type[Exception] | None
+@pytest.mark.parametrize("asynchronous", [False, True])
+@pytest.mark.parametrize("timeout", [0.75, 120.0])
+@pytest.mark.parametrize("api_base", ["https://sidecar.example/prefix", "http://127.0.0.1:8080", "http://[::1]:8080"])
+@pytest.mark.asyncio
+async def test_public_sdk_preserves_http_errors_response_and_timeout(
+    status: int,
+    body: Mapping[str, object],
+    error_type: type[Exception] | None,
+    asynchronous: bool,
+    timeout: float,
+    api_base: str,
 ) -> None:
     executor: Final = RecordingEmbeddingExecutor()
     if status == -1:
-        with pytest.raises(litellm.BadRequestError, match="search-only"):
-            litellm.vector_stores.create(custom_llm_provider="mongodb")
-        executor.call.assert_not_called()
+        if asynchronous:
+            with pytest.raises(litellm.BadRequestError, match="search-only"):
+                await litellm.vector_stores.acreate(custom_llm_provider="mongodb")
+        else:
+            with pytest.raises(litellm.BadRequestError, match="search-only"):
+                litellm.vector_stores.create(custom_llm_provider="mongodb")
         return
-
-    def respond(request: httpx.Request) -> httpx.Response:
-        assert request.url == "https://sidecar.example/prefix/v1/vector_stores/policy_index/search"
-        assert request.headers["authorization"] == "Bearer test-sidecar-key"
-        assert request.extensions["timeout"]["read"] == 0.75
-        payload: Final = json.loads(request.content)
-        assert payload["timeout_ms"] == 750
-        assert payload["query_vector"] == [0.1, 0.2, 0.3]
-        if status == 0:
-            raise httpx.ReadTimeout("timed out", request=request)
-        return httpx.Response(status, json=body)
-
-    with httpx.Client(transport=httpx.MockTransport(respond)) as transport:
-        client: Final = HTTPHandler(client=transport)
-        if error_type is not None:
-            with pytest.raises(error_type):
+    if status == -2:
+        rejected_params: Final = {**BASE_PARAMS, "api_base": str(body["api_base"])}
+        if asynchronous:
+            with pytest.raises(litellm.BadRequestError, match="requires HTTPS"):
+                await litellm.vector_stores.asearch(
+                    vector_store_id="policy_index",
+                    query="travel policy",
+                    custom_llm_provider="mongodb",
+                    _direct_vector_store_embedding_executor=executor,
+                    **rejected_params,
+                )
+        else:
+            with pytest.raises(litellm.BadRequestError, match="requires HTTPS"):
                 litellm.vector_stores.search(
                     vector_store_id="policy_index",
                     query="travel policy",
                     custom_llm_provider="mongodb",
                     _direct_vector_store_embedding_executor=executor,
-                    client=client,
-                    timeout=0.75,
-                    **BASE_PARAMS,
+                    **rejected_params,
                 )
-        else:
-            result: Final = litellm.vector_stores.search(
-                vector_store_id="policy_index",
-                query="travel policy",
-                custom_llm_provider="mongodb",
-                _direct_vector_store_embedding_executor=executor,
-                client=client,
-                timeout=0.75,
-                **BASE_PARAMS,
-            )
-            assert result == RESULT
+        executor.call.assert_not_called()
+        return
+
+    def respond(request: httpx.Request) -> httpx.Response:
+        assert request.url == f"{api_base}/v1/vector_stores/policy_index/search"
+        assert request.headers["authorization"] == "Bearer test-sidecar-key"
+        assert request.extensions["timeout"]["read"] == timeout
+        payload: Final = json.loads(request.content)
+        assert payload["timeout_ms"] == int(timeout * 1000)
+        assert payload["query_vector"] == [0.1, 0.2, 0.3]
+        if status == 0:
+            raise httpx.ReadTimeout("timed out", request=request)
+        return httpx.Response(status, json=body)
+
+    async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as async_transport:
+        with httpx.Client(transport=httpx.MockTransport(respond)) as transport:
+            client: Final = AsyncHTTPHandler() if asynchronous else HTTPHandler(client=transport)
+            if isinstance(client, AsyncHTTPHandler):
+                await client.client.aclose()
+                client.client = async_transport
+
+            async def search() -> VectorStoreSearchResponse:
+                kwargs: Final = {
+                    **BASE_PARAMS,
+                    "api_base": api_base,
+                    "vector_store_id": "policy_index",
+                    "query": "travel policy",
+                    "custom_llm_provider": "mongodb",
+                    "_direct_vector_store_embedding_executor": executor,
+                    "client": client,
+                    "timeout": timeout,
+                }
+                if asynchronous:
+                    return await litellm.vector_stores.asearch(**kwargs)
+                return litellm.vector_stores.search(**kwargs)
+
+            if error_type is not None:
+                with pytest.raises(error_type):
+                    await search()
+            else:
+                assert await search() == RESULT
     executor.call.assert_called_once_with("embedding-alias", "travel policy", {})
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.test.tsx
index 8da7098b695..03020bbc4a5 100644
--- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.test.tsx
+++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/VectorStoreForm.test.tsx
@@ -69,7 +69,7 @@ describe("VectorStoreForm", () => {
   });
 });
 
-const MONGODB_SIDECAR_URL = "http://mongodb-sidecar:8080";
+const MONGODB_SIDECAR_URL = "http://127.0.0.1:8080";
 
 const MONGODB_REQUIRED_FORM_VALUES = {
   api_base: MONGODB_SIDECAR_URL,
diff --git a/ui/litellm-dashboard/src/components/vector_store_providers.tsx b/ui/litellm-dashboard/src/components/vector_store_providers.tsx
index 6a8b2f405d2..79e711f6f98 100644
--- a/ui/litellm-dashboard/src/components/vector_store_providers.tsx
+++ b/ui/litellm-dashboard/src/components/vector_store_providers.tsx
@@ -177,8 +177,8 @@ export const vectorStoreProviderFields: Record<string, VectorStoreFieldConfig[]>
     {
       name: "api_base",
       label: "Sidecar URL",
-      tooltip: "The URL of your separately deployed MongoDB sidecar. Configure MongoDB credentials in the sidecar",
-      placeholder: "http://mongodb-sidecar:8080",
+      tooltip: "Use HTTPS for a remote sidecar, or HTTP with a loopback IP for a sidecar on the same host or Pod",
+      placeholder: "http://127.0.0.1:8080",
       required: true,
       type: "text",
     },

From 95c0f9db7d92cefafa5e2d9799786b4931c9a390 Mon Sep 17 00:00:00 2001
From: Clement <clement.ho@paradigm.co>
Date: Wed, 9 Sep 2026 01:05:23 +0800
Subject: [PATCH 57/85] fix(router): rank streaming latency routing by raw
 TTFT, not TTFT per token (#40202)

* fix(router): rank streaming latency routing by raw TTFT, not TTFT per token

Latency-based routing divided time-to-first-token by completion_tokens
before storing it, so a deployment that streamed a long answer looked
faster to first token than one that answered briefly. TTFT is now stored
as plain seconds (first token time minus request start) in both the sync
and async success handlers, which is what the routing decision compares.

Non-streaming latency normalization per output token is unchanged.

Claude-Session: https://claude.ai/code/session_01Ttd5Q9ZhRPB4ch5guos3rj

* fix(router): store streaming TTFT under a seconds-only cache key

Workers on the previous release keep writing seconds-per-token samples
under "time_to_first_token" in the shared router cache during a rolling
deploy, so mixing the new raw-seconds samples into the same list averaged
incompatible units. Raw TTFT now lives under "time_to_first_token_seconds"
and routing reads only that key.

Also fix the regression test's token counts: with 50 tokens on the fast
deployment and 500 on the slow one the old per-token formula picks the
slow deployment, so the routing assertion now catches the bug.

Claude-Session: https://claude.ai/code/session_01Ttd5Q9ZhRPB4ch5guos3rj

* test(router): cover the TTFT sliding window from the unit-test shard

Move the TTFT list trimming checks from the CircleCI-only suite into the
mapped unit test file as one sync/async parametrized test, so the changed
lines in lowest_latency.py are exercised by the GitHub unit-test shard
that reports patch coverage.

Claude-Session: https://claude.ai/code/session_01Ttd5Q9ZhRPB4ch5guos3rj
---
 litellm/router_strategy/lowest_latency.py     |  54 +++----
 .../test_lowest_latency_routing.py            | 136 ------------------
 .../router_strategy/test_lowest_latency.py    | 127 ++++++++++++++++
 3 files changed, 149 insertions(+), 168 deletions(-)

diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py
index bb6877a032b..805d4ff9080 100644
--- a/litellm/router_strategy/lowest_latency.py
+++ b/litellm/router_strategy/lowest_latency.py
@@ -32,6 +32,12 @@ def _average_latency(samples: Sequence[float]) -> float:
     return sum(samples) / len(samples)
 
 
+def _ttft_seconds(elapsed: timedelta | float) -> float:
+    if isinstance(elapsed, timedelta):
+        return elapsed.total_seconds()
+    return float(elapsed)
+
+
 class LowestLatencyLoggingHandler(CustomLogger):
     test_flag: bool = False
     logged_success: int = 0
@@ -86,14 +92,13 @@ class LowestLatencyLoggingHandler(CustomLogger):
                     # breaks JSON serialization when the router cache syncs to
                     # Redis (issue #33169)
                     response_ms = response_ms.total_seconds()
-                time_to_first_token_response_time = None
+                time_to_first_token: float | None = None
 
                 if kwargs.get("stream", None) is not None and kwargs["stream"] is True:
                     # only log ttft for streaming request
-                    time_to_first_token_response_time = kwargs.get("completion_start_time", end_time) - start_time
+                    time_to_first_token = _ttft_seconds(kwargs.get("completion_start_time", end_time) - start_time)
 
                 final_value: float = response_ms
-                time_to_first_token: float | None = None
                 total_tokens = 0
 
                 if isinstance(response_obj, ModelResponse):
@@ -111,13 +116,6 @@ class LowestLatencyLoggingHandler(CustomLogger):
                         else:
                             final_value = response_seconds
 
-                        if time_to_first_token_response_time is not None:
-                            if isinstance(time_to_first_token_response_time, timedelta):
-                                ttft_seconds = time_to_first_token_response_time.total_seconds()
-                            else:
-                                ttft_seconds = time_to_first_token_response_time
-                            time_to_first_token = safe_divide_seconds(ttft_seconds, completion_tokens)
-
                 # ------------
                 # Update usage
                 # ------------
@@ -138,14 +136,14 @@ class LowestLatencyLoggingHandler(CustomLogger):
                 ## Time to first token
                 if time_to_first_token is not None:
                     if (
-                        len(request_count_dict[id].get("time_to_first_token", []))
+                        len(request_count_dict[id].get("time_to_first_token_seconds", []))
                         < self.routing_args.max_latency_list_size
                     ):
-                        request_count_dict[id].setdefault("time_to_first_token", []).append(time_to_first_token)
+                        request_count_dict[id].setdefault("time_to_first_token_seconds", []).append(time_to_first_token)
                     else:
-                        request_count_dict[id]["time_to_first_token"] = request_count_dict[id]["time_to_first_token"][
-                            1:
-                        ] + [time_to_first_token]
+                        request_count_dict[id]["time_to_first_token_seconds"] = request_count_dict[id][
+                            "time_to_first_token_seconds"
+                        ][1:] + [time_to_first_token]
 
                 if precise_minute not in request_count_dict[id]:
                     request_count_dict[id][precise_minute] = {}
@@ -252,7 +250,7 @@ class LowestLatencyLoggingHandler(CustomLogger):
                     {model_group}_map: {
                         id: {
                             "latency": [..]
-                            "time_to_first_token": [..]
+                            "time_to_first_token_seconds": [..]
                             f"{date:hour:minute}" : {"tpm": 34, "rpm": 3}
                         }
                     }
@@ -273,14 +271,13 @@ class LowestLatencyLoggingHandler(CustomLogger):
                     # breaks JSON serialization when the router cache syncs to
                     # Redis (issue #33169)
                     response_ms = response_ms.total_seconds()
-                time_to_first_token_response_time = None
+                time_to_first_token: float | None = None
                 if kwargs.get("stream", None) is not None and kwargs["stream"] is True:
                     # only log ttft for streaming request
-                    time_to_first_token_response_time = kwargs.get("completion_start_time", end_time) - start_time
+                    time_to_first_token = _ttft_seconds(kwargs.get("completion_start_time", end_time) - start_time)
 
                 final_value: float = response_ms
                 total_tokens = 0
-                time_to_first_token: float | None = None
 
                 if isinstance(response_obj, ModelResponse):
                     _usage: Final = getattr(response_obj, "usage", None)
@@ -296,13 +293,6 @@ class LowestLatencyLoggingHandler(CustomLogger):
                             final_value = float(normalized_value)
                         else:
                             final_value = response_seconds
-
-                        if time_to_first_token_response_time is not None:
-                            if isinstance(time_to_first_token_response_time, timedelta):
-                                ttft_seconds = time_to_first_token_response_time.total_seconds()
-                            else:
-                                ttft_seconds = time_to_first_token_response_time
-                            time_to_first_token = safe_divide_seconds(ttft_seconds, completion_tokens)
                 # ------------
                 # Update usage
                 # ------------
@@ -328,14 +318,14 @@ class LowestLatencyLoggingHandler(CustomLogger):
                 ## Time to first token
                 if time_to_first_token is not None:
                     if (
-                        len(request_count_dict[id].get("time_to_first_token", []))
+                        len(request_count_dict[id].get("time_to_first_token_seconds", []))
                         < self.routing_args.max_latency_list_size
                     ):
-                        request_count_dict[id].setdefault("time_to_first_token", []).append(time_to_first_token)
+                        request_count_dict[id].setdefault("time_to_first_token_seconds", []).append(time_to_first_token)
                     else:
-                        request_count_dict[id]["time_to_first_token"] = request_count_dict[id]["time_to_first_token"][
-                            1:
-                        ] + [time_to_first_token]
+                        request_count_dict[id]["time_to_first_token_seconds"] = request_count_dict[id][
+                            "time_to_first_token_seconds"
+                        ][1:] + [time_to_first_token]
 
                 if precise_minute not in request_count_dict[id]:
                     request_count_dict[id][precise_minute] = {}
@@ -433,7 +423,7 @@ class LowestLatencyLoggingHandler(CustomLogger):
                 or float("inf")
             )
             item_latency = item_map.get("latency", [])
-            item_ttft_latency = item_map.get("time_to_first_token", [])
+            item_ttft_latency = item_map.get("time_to_first_token_seconds", [])
             item_rpm = item_map.get(precise_minute, {}).get("rpm", 0)
             item_tpm = item_map.get(precise_minute, {}).get("tpm", 0)
 
diff --git a/tests/local_testing/test_lowest_latency_routing.py b/tests/local_testing/test_lowest_latency_routing.py
index 598b1dbcaf9..aba4500199a 100644
--- a/tests/local_testing/test_lowest_latency_routing.py
+++ b/tests/local_testing/test_lowest_latency_routing.py
@@ -1077,73 +1077,6 @@ async def test_latency_list_trimming_discards_oldest_entry_async():
         ), f"Oldest latency {oldest_latency} should have been discarded"
 
 
-def test_ttft_list_trimming_discards_oldest_entry():
-    """
-    The time_to_first_token list trims the oldest entry when full, matching
-    the behavior of the latency list.
-    """
-    max_size = 3
-    test_cache = DualCache()
-    lowest_latency_logger = LowestLatencyLoggingHandler(
-        router_cache=test_cache, routing_args={"max_latency_list_size": max_size}
-    )
-
-    model_group = "gpt-3.5-turbo"
-    deployment_id = "test-deployment"
-
-    ttft_values = []
-    for i in range(max_size + 1):
-        start_time = time.time()
-        expected_ttft = float(i + 1) * 0.1  # 0.1, 0.2, 0.3, 0.4
-        completion_start_time = start_time + expected_ttft
-        end_time = start_time + float(i + 1)
-        ttft_values.append(expected_ttft)
-
-        kwargs = {
-            "litellm_params": {
-                "metadata": {
-                    "model_group": model_group,
-                    "deployment": "azure/gpt-4.1-mini",
-                },
-                "model_info": {"id": deployment_id},
-            },
-            "stream": True,
-            "completion_start_time": completion_start_time,
-        }
-        # TTFT is only recorded when response_obj is a ModelResponse.
-        response_obj = litellm.ModelResponse(
-            usage=litellm.Usage(completion_tokens=1, total_tokens=1)
-        )
-
-        lowest_latency_logger.log_success_event(
-            response_obj=response_obj,
-            kwargs=kwargs,
-            start_time=start_time,
-            end_time=end_time,
-        )
-
-    latency_key = f"{model_group}_map"
-    cached_data = test_cache.get_cache(key=latency_key)
-    ttft_list = cached_data[deployment_id].get("time_to_first_token", [])
-
-    assert (
-        len(ttft_list) == max_size
-    ), f"Expected {max_size} entries, got {len(ttft_list)}"
-
-    newest_ttft = ttft_values[-1]
-    oldest_ttft = ttft_values[0]
-    tolerance = 0.05
-
-    assert (
-        abs(ttft_list[-1] - newest_ttft) < tolerance
-    ), f"Newest TTFT {newest_ttft} should be at end of list"
-
-    for ttft in ttft_list:
-        assert (
-            abs(ttft - oldest_ttft) > tolerance
-        ), f"Oldest TTFT {oldest_ttft} should have been discarded"
-
-
 @pytest.mark.asyncio
 async def test_timeout_penalty_discards_oldest_entry():
     """
@@ -1269,72 +1202,3 @@ def test_list_order_preserved_after_multiple_trims():
         assert (
             abs(latency_list[i] - expected) < tolerance
         ), f"At index {i}, expected ~{expected}, got {latency_list[i]}"
-
-
-@pytest.mark.asyncio
-async def test_ttft_list_trimming_discards_oldest_entry_async():
-    """
-    Async counterpart: the time_to_first_token list trims the oldest entry
-    when full. Exercises the async_log_success_event TTFT path, which only
-    runs when response_obj is a ModelResponse and the call is marked as
-    streaming with a completion_start_time.
-    """
-    max_size = 3
-    test_cache = DualCache()
-    lowest_latency_logger = LowestLatencyLoggingHandler(
-        router_cache=test_cache, routing_args={"max_latency_list_size": max_size}
-    )
-
-    model_group = "gpt-3.5-turbo"
-    deployment_id = "test-deployment"
-
-    ttft_values = []
-    for i in range(max_size + 1):
-        start_time = time.time()
-        expected_ttft = float(i + 1) * 0.1  # 0.1, 0.2, 0.3, 0.4
-        completion_start_time = start_time + expected_ttft
-        end_time = start_time + float(i + 1)
-        ttft_values.append(expected_ttft)
-
-        kwargs = {
-            "litellm_params": {
-                "metadata": {
-                    "model_group": model_group,
-                    "deployment": "azure/gpt-4.1-mini",
-                },
-                "model_info": {"id": deployment_id},
-            },
-            "stream": True,
-            "completion_start_time": completion_start_time,
-        }
-        response_obj = litellm.ModelResponse(
-            usage=litellm.Usage(completion_tokens=1, total_tokens=1)
-        )
-
-        await lowest_latency_logger.async_log_success_event(
-            response_obj=response_obj,
-            kwargs=kwargs,
-            start_time=start_time,
-            end_time=end_time,
-        )
-
-    latency_key = f"{model_group}_map"
-    cached_data = await test_cache.async_get_cache(key=latency_key)
-    ttft_list = cached_data[deployment_id].get("time_to_first_token", [])
-
-    assert (
-        len(ttft_list) == max_size
-    ), f"Expected {max_size} entries, got {len(ttft_list)}"
-
-    newest_ttft = ttft_values[-1]
-    oldest_ttft = ttft_values[0]
-    tolerance = 0.05
-
-    assert (
-        abs(ttft_list[-1] - newest_ttft) < tolerance
-    ), f"Newest TTFT {newest_ttft} should be at end of list"
-
-    for ttft in ttft_list:
-        assert (
-            abs(ttft - oldest_ttft) > tolerance
-        ), f"Oldest TTFT {oldest_ttft} should have been discarded"
diff --git a/tests/test_litellm/router_strategy/test_lowest_latency.py b/tests/test_litellm/router_strategy/test_lowest_latency.py
index eb02459be68..812d7bbff32 100644
--- a/tests/test_litellm/router_strategy/test_lowest_latency.py
+++ b/tests/test_litellm/router_strategy/test_lowest_latency.py
@@ -165,6 +165,133 @@ def test_sync_chat_zero_completion_tokens_falls_back_to_seconds():
     json.dumps({"latency": latencies})
 
 
+MODEL_GROUP = "gpt-4o-mini"
+FAST_TTFT_ID = "fast-ttft-short-output"
+SLOW_TTFT_ID = "slow-ttft-long-output"
+STREAMING_DEPLOYMENTS = [
+    {"model_info": {"id": FAST_TTFT_ID}, "litellm_params": {}},
+    {"model_info": {"id": SLOW_TTFT_ID}, "litellm_params": {}},
+]
+
+
+def _streaming_kwargs(deployment_id: str, start_time: datetime, ttft_seconds: float):
+    return {
+        "litellm_params": {
+            "metadata": {"model_group": MODEL_GROUP},
+            "model_info": {"id": deployment_id},
+        },
+        "stream": True,
+        "completion_start_time": start_time + timedelta(seconds=ttft_seconds),
+    }
+
+
+def _recorded_ttft(cache: DualCache, deployment_id: str):
+    cached = cache.get_cache(key=f"{MODEL_GROUP}_map") or {}
+    return cached.get(deployment_id, {}).get("time_to_first_token_seconds", [])
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("sync_mode", [True, False], ids=["sync", "async"])
+async def test_streaming_ttft_ranking_ignores_completion_length(sync_mode: bool):
+    """Deployment A: TTFT 1s, 50 completion tokens. Deployment B: TTFT 3s, 500
+    completion tokens. Dividing TTFT by completion tokens made B look faster
+    (3/500 = 0.006 beats 1/50 = 0.02); actual TTFT must win."""
+    cache = DualCache()
+    handler = LowestLatencyLoggingHandler(router_cache=cache)
+    start_time = datetime(2026, 1, 1, 12, 0, 0)
+    end_time = start_time + timedelta(seconds=10)
+
+    samples = (
+        (FAST_TTFT_ID, 1.0, 50),
+        (SLOW_TTFT_ID, 3.0, 500),
+    )
+    for deployment_id, ttft, completion_tokens in samples:
+        kwargs = _streaming_kwargs(deployment_id, start_time, ttft)
+        response_obj = _chat_response(completion_tokens=completion_tokens)
+        if sync_mode:
+            handler.log_success_event(
+                response_obj=response_obj, kwargs=kwargs, start_time=start_time, end_time=end_time
+            )
+        else:
+            await handler.async_log_success_event(
+                response_obj=response_obj, kwargs=kwargs, start_time=start_time, end_time=end_time
+            )
+
+    assert _recorded_ttft(cache, FAST_TTFT_ID) == [pytest.approx(1.0)]
+    assert _recorded_ttft(cache, SLOW_TTFT_ID) == [pytest.approx(3.0)]
+
+    request_kwargs = {"stream": True, "metadata": {}}
+    if sync_mode:
+        picked = handler.get_available_deployments(
+            model_group=MODEL_GROUP, healthy_deployments=STREAMING_DEPLOYMENTS, request_kwargs=request_kwargs
+        )
+    else:
+        picked = await handler.async_get_available_deployments(
+            model_group=MODEL_GROUP, healthy_deployments=STREAMING_DEPLOYMENTS, request_kwargs=request_kwargs
+        )
+
+    assert picked is not None
+    assert picked["model_info"]["id"] == FAST_TTFT_ID
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("sync_mode", [True, False], ids=["sync", "async"])
+async def test_ttft_window_keeps_newest_samples_when_full(sync_mode: bool):
+    """Float timestamps, as the SDK passes them. Once max_latency_list_size
+    samples exist the oldest TTFT is dropped so the window slides."""
+    max_size = 3
+    cache = DualCache()
+    handler = LowestLatencyLoggingHandler(router_cache=cache, routing_args={"max_latency_list_size": max_size})
+    start_time = 1_700_000_000.0
+    ttfts = (0.1, 0.2, 0.3, 0.4)
+
+    for ttft in ttfts:
+        kwargs = {
+            "litellm_params": {
+                "metadata": {"model_group": MODEL_GROUP},
+                "model_info": {"id": FAST_TTFT_ID},
+            },
+            "stream": True,
+            "completion_start_time": start_time + ttft,
+        }
+        response_obj = _chat_response(completion_tokens=1)
+        if sync_mode:
+            handler.log_success_event(
+                response_obj=response_obj, kwargs=kwargs, start_time=start_time, end_time=start_time + 1.0
+            )
+        else:
+            await handler.async_log_success_event(
+                response_obj=response_obj, kwargs=kwargs, start_time=start_time, end_time=start_time + 1.0
+            )
+
+    assert _recorded_ttft(cache, FAST_TTFT_ID) == [pytest.approx(ttft) for ttft in ttfts[-max_size:]]
+
+
+@pytest.mark.asyncio
+async def test_streaming_routing_ignores_per_token_ttft_samples_from_older_workers():
+    """Workers on the previous release share the Redis map and keep writing
+    seconds-per-token under the old "time_to_first_token" key during a rolling
+    deploy. Those samples favor SLOW; routing must only read the seconds key."""
+    cache = DualCache()
+    handler = LowestLatencyLoggingHandler(router_cache=cache)
+    cache.set_cache(
+        key=f"{MODEL_GROUP}_map",
+        value={
+            FAST_TTFT_ID: {"time_to_first_token": [0.02], "time_to_first_token_seconds": [1.0]},
+            SLOW_TTFT_ID: {"time_to_first_token": [0.006], "time_to_first_token_seconds": [3.0]},
+        },
+    )
+
+    picked = await handler.async_get_available_deployments(
+        model_group=MODEL_GROUP,
+        healthy_deployments=STREAMING_DEPLOYMENTS,
+        request_kwargs={"stream": True, "metadata": {}},
+    )
+
+    assert picked is not None
+    assert picked["model_info"]["id"] == FAST_TTFT_ID
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
     "cached_entry",

From 978f93ce9cf3e2748494a1c8b67b48488930b06b Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 8 Sep 2026 10:05:32 -0700
Subject: [PATCH 58/85] fix(tool_permission): log expected skip and deny events
 below WARNING (#40208)

The no-tools skip path in async_pre_call_hook now logs at DEBUG and the
denied-by-rule messages in async_pre_call_hook and _evaluate_tool_calls
log at INFO. The malformed tool arguments warning is unchanged. Adds
regression tests pinning each level

Co-authored-by: yucheng <yucheng@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../guardrail_hooks/tool_permission.py        |  6 +-
 .../guardrail_hooks/test_tool_permission.py   | 75 +++++++++++++++++++
 2 files changed, 78 insertions(+), 3 deletions(-)

diff --git a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py
index a8b33109900..e20f0b320b9 100644
--- a/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py
+++ b/litellm/proxy/guardrails/guardrail_hooks/tool_permission.py
@@ -547,7 +547,7 @@ class ToolPermissionGuardrail(CustomGuardrail):
 
         for _tool_call, is_allowed, _rule_id, message in checked:
             if not is_allowed and message is not None:
-                verbose_proxy_logger.warning("Tool Permission Guardrail: %s", message)
+                verbose_proxy_logger.info("Tool Permission Guardrail: %s", message)
                 if self.on_disallowed_action == "block":
                     raise GuardrailRaisedException(
                         guardrail_name=self.guardrail_name, message=message, blocked_content=True
@@ -809,7 +809,7 @@ class ToolPermissionGuardrail(CustomGuardrail):
 
         new_tools: Final = self._collect_request_tools(data)
         if not new_tools:
-            verbose_proxy_logger.warning(
+            verbose_proxy_logger.debug(
                 "Tool Permission Guardrail: not running guardrail. No tools or functions in data"
             )
             return data
@@ -820,7 +820,7 @@ class ToolPermissionGuardrail(CustomGuardrail):
             is_allowed, _, message = self._check_tool_permission(tool_name, tool_type)
 
             if not is_allowed and message is not None:
-                verbose_proxy_logger.warning("Tool Permission Guardrail: %s", message)
+                verbose_proxy_logger.info("Tool Permission Guardrail: %s", message)
                 if self.on_disallowed_action == "block":
                     raise HTTPException(
                         status_code=400,
diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_permission.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_permission.py
index 0dbd4591ac9..87a1b84acc5 100644
--- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_permission.py
+++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_tool_permission.py
@@ -3,6 +3,7 @@ Unit tests for Tool Permission Guardrail (OpenAI tool_calls semantics)
 """
 
 import json
+import logging
 import re
 from unittest.mock import patch
 
@@ -520,6 +521,80 @@ class TestToolPermissionGuardrail:
                 )
         assert excinfo.value.status_code == 400
 
+    @pytest.mark.asyncio
+    async def test_async_pre_call_hook_without_tools_logs_skip_at_debug(self, caplog):
+        data = {"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}]}
+
+        with caplog.at_level(logging.DEBUG, logger="LiteLLM Proxy"):
+            with patch.object(self.guardrail, "should_run_guardrail", return_value=True):
+                result = await self.guardrail.async_pre_call_hook(
+                    user_api_key_dict=UserAPIKeyAuth(),
+                    cache=DualCache(default_in_memory_ttl=1),
+                    data=data,
+                    call_type="completion",
+                )
+
+        assert result is data
+        skip_levels = [r.levelno for r in caplog.records if "No tools or functions in data" in r.getMessage()]
+        assert skip_levels == [logging.DEBUG]
+        assert [r.getMessage() for r in caplog.records if r.levelno >= logging.WARNING] == []
+
+    @pytest.mark.asyncio
+    async def test_async_pre_call_hook_denied_tool_logs_at_info(self, caplog):
+        data = {"tools": [{"type": "function", "function": {"name": "Read"}}]}
+
+        with caplog.at_level(logging.DEBUG, logger="LiteLLM Proxy"):
+            with patch.object(self.guardrail, "should_run_guardrail", return_value=True):
+                with pytest.raises(HTTPException):
+                    await self.guardrail.async_pre_call_hook(
+                        user_api_key_dict=UserAPIKeyAuth(),
+                        cache=DualCache(default_in_memory_ttl=1),
+                        data=data,
+                        call_type="completion",
+                    )
+
+        denied_levels = [
+            r.levelno
+            for r in caplog.records
+            if r.getMessage() == "Tool Permission Guardrail: Tool 'Read' denied by rule 'deny_read'"
+        ]
+        assert denied_levels == [logging.INFO]
+        assert [r.getMessage() for r in caplog.records if r.levelno >= logging.WARNING] == []
+
+    @pytest.mark.asyncio
+    async def test_async_post_call_success_hook_denied_tool_logs_at_info(self, caplog):
+        tool_call = {"function": {"name": "Read", "arguments": "{}"}, "type": "function"}
+        response = ModelResponse(choices=[Choices(message={"tool_calls": [tool_call]})])
+
+        with caplog.at_level(logging.DEBUG, logger="LiteLLM Proxy"):
+            with patch.object(self.guardrail, "should_run_guardrail", return_value=True):
+                with pytest.raises(GuardrailRaisedException):
+                    await self.guardrail.async_post_call_success_hook(
+                        data={"guardrails": ["test-tool-permission"]},
+                        user_api_key_dict=UserAPIKeyAuth(),
+                        response=response,
+                    )
+
+        denied_levels = [
+            r.levelno
+            for r in caplog.records
+            if r.getMessage() == "Tool Permission Guardrail: Tool 'Read' denied by rule 'deny_read'"
+        ]
+        assert denied_levels == [logging.INFO]
+        assert [r.getMessage() for r in caplog.records if r.levelno >= logging.WARNING] == []
+
+    def test_parse_tool_call_arguments_malformed_json_logs_warning(self, caplog):
+        tool_call = ChatCompletionMessageToolCall(function={"name": "Bash", "arguments": "{not json"}, id="call_1")
+
+        with caplog.at_level(logging.DEBUG, logger="LiteLLM Proxy"):
+            parsed, error = self.guardrail._parse_tool_call_arguments(tool_call)
+
+        assert parsed is None
+        assert error == "arguments could not be parsed"
+        warning_messages = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING]
+        assert len(warning_messages) == 1
+        assert warning_messages[0].startswith("Tool Permission Guardrail: Failed to decode arguments for tool Bash")
+
     @pytest.mark.asyncio
     async def test_async_pre_call_hook_blocks_legacy_functions(self):
         data = {

From 99824533ff7a99c07e0c768f61fcd5b5204a7794 Mon Sep 17 00:00:00 2001
From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 8 Sep 2026 10:06:53 -0700
Subject: [PATCH 59/85] fix(proxy): kill the whole prisma process group when a
 boot migration command times out (#39509)

* fix(proxy): kill the whole prisma process group when a boot migration command times out

* fix(proxy): keep boot alive when the installed proxy extras has no prisma runner
---
 litellm/proxy/db/check_migration.py           | 39 ++++++---
 litellm/proxy/db/prisma_client.py             | 22 ++++--
 tests/test_litellm/proxy/db/conftest.py       | 79 +++++++++++++++++++
 .../proxy/db/test_check_migration.py          | 32 ++++++++
 .../proxy/db/test_prisma_client.py            | 59 +++++++++-----
 5 files changed, 197 insertions(+), 34 deletions(-)

diff --git a/litellm/proxy/db/check_migration.py b/litellm/proxy/db/check_migration.py
index b7a07d4eeea..6e2a06c96e1 100644
--- a/litellm/proxy/db/check_migration.py
+++ b/litellm/proxy/db/check_migration.py
@@ -46,17 +46,32 @@ def extract_sql_commands(diff_output: str) -> list[str]:
 
 def check_prisma_schema_diff_helper(db_url: str) -> tuple[bool, list[str]]:
     """Checks for differences between current database and Prisma schema.
+
+    Never raises: a diff that cannot be produced, because the runner is missing,
+    because the command failed, or because it outlived its budget, is reported as
+    "no diff" so boot continues.
+
     Returns:
         A tuple containing:
         - A boolean indicating if differences were found (True) or not (False).
-        - A string with the diff output or error message.
-    Raises:
-        subprocess.CalledProcessError: If the Prisma command fails.
-        Exception: For any other errors during execution.
+        - The SQL commands that would close the diff, empty when there is none.
     """
-    verbose_logger.debug("Checking for Prisma schema diff...")
     try:
-        result: Final = subprocess.run(
+        from litellm_proxy_extras.prisma_toolchain import (
+            PRISMA_COMMAND_TIMEOUT_ENV_VAR,
+            prisma_command_timeout,
+            run_prisma,
+        )
+    except ImportError as e:
+        print(  # noqa: T201  # boot-time operator output, same channel as this helper's other messages
+            f"Skipping the migration diff: litellm-proxy-extras has no Prisma runner. Error: {e}"
+        )
+        return False, []
+
+    verbose_logger.debug("Checking for Prisma schema diff...")
+    timeout: Final = prisma_command_timeout()
+    try:
+        result: Final = run_prisma(
             [
                 "prisma",
                 "migrate",
@@ -67,12 +82,10 @@ def check_prisma_schema_diff_helper(db_url: str) -> tuple[bool, list[str]]:
                 "./schema.prisma",
                 "--script",
             ],
-            capture_output=True,
-            text=True,
-            check=True,
+            timeout=timeout,
+            env=os.environ.copy(),
         )
 
-        # return True, "Migration diff generated successfully."
         sql_commands: Final = extract_sql_commands(result.stdout)
 
         if sql_commands:
@@ -83,6 +96,12 @@ def check_prisma_schema_diff_helper(db_url: str) -> tuple[bool, list[str]]:
             return True, sql_commands
         else:
             return False, []
+    except subprocess.TimeoutExpired:
+        print(  # noqa: T201  # boot-time operator output, same channel as this helper's other messages
+            f"Timed out after {timeout}s generating the migration diff. "
+            f"Raise {PRISMA_COMMAND_TIMEOUT_ENV_VAR} if this database needs longer."
+        )
+        return False, []
     except subprocess.CalledProcessError as e:
         error_message: Final = f"Failed to generate migration diff. Error: {e.stderr}"
         print(error_message)  # noqa: T201
diff --git a/litellm/proxy/db/prisma_client.py b/litellm/proxy/db/prisma_client.py
index 2190ae55fd2..9ea2432f2f5 100644
--- a/litellm/proxy/db/prisma_client.py
+++ b/litellm/proxy/db/prisma_client.py
@@ -937,9 +937,17 @@ class PrismaManager:
                         use_v2_resolver=use_v2_resolver,
                     )
                 else:
+                    try:
+                        from litellm_proxy_extras.prisma_toolchain import (
+                            prisma_command_timeout,
+                            run_prisma,
+                        )
+                    except ImportError as e:
+                        verbose_proxy_logger.error("\x1b[1;31mLiteLLM: Failed to import proxy extras. Got %s\x1b[0m", e)
+                        return False
+
                     PrismaManager._raise_if_partitioned_spend_logs()
-                    # Use prisma db push with increased timeout
-                    subprocess.run(
+                    run_prisma(
                         [
                             "prisma",
                             "db",
@@ -947,13 +955,15 @@ class PrismaManager:
                             "--accept-data-loss",
                             "--skip-generate",
                         ],
-                        timeout=60,
-                        check=True,
+                        timeout=prisma_command_timeout(),
+                        env=os.environ.copy(),
+                        stdout=None,
+                        stderr=None,
                     )
                     PrismaManager._apply_replica_identity_full_if_requested()
                     return True
-            except subprocess.TimeoutExpired:
-                verbose_proxy_logger.warning("Attempt %s timed out", attempt + 1)
+            except subprocess.TimeoutExpired as e:
+                verbose_proxy_logger.warning("Attempt %s timed out after %.0fs", attempt + 1, e.timeout)
                 time.sleep(random.randrange(5, 15))
             except subprocess.CalledProcessError as e:
                 attempts_left = 3 - attempt
diff --git a/tests/test_litellm/proxy/db/conftest.py b/tests/test_litellm/proxy/db/conftest.py
index 6f67b91ac1d..03d7ea81257 100644
--- a/tests/test_litellm/proxy/db/conftest.py
+++ b/tests/test_litellm/proxy/db/conftest.py
@@ -1,5 +1,11 @@
+import json
 import os
+import signal
+import sys
+import time
 from collections.abc import Generator
+from dataclasses import dataclass
+from pathlib import Path
 from typing import Optional
 
 import pytest
@@ -75,3 +81,76 @@ def reset_entra_token_provider_cache() -> Generator[None, None, None]:
 def unset_database_url(monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.setenv("DATABASE_URL", "about-to-be-unset")
     monkeypatch.delenv("DATABASE_URL")
+
+
+FAKE_PRISMA_CLI = """#!{python}
+import json
+import os
+import pathlib
+import subprocess
+import sys
+import time
+
+calls_file = pathlib.Path(os.environ["FAKE_PRISMA_CALLS"])
+earlier_calls = calls_file.read_text().splitlines() if calls_file.exists() else []
+with calls_file.open("a") as log:
+    print(json.dumps(sys.argv[1:]), file=log)
+if not earlier_calls and os.environ.get("FAKE_PRISMA_HANG_FIRST"):
+    grandchild = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(600)"])
+    pathlib.Path(os.environ["FAKE_PRISMA_GRANDCHILD_PIDFILE"]).write_text(str(grandchild.pid))
+    time.sleep(600)
+sys.exit(0)
+"""
+
+
+@dataclass(frozen=True, slots=True)
+class FakePrismaCli:
+    """A stand-in `prisma` on PATH, recording every invocation.
+
+    With FAKE_PRISMA_HANG_FIRST set it hangs on its first call from a process tree
+    of its own, the way the real CLI wraps Node around a Rust schema engine, so a
+    timeout that kills only the direct child leaves the rest of that tree running.
+    """
+
+    calls_file: Path
+    grandchild_pidfile: Path
+
+    @property
+    def calls(self) -> list[list[str]]:
+        if not self.calls_file.exists():
+            return []
+        return [json.loads(line) for line in self.calls_file.read_text().splitlines()]
+
+    def grandchild_is_gone(self, within_seconds: float) -> bool:
+        deadline = time.monotonic() + within_seconds
+        while time.monotonic() < deadline:
+            try:
+                os.kill(int(self.grandchild_pidfile.read_text()), 0)
+            except ProcessLookupError:
+                return True
+            time.sleep(0.05)
+        return False
+
+
+@pytest.fixture
+def fake_prisma_cli(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Generator[FakePrismaCli, None, None]:
+    bin_dir = tmp_path / "fakebin"
+    bin_dir.mkdir()
+    script = bin_dir / "prisma"
+    script.write_text(FAKE_PRISMA_CLI.format(python=sys.executable))
+    script.chmod(0o755)
+    cli = FakePrismaCli(
+        calls_file=tmp_path / "calls.jsonl",
+        grandchild_pidfile=tmp_path / "grandchild.pid",
+    )
+    monkeypatch.setenv("PATH", f"{bin_dir}{os.pathsep}{os.environ['PATH']}")
+    monkeypatch.setenv("FAKE_PRISMA_CALLS", str(cli.calls_file))
+    monkeypatch.setenv("FAKE_PRISMA_GRANDCHILD_PIDFILE", str(cli.grandchild_pidfile))
+    monkeypatch.setenv("LITELLM_PRISMA_COMMAND_TIMEOUT", "1")
+    monkeypatch.delenv("FAKE_PRISMA_HANG_FIRST", raising=False)
+    yield cli
+    if cli.grandchild_pidfile.exists():
+        try:
+            os.kill(int(cli.grandchild_pidfile.read_text()), signal.SIGKILL)
+        except ProcessLookupError:
+            pass
diff --git a/tests/test_litellm/proxy/db/test_check_migration.py b/tests/test_litellm/proxy/db/test_check_migration.py
index 9e2f6a1089c..74a6841cb23 100644
--- a/tests/test_litellm/proxy/db/test_check_migration.py
+++ b/tests/test_litellm/proxy/db/test_check_migration.py
@@ -37,3 +37,35 @@ def test_check_migration_out_of_sync(mocker):
     check_migration.verbose_logger.exception.assert_called_once()
     actual_message = check_migration.verbose_logger.exception.call_args[0][0]
     assert "prisma schema out of sync with db" in actual_message
+
+
+@pytest.mark.timeout(30)
+def test_migrate_diff_stops_at_its_budget_and_takes_its_process_tree_with_it(fake_prisma_cli, monkeypatch):
+    """
+    `prisma migrate diff` ran unbounded, so a database that never answers hung boot
+    before uvicorn ever started, and interrupting the proxy orphaned the schema engine.
+    """
+    from litellm.proxy.db.check_migration import check_prisma_schema_diff_helper
+
+    monkeypatch.setenv("FAKE_PRISMA_HANG_FIRST", "1")
+
+    assert check_prisma_schema_diff_helper("postgresql://u:p@localhost:9/x") == (False, [])
+    assert fake_prisma_cli.calls == [
+        ["migrate", "diff", "--from-url", "postgresql://u:p@localhost:9/x",
+         "--to-schema-datamodel", "./schema.prisma", "--script"]
+    ]
+    assert fake_prisma_cli.grandchild_is_gone(within_seconds=5)
+
+
+def test_migrate_diff_without_the_prisma_runner_skips_instead_of_crashing_boot(monkeypatch):
+    """
+    Boot calls this helper directly, so an ImportError here takes the proxy down before
+    uvicorn starts. An install without the runner must lose the diagnostic, not the proxy.
+    """
+    import sys
+
+    from litellm.proxy.db.check_migration import check_prisma_schema_diff_helper
+
+    monkeypatch.setitem(sys.modules, "litellm_proxy_extras.prisma_toolchain", None)
+
+    assert check_prisma_schema_diff_helper("postgresql://u:p@localhost:9/x") == (False, [])
diff --git a/tests/test_litellm/proxy/db/test_prisma_client.py b/tests/test_litellm/proxy/db/test_prisma_client.py
index f0983d6bf62..7f43e483557 100644
--- a/tests/test_litellm/proxy/db/test_prisma_client.py
+++ b/tests/test_litellm/proxy/db/test_prisma_client.py
@@ -10,7 +10,7 @@ from fastapi.testclient import TestClient
 
 
 
-from litellm.proxy.db.prisma_client import PrismaWrapper, should_update_prisma_schema
+from litellm.proxy.db.prisma_client import PrismaManager, PrismaWrapper, should_update_prisma_schema
 
 
 @pytest.fixture(autouse=True)
@@ -193,7 +193,10 @@ async def test_recreate_prisma_client_recovers_from_disconnected_client(
     mock_new_prisma.connect.assert_awaited_once()
 
 
-def test_db_push_applies_replica_identity_full_when_requested(monkeypatch):
+DB_PUSH_ARGV = ["db", "push", "--accept-data-loss", "--skip-generate"]
+
+
+def test_db_push_applies_replica_identity_full_when_requested(monkeypatch, fake_prisma_cli, unset_database_url):
     """`prisma db push` bypasses litellm-proxy-extras, so it needs its own call
     into the opt-in REPLICA IDENTITY FULL step."""
     from litellm.proxy.db.prisma_client import PrismaManager
@@ -208,14 +211,13 @@ def test_db_push_applies_replica_identity_full_when_requested(monkeypatch):
         staticmethod(lambda: applied.append(True)),
     )
 
-    with patch("litellm.proxy.db.prisma_client.subprocess.run") as mock_run:
-        assert PrismaManager.setup_database(use_migrate=False) is True
+    assert PrismaManager.setup_database(use_migrate=False) is True
 
-    assert mock_run.call_args[0][0][:3] == ["prisma", "db", "push"]
+    assert fake_prisma_cli.calls == [DB_PUSH_ARGV]
     assert applied == [True]
 
 
-def test_db_push_is_rejected_when_spend_logs_is_partitioned(monkeypatch):
+def test_db_push_is_rejected_when_spend_logs_is_partitioned(monkeypatch, fake_prisma_cli, unset_database_url):
     """A doc-partitioned LiteLLM_SpendLogs makes `prisma db push` rewrite the
     primary key back to ("request_id"), which Postgres rejects; the guard must
     fail fast with guidance instead of running the push."""
@@ -228,29 +230,23 @@ def test_db_push_is_rejected_when_spend_logs_is_partitioned(monkeypatch):
     monkeypatch.setattr(
         ProxyExtrasDBManager, "spend_logs_is_partitioned", staticmethod(lambda: True)
     )
-    with patch(  # test-quality-ok: subprocess.run is the external prisma CLI boundary, asserted never reached
-        "litellm.proxy.db.prisma_client.subprocess.run"
-    ) as mock_run:
-        with pytest.raises(RuntimeError) as err:
-            PrismaManager.setup_database(use_migrate=False)
+    with pytest.raises(RuntimeError) as err:
+        PrismaManager.setup_database(use_migrate=False)
 
     assert str(err.value) == PARTITIONED_SPEND_LOGS_PUSH_ERROR
-    mock_run.assert_not_called()
+    assert fake_prisma_cli.calls == []
 
 
-def test_db_push_proceeds_when_spend_logs_is_not_partitioned(monkeypatch):
+def test_db_push_proceeds_when_spend_logs_is_not_partitioned(monkeypatch, fake_prisma_cli, unset_database_url):
     from litellm.proxy.db.prisma_client import PrismaManager
     from litellm_proxy_extras.utils import ProxyExtrasDBManager
 
     monkeypatch.setattr(
         ProxyExtrasDBManager, "spend_logs_is_partitioned", staticmethod(lambda: False)
     )
-    with patch(  # test-quality-ok: subprocess.run is the external prisma CLI boundary, not SDK logic
-        "litellm.proxy.db.prisma_client.subprocess.run"
-    ) as mock_run:
-        assert PrismaManager.setup_database(use_migrate=False) is True
+    assert PrismaManager.setup_database(use_migrate=False) is True
 
-    assert mock_run.call_args[0][0][:3] == ["prisma", "db", "push"]
+    assert fake_prisma_cli.calls == [DB_PUSH_ARGV]
 
 
 def _entra_jwt(expires_in_seconds: int) -> str:
@@ -377,3 +373,30 @@ def test_minting_without_the_database_env_vars_names_them(azure_env, monkeypatch
 
     with pytest.raises(RuntimeError, match="DATABASE_HOST"):
         wrapper.get_rds_iam_token()
+
+
+@pytest.mark.timeout(45)
+def test_db_push_timeout_takes_its_process_tree_with_it(fake_prisma_cli, unset_database_url, monkeypatch):
+    """
+    A timed-out `db push` used to leave Node and the schema engine writing the schema,
+    so the next attempt pushed into a database the abandoned one was still mutating.
+    """
+    monkeypatch.delenv("LITELLM_SET_REPLICA_IDENTITY_FULL", raising=False)
+    monkeypatch.setenv("FAKE_PRISMA_HANG_FIRST", "1")
+
+    assert PrismaManager.setup_database(use_migrate=False) is True
+    assert fake_prisma_cli.calls == [DB_PUSH_ARGV, DB_PUSH_ARGV]
+    assert fake_prisma_cli.grandchild_is_gone(within_seconds=5)
+
+
+def test_db_push_without_the_prisma_runner_fails_the_migration_instead_of_crashing_boot(
+    fake_prisma_cli, unset_database_url, monkeypatch
+):
+    """
+    An ImportError out of setup_database escapes the caller's RuntimeError handler and
+    kills boot, bypassing the operator's enforce_prisma_migration_check choice.
+    """
+    monkeypatch.setitem(sys.modules, "litellm_proxy_extras.prisma_toolchain", None)
+
+    assert PrismaManager.setup_database(use_migrate=False) is False
+    assert fake_prisma_cli.calls == []

From a85c3152ca59b5b56d1168e2a384e29a4fb0b861 Mon Sep 17 00:00:00 2001
From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 8 Sep 2026 10:08:24 -0700
Subject: [PATCH 60/85] fix(proxy): load db credentials in the model reconcile
 so a worker never serves a model before its credential (#39876)

* fix(proxy): load db credentials inside the model reconcile so a worker never serves a model before its credential

* fix(proxy): load db credentials in the model read-through so a request miss never adds a model before its credential

* fix(proxy): read credentials from the writer db before the router update and look a credential up once

* test(proxy): assert the credential is loaded when db models reach the router instead of the call order
---
 .../litellm_core_utils/credential_accessor.py |  15 ++-
 .../common_utils/registry_read_through.py     |   1 +
 litellm/proxy/proxy_server.py                 |  31 +----
 litellm/utils.py                              |  18 ++-
 .../common_utils/test_config_sync_pubsub.py   |   7 +-
 .../test_registry_read_through.py             |  45 +++++++
 .../test_scheduled_job_stagger.py             |   2 +-
 .../proxy/proxy_server/test_proxy_config.py   | 123 ++++++++++++++++++
 tests/test_litellm/proxy/test_proxy_server.py |  68 +++++-----
 tests/test_litellm/test_utils.py              |  44 +++++++
 10 files changed, 278 insertions(+), 76 deletions(-)

diff --git a/litellm/litellm_core_utils/credential_accessor.py b/litellm/litellm_core_utils/credential_accessor.py
index 7071750b970..7b4e8240b69 100644
--- a/litellm/litellm_core_utils/credential_accessor.py
+++ b/litellm/litellm_core_utils/credential_accessor.py
@@ -7,16 +7,19 @@ from litellm.types.utils import CredentialItem
 
 
 class CredentialAccessor:
+    @staticmethod
+    def find_credential(credential_name: str) -> CredentialItem | None:
+        return next(
+            (credential for credential in litellm.credential_list if credential.credential_name == credential_name),
+            None,
+        )
+
     @staticmethod
     def get_credential_values(credential_name: str) -> dict:
         """Safe accessor for credentials."""
 
-        if not litellm.credential_list:
-            return {}
-        for credential in litellm.credential_list:
-            if credential.credential_name == credential_name:
-                return credential.credential_values.copy()
-        return {}
+        credential: Final = CredentialAccessor.find_credential(credential_name)
+        return {} if credential is None else credential.credential_values.copy()
 
     @staticmethod
     def upsert_credentials(credentials: list[CredentialItem]):
diff --git a/litellm/proxy/common_utils/registry_read_through.py b/litellm/proxy/common_utils/registry_read_through.py
index 460b348e188..63da5d15207 100644
--- a/litellm/proxy/common_utils/registry_read_through.py
+++ b/litellm/proxy/common_utils/registry_read_through.py
@@ -125,6 +125,7 @@ async def _resync_model_deployments(model_name: str) -> bool:
         )
         return proxy_server.llm_router is not None
     async with proxy_server.MODEL_RECONCILE_LOCK:
+        await proxy_server.proxy_config.get_credentials(prisma_client=prisma_client)
         proxy_server.proxy_config._add_deployment(db_models=rows)
         proxy_server.llm_model_list = router.get_model_list()
     return True
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 32b6b841af7..1d6baf5fded 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -7109,11 +7109,10 @@ class ProxyConfig:
                 ],
             )
 
-            # Only load models from DB if "models" is in supported_db_objects (or if supported_db_objects is not set)
-            if self._should_load_db_object(object_type="models"):
-                new_models: Final = await self._get_models_from_db(prisma_client=prisma_client)
-
-                # update llm router
+            load_models: Final = self._should_load_db_object(object_type="models")
+            new_models: Final = await self._get_models_from_db(prisma_client=prisma_client) if load_models else None
+            await self.get_credentials(prisma_client=prisma_client)
+            if load_models:
                 still_desired_ids = await self._update_llm_router(
                     new_models=new_models, proxy_logging_obj=proxy_logging_obj
                 )
@@ -7153,12 +7152,9 @@ class ProxyConfig:
         async def _resync_config_from_db() -> None:
             await self.add_deployment(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj)
 
-        async def _resync_credentials_from_db() -> None:
-            await self.get_credentials(prisma_client=prisma_client)
-
         subscriber: Final = ConfigSyncSubscriber(
             redis_cache=redis_cache,
-            resync_callbacks=(_resync_config_from_db, _resync_credentials_from_db),
+            resync_callbacks=(_resync_config_from_db,),
         )
         self.config_sync_subscriber = subscriber
         subscriber.start()
@@ -8013,7 +8009,7 @@ class ProxyConfig:
 
     async def get_credentials(self, prisma_client: PrismaClient):
         try:
-            credentials = await CredentialsRepository(prisma_client).find_all()
+            credentials = await CredentialsRepository(WriterPinnedClient(prisma_client.db)).find_all()
             credentials = [self.decrypt_credentials(cred) for cred in credentials]
             await self.delete_credentials(credentials)  # delete credentials that are not in the all-up list
             CredentialAccessor.upsert_credentials(credentials)  # upsert credentials that are in the all-up list
@@ -9597,19 +9593,6 @@ class ProxyStartupEvent:
         )
 
         if store_model_in_db is True:
-            ### GET STORED CREDENTIALS ###
-            scheduler.add_job(
-                proxy_config.get_credentials,
-                "interval",
-                seconds=config_reload_interval_seconds,
-                # REMOVED jitter parameter - major cause of memory leak
-                args=[prisma_client],
-                id="get_credentials_job",
-                replace_existing=True,
-                misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME,
-            )
-            await proxy_config.get_credentials(prisma_client=prisma_client)
-
             # MEMORY LEAK FIX: Increase interval from 10s to 30s minimum
             # Frequent polling was causing excessive memory allocations
             scheduler.add_job(
@@ -9623,7 +9606,7 @@ class ProxyStartupEvent:
                 misfire_grace_time=APSCHEDULER_MISFIRE_GRACE_TIME,
             )
 
-            # this will load all existing models on proxy startup
+            # this will load all existing credentials and models on proxy startup
             await proxy_config.add_deployment(prisma_client=prisma_client, proxy_logging_obj=proxy_logging_obj)
 
             proxy_config.start_config_sync_subscriber(
diff --git a/litellm/utils.py b/litellm/utils.py
index d0e11bc9551..141ec323776 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -672,11 +672,19 @@ def load_credentials_from_list(kwargs: dict):
     CredentialAccessor: Final = getattr(sys.modules[__name__], "CredentialAccessor")
 
     credential_name: Final = kwargs.get("litellm_credential_name")
-    if credential_name and litellm.credential_list:
-        credential_accessor: Final[Mapping[str, object]] = CredentialAccessor.get_credential_values(credential_name)
-        for key, value in credential_accessor.items():
-            if key not in kwargs:
-                kwargs[key] = value
+    if not credential_name:
+        return
+    credential: Final = CredentialAccessor.find_credential(credential_name)
+    if credential is None:
+        verbose_logger.warning(
+            "litellm_credential_name=%s matched none of the %d loaded credentials; the request runs without it",
+            credential_name,
+            len(litellm.credential_list),
+        )
+        return
+    for key, value in credential.credential_values.items():
+        if key not in kwargs:
+            kwargs[key] = value
 
 
 def get_dynamic_callbacks(
diff --git a/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py b/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py
index 3bb3f75b9b8..2a0ebf9492a 100644
--- a/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py
+++ b/tests/test_litellm/proxy/common_utils/test_config_sync_pubsub.py
@@ -824,7 +824,7 @@ class _StopFailingSubscriber(ConfigSyncSubscriber):
         raise RuntimeError("stop failed")
 
 
-async def test_proxy_config_subscriber_resyncs_deployments_and_credentials() -> None:
+async def test_proxy_config_subscriber_resyncs_deployments_only() -> None:
     from litellm.proxy.proxy_server import ProxyConfig
 
     cache = _FakeRedisCache(_ScriptedPubSubRedisClient([_QueuePubSub()]))
@@ -852,10 +852,7 @@ async def test_proxy_config_subscriber_resyncs_deployments_and_credentials() ->
         await callback()
     await config.stop_config_sync_subscriber()
 
-    assert calls == [
-        ("add_deployment", prisma_client, proxy_logging_obj),
-        ("get_credentials", prisma_client, None),
-    ]
+    assert calls == [("add_deployment", prisma_client, proxy_logging_obj)]
     assert config.config_sync_subscriber is None
     assert subscriber._task is None
 
diff --git a/tests/test_litellm/proxy/common_utils/test_registry_read_through.py b/tests/test_litellm/proxy/common_utils/test_registry_read_through.py
index f0fbdea4e85..6f7c20166c5 100644
--- a/tests/test_litellm/proxy/common_utils/test_registry_read_through.py
+++ b/tests/test_litellm/proxy/common_utils/test_registry_read_through.py
@@ -393,6 +393,51 @@ async def test_resync_model_deployments_mutates_router_under_model_reconcile_loc
     assert not proxy_server.MODEL_RECONCILE_LOCK.locked()
 
 
+@pytest.mark.asyncio
+async def test_resync_model_deployments_loads_db_credentials_before_reconciling_models(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    from unittest.mock import AsyncMock, MagicMock
+
+    import litellm
+    import litellm.proxy.proxy_server as proxy_server
+    from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
+    from litellm.proxy.common_utils.registry_read_through import _resync_model_deployments
+    from litellm.types.utils import CredentialItem
+
+    rows: Final = [MagicMock()]
+    prisma_client: Final = MagicMock()
+    prisma_client.db.litellm_proxymodeltable.find_many = AsyncMock(return_value=rows)
+    router: Final = MagicMock()
+    router.get_model_list.return_value = []
+    installed: Final = MagicMock()
+
+    async def load_credentials_from_db(prisma_client: object) -> None:
+        CredentialAccessor.upsert_credentials(
+            [
+                CredentialItem(
+                    credential_name="openai-cred",
+                    credential_values={"api_key": "sk-from-db"},
+                    credential_info={},
+                )
+            ]
+        )
+
+    def install_models(db_models: object) -> None:
+        installed(db_models=db_models, credential=CredentialAccessor.get_credential_values("openai-cred"))
+
+    monkeypatch.setattr(litellm, "credential_list", [])
+    monkeypatch.setattr(proxy_server, "prisma_client", prisma_client)
+    monkeypatch.setattr(proxy_server, "store_model_in_db", True)
+    monkeypatch.setattr(proxy_server, "llm_router", router)
+    monkeypatch.setattr(proxy_server, "llm_model_list", None)
+    monkeypatch.setattr(proxy_server.proxy_config, "get_credentials", load_credentials_from_db)
+    monkeypatch.setattr(proxy_server.proxy_config, "_add_deployment", install_models)
+
+    assert await _resync_model_deployments("model-created-on-a-sibling-replica") is True
+    installed.assert_called_once_with(db_models=rows, credential={"api_key": "sk-from-db"})
+
+
 @pytest.mark.asyncio
 async def test_resync_model_deployments_respects_supported_db_objects(monkeypatch):
     from unittest.mock import AsyncMock, MagicMock
diff --git a/tests/test_litellm/proxy/common_utils/test_scheduled_job_stagger.py b/tests/test_litellm/proxy/common_utils/test_scheduled_job_stagger.py
index ca4d62737b6..54e0aa74a25 100644
--- a/tests/test_litellm/proxy/common_utils/test_scheduled_job_stagger.py
+++ b/tests/test_litellm/proxy/common_utils/test_scheduled_job_stagger.py
@@ -23,7 +23,7 @@ from litellm.proxy.common_utils.scheduled_job_stagger import (
 )
 
 OPERATOR_CRON_JOB_ID = "spend_log_cleanup_job"
-SHARED_INTERVAL_JOB_IDS = ("periodic_reload_job", "get_credentials_job", "add_deployment_job")
+SHARED_INTERVAL_JOB_IDS = ("periodic_reload_job", "add_deployment_job")
 
 
 async def _noop() -> None: ...
diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py
index 770cec1834e..ae4ec4086bb 100644
--- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py
+++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py
@@ -2939,6 +2939,129 @@ async def test_ProxyConfig_add_deployment_applies_db_router_settings(monkeypatch
     fake_router.update_settings.assert_called_once_with(routing_strategy="latency-based-routing")
 
 
+def _stub_add_deployment_collaborators(
+    monkeypatch: pytest.MonkeyPatch, pc: ProxyConfig, fake_prisma: MagicMock
+) -> None:
+    from litellm.proxy import proxy_server
+
+    fake_router = MagicMock()
+    fake_router.get_model_list = MagicMock(return_value=[])
+
+    async def fake_get_config(*args: object, **kwargs: object) -> dict[str, object]:
+        return {}
+
+    monkeypatch.setattr(litellm, "credential_list", [])
+    monkeypatch.setattr(pc, "get_config", fake_get_config)
+    monkeypatch.setattr(pc, "_init_non_llm_objects_in_db", AsyncMock())
+    monkeypatch.setattr(proxy_server, "prefetch_config_params", AsyncMock())
+    monkeypatch.setattr(proxy_server, "get_config_param", AsyncMock(return_value=None))
+    monkeypatch.setattr(proxy_server, "llm_router", fake_router)
+    monkeypatch.setattr(proxy_server, "master_key", "sk-master")
+    monkeypatch.setattr(proxy_server, "prisma_client", fake_prisma)
+    monkeypatch.setattr(proxy_server, "proxy_config", pc)
+    monkeypatch.delenv("LITELLM_SALT_KEY", raising=False)
+
+
+def _encrypted_credential_row(credential_name: str, api_key: str) -> dict[str, object]:
+    from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper
+
+    return {
+        "credential_name": credential_name,
+        "credential_values": {"api_key": encrypt_value_helper(api_key, new_encryption_key="sk-master")},
+        "credential_info": {"custom_llm_provider": "openai"},
+    }
+
+
+def _fake_prisma_with_encrypted_credential(credential_name: str, api_key: str) -> MagicMock:
+    fake_prisma = MagicMock()
+    fake_prisma.db.litellm_credentialstable.find_many = AsyncMock(
+        return_value=[_encrypted_credential_row(credential_name, api_key)]
+    )
+    return fake_prisma
+
+
+@pytest.mark.asyncio
+async def test_ProxyConfig_add_deployment_loads_db_credentials_before_reconciling_models(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
+    from litellm.proxy import proxy_server
+    from litellm.utils import load_credentials_from_list
+
+    pc = ProxyConfig()
+    fake_prisma = MagicMock()
+    fake_prisma.db.litellm_credentialstable.find_many = AsyncMock(return_value=[])
+    _stub_add_deployment_collaborators(monkeypatch, pc, fake_prisma)
+    monkeypatch.setattr(proxy_server, "general_settings", {})
+    installed = MagicMock()
+
+    async def read_models_while_a_credential_lands(prisma_client: object) -> list[MagicMock]:
+        fake_prisma.db.litellm_credentialstable.find_many.return_value = [
+            _encrypted_credential_row("openai-cred", "sk-from-db")
+        ]
+        return [MagicMock()]
+
+    async def install_models(new_models: object, proxy_logging_obj: object) -> None:
+        installed(credential=CredentialAccessor.get_credential_values("openai-cred"))
+
+    monkeypatch.setattr(pc, "_get_models_from_db", read_models_while_a_credential_lands)
+    monkeypatch.setattr(pc, "_update_llm_router", install_models)
+
+    await pc.add_deployment(prisma_client=fake_prisma, proxy_logging_obj=MagicMock())
+
+    installed.assert_called_once_with(credential={"api_key": "sk-from-db"})
+    assert CredentialAccessor.get_credential_values("openai-cred") == {"api_key": "sk-from-db"}
+    request_kwargs = {"litellm_credential_name": "openai-cred"}
+    load_credentials_from_list(request_kwargs)
+    assert request_kwargs == {"litellm_credential_name": "openai-cred", "api_key": "sk-from-db"}
+
+
+@pytest.mark.asyncio
+async def test_ProxyConfig_add_deployment_loads_db_credentials_even_when_models_are_not_db_objects(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
+    from litellm.proxy import proxy_server
+
+    pc = ProxyConfig()
+    fake_prisma = _fake_prisma_with_encrypted_credential("openai-cred", "sk-from-db")
+    _stub_add_deployment_collaborators(monkeypatch, pc, fake_prisma)
+    monkeypatch.setattr(proxy_server, "general_settings", {"supported_db_objects": ["mcp"]})
+    models_fetch = AsyncMock(return_value=[])
+    monkeypatch.setattr(pc, "_get_models_from_db", models_fetch)
+
+    await pc.add_deployment(prisma_client=fake_prisma, proxy_logging_obj=MagicMock())
+
+    models_fetch.assert_not_awaited()
+    assert CredentialAccessor.get_credential_values("openai-cred") == {"api_key": "sk-from-db"}
+
+
+@pytest.mark.asyncio
+async def test_ProxyConfig_get_credentials_reads_from_writer_not_replica(monkeypatch: pytest.MonkeyPatch) -> None:
+    from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
+    from litellm.proxy.db.prisma_client import PrismaWrapper
+    from litellm.proxy.db.routing_prisma_wrapper import RoutingPrismaWrapper
+
+    pc = ProxyConfig()
+    writer_inner = MagicMock(name="writer_prisma")
+    reader_inner = MagicMock(name="reader_prisma")
+    writer_inner.litellm_credentialstable.find_many = AsyncMock(
+        return_value=[_encrypted_credential_row("openai-cred", "sk-from-writer")]
+    )
+    reader_inner.litellm_credentialstable.find_many = AsyncMock(return_value=[])
+    fake_prisma = MagicMock()
+    fake_prisma.db = RoutingPrismaWrapper(
+        writer=PrismaWrapper(original_prisma=writer_inner, iam_token_db_auth=False),
+        reader=PrismaWrapper(original_prisma=reader_inner, iam_token_db_auth=False),
+    )
+    _stub_add_deployment_collaborators(monkeypatch, pc, fake_prisma)
+
+    await pc.get_credentials(prisma_client=fake_prisma)
+
+    assert CredentialAccessor.get_credential_values("openai-cred") == {"api_key": "sk-from-writer"}
+    reader_inner.litellm_credentialstable.find_many.assert_not_awaited()
+
+
 # ---------------------------------------------------------------------------
 # ProxyConfig._add_general_settings_from_db_config
 # ---------------------------------------------------------------------------
diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py
index b7bb58378d4..937e4f15741 100644
--- a/tests/test_litellm/proxy/test_proxy_server.py
+++ b/tests/test_litellm/proxy/test_proxy_server.py
@@ -822,16 +822,16 @@ def _mock_scheduled_proxy_config() -> MagicMock:
 
 
 @pytest.mark.asyncio
-async def test_initialize_scheduled_jobs_credentials(monkeypatch):
-    """
-    Test that get_credentials is only called when store_model_in_db is True
-    """
+async def test_initialize_scheduled_jobs_loads_credentials_only_through_add_deployment(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
     monkeypatch.delenv("DISABLE_PRISMA_SCHEMA_UPDATE", raising=False)
     monkeypatch.delenv("STORE_MODEL_IN_DB", raising=False)
+    from apscheduler.schedulers.asyncio import AsyncIOScheduler
+
     from litellm.proxy.proxy_server import ProxyStartupEvent
     from litellm.proxy.utils import ProxyLogging
 
-    # Mock dependencies
     mock_prisma_client = MagicMock()
     mock_proxy_logging = MagicMock(spec=ProxyLogging)
     mock_proxy_logging.slack_alerting_instance = MagicMock()
@@ -841,25 +841,6 @@ async def test_initialize_scheduled_jobs_credentials(monkeypatch):
     with (
         patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config),
         patch("litellm.proxy.proxy_server.store_model_in_db", False),
-    ):  # set store_model_in_db to False
-        # Test when store_model_in_db is False
-        await ProxyStartupEvent.initialize_scheduled_background_jobs(
-            general_settings={},
-            prisma_client=mock_prisma_client,
-            proxy_budget_rescheduler_min_time=1,
-            proxy_budget_rescheduler_max_time=2,
-            proxy_batch_write_at=5,
-            proxy_logging_obj=mock_proxy_logging,
-        )
-
-        # Verify get_credentials was not called
-        mock_proxy_config.get_credentials.assert_not_called()
-
-    # Now test with store_model_in_db = True
-    with (
-        patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config),
-        patch("litellm.proxy.proxy_server.store_model_in_db", True),
-        patch("litellm.proxy.proxy_server.get_secret_bool", return_value=True),
     ):
         await ProxyStartupEvent.initialize_scheduled_background_jobs(
             general_settings={},
@@ -870,12 +851,31 @@ async def test_initialize_scheduled_jobs_credentials(monkeypatch):
             proxy_logging_obj=mock_proxy_logging,
         )
 
-        # Verify get_credentials was called both directly and scheduled
-        assert mock_proxy_config.get_credentials.call_count == 1  # Direct call
+        mock_proxy_config.get_credentials.assert_not_called()
+        mock_proxy_config.add_deployment.assert_not_called()
 
-        # Verify a scheduled job was added for get_credentials
-        mock_scheduler_calls = [call[0] for call in mock_proxy_config.get_credentials.mock_calls]
-        assert len(mock_scheduler_calls) > 0
+    scheduler = AsyncIOScheduler()
+    try:
+        with (
+            patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config),
+            patch("litellm.proxy.proxy_server.store_model_in_db", True),
+            patch("litellm.proxy.proxy_server.AsyncIOScheduler", return_value=scheduler),
+        ):
+            await ProxyStartupEvent.initialize_scheduled_background_jobs(
+                general_settings={},
+                prisma_client=mock_prisma_client,
+                proxy_budget_rescheduler_min_time=1,
+                proxy_budget_rescheduler_max_time=2,
+                proxy_batch_write_at=5,
+                proxy_logging_obj=mock_proxy_logging,
+            )
+
+        assert scheduler.get_job("get_credentials_job") is None
+        assert scheduler.get_job("add_deployment_job") is not None
+        mock_proxy_config.get_credentials.assert_not_called()
+        assert mock_proxy_config.add_deployment.call_count == 1
+    finally:
+        scheduler.shutdown(wait=False)
 
 
 @pytest.mark.asyncio
@@ -924,7 +924,7 @@ async def test_periodic_reload_job_scheduled_without_store_model_in_db(monkeypat
 @pytest.mark.asyncio
 async def test_initialize_scheduled_jobs_uses_configured_config_reload_interval(monkeypatch):
     """
-    The DB config-reload jobs (add_deployment, get_credentials) that keep multi-pod
+    The DB config-reload job (add_deployment) that keeps multi-pod
     deployments in sync must be scheduled at the configured
     proxy_config_reload_interval_seconds, not a hardcoded value.
     """
@@ -967,7 +967,7 @@ async def test_initialize_scheduled_jobs_uses_configured_config_reload_interval(
         if "id" in job_call.kwargs
     }
     assert scheduled_seconds["add_deployment_job"] == configured_interval
-    assert scheduled_seconds["get_credentials_job"] == configured_interval
+    assert "get_credentials_job" not in scheduled_seconds
 
 
 @pytest.mark.asyncio
@@ -1011,7 +1011,7 @@ async def test_initialize_scheduled_jobs_rejects_non_positive_config_reload_inte
         if "id" in job_call.kwargs
     }
     assert scheduled_seconds["add_deployment_job"] == 30
-    assert scheduled_seconds["get_credentials_job"] == 30
+    assert "get_credentials_job" not in scheduled_seconds
 
 
 @pytest.mark.asyncio
@@ -7446,10 +7446,8 @@ async def test_store_model_in_db_db_override_when_config_false():
         # store_model_in_db should now be True (overridden by DB)
         assert ps.store_model_in_db is True
 
-        # add_deployment and get_credentials should have been called
-        # since store_model_in_db is now True
         assert mock_proxy_config.add_deployment.call_count == 1
-        assert mock_proxy_config.get_credentials.call_count == 1
+        mock_proxy_config.get_credentials.assert_not_called()
 
 
 @pytest.mark.asyncio
diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py
index 8a56a84ade7..4ce68b71079 100644
--- a/tests/test_litellm/test_utils.py
+++ b/tests/test_litellm/test_utils.py
@@ -6092,3 +6092,47 @@ class TestFinalOptionalParamsLineRedaction:
 
         assert "'max_tokens': 17" in printed
         assert "'temperature': 0.25" in printed
+
+
+def _credential_warnings(caplog: pytest.LogCaptureFixture) -> list[str]:
+    return [record.getMessage() for record in caplog.records if "litellm_credential_name=" in record.getMessage()]
+
+
+def test_load_credentials_from_list_warns_when_the_named_credential_is_not_loaded(
+    monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+) -> None:
+    from litellm.utils import load_credentials_from_list
+
+    monkeypatch.setattr(litellm, "credential_list", [])
+    request_kwargs = {"litellm_credential_name": "openai-cred", "model": "openai/gpt-5.4-mini"}
+    with caplog.at_level(logging.WARNING, logger=verbose_logger.name):
+        load_credentials_from_list(request_kwargs)
+
+    assert request_kwargs == {"litellm_credential_name": "openai-cred", "model": "openai/gpt-5.4-mini"}
+    assert _credential_warnings(caplog) == [
+        "litellm_credential_name=openai-cred matched none of the 0 loaded credentials; the request runs without it"
+    ]
+
+
+def test_load_credentials_from_list_fills_kwargs_from_the_loaded_credential_without_warning(
+    monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+) -> None:
+    from litellm.types.utils import CredentialItem
+    from litellm.utils import load_credentials_from_list
+
+    loaded = CredentialItem(
+        credential_name="openai-cred",
+        credential_values={"api_key": "sk-from-db", "api_base": "https://credential.example"},
+        credential_info={},
+    )
+    monkeypatch.setattr(litellm, "credential_list", [loaded])
+    request_kwargs = {"litellm_credential_name": "openai-cred", "api_base": "https://request.example"}
+    with caplog.at_level(logging.WARNING, logger=verbose_logger.name):
+        load_credentials_from_list(request_kwargs)
+
+    assert request_kwargs == {
+        "litellm_credential_name": "openai-cred",
+        "api_base": "https://request.example",
+        "api_key": "sk-from-db",
+    }
+    assert _credential_warnings(caplog) == []

From f769aa4675de225a04fdc055c092318d94612502 Mon Sep 17 00:00:00 2001
From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 8 Sep 2026 10:11:20 -0700
Subject: [PATCH 61/85] fix(router): give cooldowns their own cache so siblings
 see a bench in ~1s (#40025)

Cooldown entries rode the router-wide DualCache, which re-reads a key that is
missing from memory at most once every 10s. A deployment benched on one replica
therefore kept taking traffic on its siblings for up to 10 seconds, and the same
shared in-memory tier could evict a live cooldown once 200 unrelated router keys
crowded it out, which sent even the benching replica back to the dead deployment.

CooldownCache now owns a DualCache over the router's Redis with a 1s read
interval and an in-memory tier that only holds cooldown keys. Redis is attached
lazily because the router builds the cooldown cache before it wires Redis up.
---
 litellm/constants.py                          |   3 +
 litellm/router_utils/cooldown_cache.py        |  48 +++--
 .../test_router_cooldown_per_deployment.py    |  16 +-
 .../router_utils/test_cooldown_cache.py       | 184 ++++++++++++++++--
 4 files changed, 212 insertions(+), 39 deletions(-)

diff --git a/litellm/constants.py b/litellm/constants.py
index defc9337e9b..8ef9523a60a 100644
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -73,6 +73,9 @@ DEFAULT_MAX_TOKENS: Final = int(os.getenv("DEFAULT_MAX_TOKENS", 4096))
 DEFAULT_ALLOWED_FAILS: Final = int(os.getenv("DEFAULT_ALLOWED_FAILS", 3))
 DEFAULT_REDIS_SYNC_INTERVAL: Final = int(os.getenv("DEFAULT_REDIS_SYNC_INTERVAL", 1))
 DEFAULT_COOLDOWN_TIME_SECONDS: Final = int(os.getenv("DEFAULT_COOLDOWN_TIME_SECONDS", 5))
+DEFAULT_COOLDOWN_REDIS_READ_INTERVAL_SECONDS: Final = float(
+    os.getenv("DEFAULT_COOLDOWN_REDIS_READ_INTERVAL_SECONDS", "1")
+)
 DEFAULT_REPLICATE_POLLING_RETRIES: Final = int(os.getenv("DEFAULT_REPLICATE_POLLING_RETRIES", 5))
 DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = int(os.getenv("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1))
 DEFAULT_IMAGE_TOKEN_COUNT: Final = int(os.getenv("DEFAULT_IMAGE_TOKEN_COUNT", 250))
diff --git a/litellm/router_utils/cooldown_cache.py b/litellm/router_utils/cooldown_cache.py
index 9e7f457f631..ef29f7d8fd3 100644
--- a/litellm/router_utils/cooldown_cache.py
+++ b/litellm/router_utils/cooldown_cache.py
@@ -12,6 +12,7 @@ from typing_extensions import TypedDict
 from litellm import verbose_logger
 from litellm.caching.caching import DualCache
 from litellm.caching.in_memory_cache import InMemoryCache
+from litellm.constants import DEFAULT_COOLDOWN_REDIS_READ_INTERVAL_SECONDS
 from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
 
 if TYPE_CHECKING:
@@ -36,10 +37,19 @@ _MAX_CORRECTED_IN_MEMORY_TTL_SECONDS: Final = 60.0
 
 
 class CooldownCache:
-    def __init__(self, cache: DualCache, default_cooldown_time: float):
+    def __init__(
+        self,
+        cache: DualCache,
+        default_cooldown_time: float,
+        redis_read_interval_seconds: float = DEFAULT_COOLDOWN_REDIS_READ_INTERVAL_SECONDS,
+    ):
         self.cache = cache
         self.default_cooldown_time = default_cooldown_time
         self.in_memory_cache = InMemoryCache()
+        self._cooldown_store = DualCache(
+            in_memory_cache=self.in_memory_cache,
+            default_redis_batch_cache_expiry=redis_read_interval_seconds,
+        )
         # Initialize the masker with custom settings for exception strings
         self.exception_masker = SensitiveDataMasker(
             visible_prefix=50,  # Show first 50 characters
@@ -48,6 +58,21 @@ class CooldownCache:
             mask_short_values=False,  # Truncate long messages only; keep short ones readable
         )
 
+    @property
+    def cooldown_store(self) -> DualCache:
+        """
+        The cache cooldown entries live in, with the router's Redis attached on first use.
+
+        It is kept separate from the router-wide cache so that a key missing from memory is
+        re-read from Redis every `redis_read_interval_seconds` rather than on the router
+        cache's much longer batch interval, which is what lets a sibling replica see a
+        cooldown another replica wrote, and so that unrelated router keys cannot evict a
+        cooldown from the in-memory tier before it expires. Redis is attached lazily because
+        the router builds its cooldown cache before it wires up the shared Redis client.
+        """
+        self._cooldown_store.attach_redis_cache(self.cache.redis_cache)
+        return self._cooldown_store
+
     def _common_add_cooldown_logic(
         self, model_id: str, original_exception, exception_status, cooldown_time: float
     ) -> tuple[str, CooldownCacheValue]:
@@ -93,7 +118,7 @@ class CooldownCache:
             )
 
             # Set the cache with a TTL equal to the cooldown time
-            self.cache.set_cache(
+            self.cooldown_store.set_cache(
                 value=cooldown_data,
                 key=cooldown_key,
                 ttl=_cooldown_time,
@@ -122,13 +147,13 @@ class CooldownCache:
         cooldown_cache_value: Final = CooldownCacheValue(**result)  # pyright: ignore[reportUnknownArgumentType] - result comes from an untyped cache read, not from our own code
         remaining: Final = (cooldown_cache_value["timestamp"] + cooldown_cache_value["cooldown_time"]) - current_time
         if remaining <= 0:
-            self.cache.in_memory_cache.delete_cache(key)
+            self.in_memory_cache.delete_cache(key)
             return None
-        current_expiry: Final = self.cache.in_memory_cache.ttl_dict.get(key)
+        current_expiry: Final = self.in_memory_cache.ttl_dict.get(key)
         if current_expiry is not None and current_expiry > current_time + remaining + 5:
             corrected_ttl: Final = min(remaining, _MAX_CORRECTED_IN_MEMORY_TTL_SECONDS)
-            self.cache.in_memory_cache.delete_cache(key)
-            self.cache.in_memory_cache.set_cache(key, result, ttl=corrected_ttl)
+            self.in_memory_cache.delete_cache(key)
+            self.in_memory_cache.set_cache(key, result, ttl=corrected_ttl)
         return cooldown_cache_value
 
     async def async_get_active_cooldowns(
@@ -137,12 +162,7 @@ class CooldownCache:
         # Generate the keys for the deployments
         keys: Final = [CooldownCache.get_cooldown_cache_key(model_id) for model_id in model_ids]
 
-        # Retrieve the values for the keys using mget
-        ## more likely to be none if no models ratelimited. So just check redis every 1s
-        ## each redis call adds ~100ms latency.
-
-        ## check in memory cache first
-        results: Final = await self.cache.async_batch_get_cache(keys=keys, parent_otel_span=parent_otel_span)
+        results: Final = await self.cooldown_store.async_batch_get_cache(keys=keys, parent_otel_span=parent_otel_span)
         active_cooldowns: Final[list[tuple[str, CooldownCacheValue]]] = []
 
         if results is None or all(v is None for v in results):
@@ -164,7 +184,7 @@ class CooldownCache:
         # Generate the keys for the deployments
         keys: Final = [CooldownCache.get_cooldown_cache_key(model_id) for model_id in model_ids]
         # Retrieve the values for the keys using mget
-        results: Final = self.cache.batch_get_cache(keys=keys, parent_otel_span=parent_otel_span) or []
+        results: Final = self.cooldown_store.batch_get_cache(keys=keys, parent_otel_span=parent_otel_span) or []
 
         active_cooldowns: Final = []
         current_time: Final = time.time()
@@ -184,7 +204,7 @@ class CooldownCache:
         keys: Final = [f"deployment:{model_id}:cooldown" for model_id in model_ids]
 
         # Retrieve the values for the keys using mget
-        results: Final = self.cache.batch_get_cache(keys=keys, parent_otel_span=parent_otel_span) or []
+        results: Final = self.cooldown_store.batch_get_cache(keys=keys, parent_otel_span=parent_otel_span) or []
 
         min_cooldown_time: float | None = None
         # Process the results
diff --git a/tests/router_unit_tests/test_router_cooldown_per_deployment.py b/tests/router_unit_tests/test_router_cooldown_per_deployment.py
index b8ae8a8c013..228dfed38b9 100644
--- a/tests/router_unit_tests/test_router_cooldown_per_deployment.py
+++ b/tests/router_unit_tests/test_router_cooldown_per_deployment.py
@@ -246,12 +246,12 @@ class TestCooldownCacheTTLCorrection:
             "timestamp": time.time() - 120.0,
             "cooldown_time": 60.0,
         }
-        cc.cache.in_memory_cache.set_cache(key, expired_value, ttl=600)
+        cc.in_memory_cache.set_cache(key, expired_value, ttl=600)
 
         active = cc.get_active_cooldowns(model_ids=[model_id], parent_otel_span=None)
 
         assert active == [], "Expired cooldown entry must not appear in active cooldowns"
-        assert cc.cache.in_memory_cache.get_cache(key) is None, "Expired entry must be evicted from in-memory cache"
+        assert cc.in_memory_cache.get_cache(key) is None, "Expired entry must be evicted from in-memory cache"
 
     def test_active_entry_is_returned(self):
         """
@@ -267,7 +267,7 @@ class TestCooldownCacheTTLCorrection:
             "timestamp": time.time(),
             "cooldown_time": 60.0,
         }
-        cc.cache.in_memory_cache.set_cache(key, active_value, ttl=60)
+        cc.in_memory_cache.set_cache(key, active_value, ttl=60)
 
         active = cc.get_active_cooldowns(model_ids=[model_id], parent_otel_span=None)
 
@@ -290,14 +290,14 @@ class TestCooldownCacheTTLCorrection:
             "timestamp": time.time() - (60.0 - remaining),
             "cooldown_time": 60.0,
         }
-        cc.cache.in_memory_cache.set_cache(key, value, ttl=600)
+        cc.in_memory_cache.set_cache(key, value, ttl=600)
 
-        before_expiry = cc.cache.in_memory_cache.ttl_dict.get(key)
+        before_expiry = cc.in_memory_cache.ttl_dict.get(key)
         assert before_expiry is not None
 
         cc.get_active_cooldowns(model_ids=[model_id], parent_otel_span=None)
 
-        after_expiry = cc.cache.in_memory_cache.ttl_dict.get(key)
+        after_expiry = cc.in_memory_cache.ttl_dict.get(key)
         assert after_expiry is not None
         corrected_remaining = after_expiry - time.time()
         assert corrected_remaining <= 60.0, "Corrected TTL must not exceed 60s"
@@ -318,12 +318,12 @@ class TestCooldownCacheTTLCorrection:
             "timestamp": time.time() - 120.0,
             "cooldown_time": 60.0,
         }
-        cc.cache.in_memory_cache.set_cache(key, expired_value, ttl=600)
+        cc.in_memory_cache.set_cache(key, expired_value, ttl=600)
 
         active = await cc.async_get_active_cooldowns(model_ids=[model_id], parent_otel_span=None)
 
         assert active == [], "Expired entry must not appear in async active cooldowns"
-        assert cc.cache.in_memory_cache.get_cache(key) is None
+        assert cc.in_memory_cache.get_cache(key) is None
 
 
 class TestFallbackDeploymentCooldown:
diff --git a/tests/test_litellm/router_utils/test_cooldown_cache.py b/tests/test_litellm/router_utils/test_cooldown_cache.py
index 68e9aeaa4fc..6f90fa8465f 100644
--- a/tests/test_litellm/router_utils/test_cooldown_cache.py
+++ b/tests/test_litellm/router_utils/test_cooldown_cache.py
@@ -268,12 +268,12 @@ class TestCooldownCacheTTLCorrection:
             "timestamp": time.time() - 120.0,
             "cooldown_time": 60.0,
         }
-        cc.cache.in_memory_cache.set_cache(key, expired_value, ttl=600)
+        cc.in_memory_cache.set_cache(key, expired_value, ttl=600)
 
         active = cc.get_active_cooldowns(model_ids=[model_id], parent_otel_span=None)
 
         assert active == [], "Expired cooldown entry must not appear in active cooldowns"
-        assert cc.cache.in_memory_cache.get_cache(key) is None, "Expired entry must be evicted from in-memory cache"
+        assert cc.in_memory_cache.get_cache(key) is None, "Expired entry must be evicted from in-memory cache"
 
     def test_active_entry_is_returned(self):
         """
@@ -289,7 +289,7 @@ class TestCooldownCacheTTLCorrection:
             "timestamp": time.time(),
             "cooldown_time": 60.0,
         }
-        cc.cache.in_memory_cache.set_cache(key, active_value, ttl=60)
+        cc.in_memory_cache.set_cache(key, active_value, ttl=60)
 
         active = cc.get_active_cooldowns(model_ids=[model_id], parent_otel_span=None)
 
@@ -312,14 +312,14 @@ class TestCooldownCacheTTLCorrection:
             "timestamp": time.time() - (60.0 - remaining),
             "cooldown_time": 60.0,
         }
-        cc.cache.in_memory_cache.set_cache(key, value, ttl=600)
+        cc.in_memory_cache.set_cache(key, value, ttl=600)
 
-        before_expiry = cc.cache.in_memory_cache.ttl_dict.get(key)
+        before_expiry = cc.in_memory_cache.ttl_dict.get(key)
         assert before_expiry is not None
 
         cc.get_active_cooldowns(model_ids=[model_id], parent_otel_span=None)
 
-        after_expiry = cc.cache.in_memory_cache.ttl_dict.get(key)
+        after_expiry = cc.in_memory_cache.ttl_dict.get(key)
         assert after_expiry is not None
         corrected_remaining = after_expiry - time.time()
         assert corrected_remaining <= 60.0, "Corrected TTL must not exceed 60s"
@@ -340,12 +340,12 @@ class TestCooldownCacheTTLCorrection:
             "timestamp": time.time() - 120.0,
             "cooldown_time": 60.0,
         }
-        cc.cache.in_memory_cache.set_cache(key, expired_value, ttl=600)
+        cc.in_memory_cache.set_cache(key, expired_value, ttl=600)
 
         active = await cc.async_get_active_cooldowns(model_ids=[model_id], parent_otel_span=None)
 
         assert active == [], "Expired entry must not appear in async active cooldowns"
-        assert cc.cache.in_memory_cache.get_cache(key) is None
+        assert cc.in_memory_cache.get_cache(key) is None
 
     @pytest.mark.asyncio
     async def test_async_active_entry_is_returned(self):
@@ -363,7 +363,7 @@ class TestCooldownCacheTTLCorrection:
             "timestamp": time.time(),
             "cooldown_time": 60.0,
         }
-        cc.cache.in_memory_cache.set_cache(key, active_value, ttl=60)
+        cc.in_memory_cache.set_cache(key, active_value, ttl=60)
 
         active = await cc.async_get_active_cooldowns(model_ids=[model_id], parent_otel_span=None)
 
@@ -389,18 +389,18 @@ class TestCorrectedActiveCooldown:
         cc = self._make_cooldown_cache()
         key = "deployment:expired-dep:cooldown"
         entry = self._entry(timestamp=time.time() - 120.0, cooldown_time=60.0)
-        cc.cache.in_memory_cache.set_cache(key, dict(entry), ttl=600)
+        cc.in_memory_cache.set_cache(key, dict(entry), ttl=600)
 
         result = cc._corrected_active_cooldown(key, dict(entry), current_time=time.time())
 
         assert result is None
-        assert cc.cache.in_memory_cache.get_cache(key) is None
+        assert cc.in_memory_cache.get_cache(key) is None
 
     def test_active_entry_within_window_returns_value(self):
         cc = self._make_cooldown_cache()
         key = "deployment:active-dep:cooldown"
         entry = self._entry(timestamp=time.time(), cooldown_time=60.0)
-        cc.cache.in_memory_cache.set_cache(key, dict(entry), ttl=60)
+        cc.in_memory_cache.set_cache(key, dict(entry), ttl=60)
 
         result = cc._corrected_active_cooldown(key, dict(entry), current_time=time.time())
 
@@ -412,12 +412,12 @@ class TestCorrectedActiveCooldown:
         key = "deployment:backfilled-dep:cooldown"
         remaining = 30.0
         entry = self._entry(timestamp=time.time() - (60.0 - remaining), cooldown_time=60.0)
-        cc.cache.in_memory_cache.set_cache(key, dict(entry), ttl=600)
+        cc.in_memory_cache.set_cache(key, dict(entry), ttl=600)
 
         result = cc._corrected_active_cooldown(key, dict(entry), current_time=time.time())
 
         assert result is not None
-        corrected_expiry = cc.cache.in_memory_cache.ttl_dict.get(key)
+        corrected_expiry = cc.in_memory_cache.ttl_dict.get(key)
         assert corrected_expiry is not None
         assert corrected_expiry - time.time() <= 60.0
 
@@ -425,10 +425,160 @@ class TestCorrectedActiveCooldown:
         cc = self._make_cooldown_cache()
         key = "deployment:normal-dep:cooldown"
         entry = self._entry(timestamp=time.time(), cooldown_time=60.0)
-        cc.cache.in_memory_cache.set_cache(key, dict(entry), ttl=60)
-        original_expiry = cc.cache.in_memory_cache.ttl_dict.get(key)
+        cc.in_memory_cache.set_cache(key, dict(entry), ttl=60)
+        original_expiry = cc.in_memory_cache.ttl_dict.get(key)
 
         cc._corrected_active_cooldown(key, dict(entry), current_time=time.time())
 
-        after_expiry = cc.cache.in_memory_cache.ttl_dict.get(key)
+        after_expiry = cc.in_memory_cache.ttl_dict.get(key)
         assert after_expiry == original_expiry
+
+
+class SharedRedisDouble:
+    """
+    In-process stand-in for RedisCache, shared by several DualCache instances so that
+    tests can model two proxy replicas talking to one Redis.
+    """
+
+    def __init__(self) -> None:
+        self.store: dict = {}  # mutable-ok: stands in for Redis' own mutable keyspace
+
+    def set_cache(self, key, value, **kwargs):
+        self.store[key] = value
+
+    async def async_set_cache(self, key, value, **kwargs):
+        self.store[key] = value
+
+    def batch_get_cache(self, key_list, parent_otel_span=None, **kwargs):
+        return {key: self.store.get(key) for key in key_list}
+
+    async def async_batch_get_cache(self, key_list, parent_otel_span=None, **kwargs):
+        return {key: self.store.get(key) for key in key_list}
+
+
+class TestCooldownPropagationBetweenReplicas:
+    """
+    A cooldown written by one replica has to reach its siblings quickly. The router's own
+    DualCache re-reads a key that is missing from memory only every 10s, so cooldown reads
+    get their own cache with a much shorter Redis read interval.
+    """
+
+    def _make_replica(self, redis: SharedRedisDouble, read_interval: float | None = None) -> CooldownCache:
+        router_cache = DualCache(in_memory_cache=InMemoryCache(), redis_cache=redis)
+        if read_interval is None:
+            return CooldownCache(cache=router_cache, default_cooldown_time=60.0)
+        return CooldownCache(
+            cache=router_cache,
+            default_cooldown_time=60.0,
+            redis_read_interval_seconds=read_interval,
+        )
+
+    @pytest.mark.asyncio
+    async def test_sibling_replica_sees_cooldown_within_configured_read_interval(self):
+        redis = SharedRedisDouble()
+        replica_a = self._make_replica(redis, read_interval=0.25)
+        replica_b = self._make_replica(redis, read_interval=0.25)
+        model_id = "shared-deployment"
+
+        assert await replica_b.async_get_active_cooldowns([model_id], parent_otel_span=None) == []
+
+        replica_a.add_deployment_to_cooldown(
+            model_id=model_id,
+            original_exception=Exception("Internal server error"),
+            exception_status=500,
+            cooldown_time=60.0,
+        )
+
+        time.sleep(0.3)
+
+        active = await replica_b.async_get_active_cooldowns([model_id], parent_otel_span=None)
+        assert [model_id] == [entry[0] for entry in active], (
+            "sibling replica must pick up a cooldown written by another replica within the read interval"
+        )
+
+    @pytest.mark.asyncio
+    async def test_sibling_replica_sees_cooldown_within_default_read_interval(self):
+        redis = SharedRedisDouble()
+        replica_a = self._make_replica(redis)
+        replica_b = self._make_replica(redis)
+        model_id = "default-interval-deployment"
+
+        assert await replica_b.async_get_active_cooldowns([model_id], parent_otel_span=None) == []
+
+        replica_a.add_deployment_to_cooldown(
+            model_id=model_id,
+            original_exception=Exception("Internal server error"),
+            exception_status=500,
+            cooldown_time=60.0,
+        )
+
+        time.sleep(1.2)
+
+        active = await replica_b.async_get_active_cooldowns([model_id], parent_otel_span=None)
+        assert [model_id] == [entry[0] for entry in active], (
+            "the shipped default read interval must let a sibling replica see a cooldown about a second later"
+        )
+
+    def test_sync_read_path_sees_sibling_cooldown_within_read_interval(self):
+        redis = SharedRedisDouble()
+        replica_a = self._make_replica(redis, read_interval=0.25)
+        replica_b = self._make_replica(redis, read_interval=0.25)
+        model_id = "sync-shared-deployment"
+
+        assert replica_b.get_active_cooldowns([model_id], parent_otel_span=None) == []
+
+        replica_a.add_deployment_to_cooldown(
+            model_id=model_id,
+            original_exception=Exception("Internal server error"),
+            exception_status=500,
+            cooldown_time=60.0,
+        )
+
+        time.sleep(0.3)
+
+        active = replica_b.get_active_cooldowns([model_id], parent_otel_span=None)
+        assert [model_id] == [entry[0] for entry in active]
+
+    @pytest.mark.asyncio
+    async def test_redis_attached_after_construction_is_still_used(self):
+        redis = SharedRedisDouble()
+        router_cache = DualCache(in_memory_cache=InMemoryCache())
+        writer = CooldownCache(cache=router_cache, default_cooldown_time=60.0, redis_read_interval_seconds=0.25)
+        router_cache.attach_redis_cache(redis)
+        reader = self._make_replica(redis, read_interval=0.25)
+        model_id = "late-redis-deployment"
+
+        writer.add_deployment_to_cooldown(
+            model_id=model_id,
+            original_exception=Exception("Internal server error"),
+            exception_status=500,
+            cooldown_time=60.0,
+        )
+
+        active = await reader.async_get_active_cooldowns([model_id], parent_otel_span=None)
+        assert [model_id] == [entry[0] for entry in active], (
+            "a router that wires Redis after building its cooldown cache must still publish cooldowns to it"
+        )
+
+
+class TestCooldownSurvivesUnrelatedCacheTraffic:
+    @pytest.mark.asyncio
+    async def test_unrelated_router_cache_writes_do_not_evict_active_cooldown(self):
+        router_cache = DualCache(in_memory_cache=InMemoryCache())
+        cc = CooldownCache(cache=router_cache, default_cooldown_time=60.0)
+        model_id = "busy-router-deployment"
+
+        cc.add_deployment_to_cooldown(
+            model_id=model_id,
+            original_exception=Exception("Internal server error"),
+            exception_status=500,
+            cooldown_time=30.0,
+        )
+
+        for i in range(400):
+            router_cache.set_cache(key=f"unrelated-router-key-{i}", value={"n": i})
+
+        active = await cc.async_get_active_cooldowns([model_id], parent_otel_span=None)
+        assert [model_id] == [entry[0] for entry in active], (
+            "unrelated router cache traffic must not evict a cooldown that is still running"
+        )

From 3448175184d8b68dbcf9ec007d62f85651bb0f41 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 8 Sep 2026 11:44:49 -0700
Subject: [PATCH 62/85] fix(responses bridge): keep mid-conversation system
 messages in input

Only the leading run of system messages (before the first non-system
message) is joined into the Responses `instructions` field. A system
message that arrives after a user, assistant, or tool turn now becomes a
system input item at its position, whether its content is a string or a
list, so a client that re-sends the same reminder as a string on the
next request produces byte-identical input and `instructions` stays
stable. Claude Code >= 2.1.237 appends such reminders after every user
turn, and folding them into `instructions` made Azure treat every
request as a cold prompt (cached_tokens 0 on every request of a
session).

Fixes #40198
---
 .../transformation.py                         |   9 +-
 ...responses_transformation_transformation.py | 140 ++++++++++++++++++
 2 files changed, 147 insertions(+), 2 deletions(-)

diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py
index 87350b5479c..4fe069b0b7d 100644
--- a/litellm/completion_extras/litellm_responses_transformation/transformation.py
+++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py
@@ -370,7 +370,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             and isinstance(tool_call.get("custom"), dict)
         )
 
-        for msg in messages:
+        leading_system_count: Final = next(
+            (index for index, msg in enumerate(messages) if msg.get("role") != "system"),
+            len(messages),
+        )
+
+        for index, msg in enumerate(messages):
             role = msg.get("role")
             content = msg.get("content", "")
             tool_calls = msg.get("tool_calls")
@@ -378,7 +383,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
 
             if role == "system":
                 # Extract system message as instructions
-                if isinstance(content, str):
+                if isinstance(content, str) and index < leading_system_count:
                     if instructions:
                         # Concatenate multiple system prompts with a space
                         instructions = f"{instructions} {content}"
diff --git a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py
index d4d47b145d1..e4ada0a9b31 100644
--- a/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py
+++ b/tests/test_litellm/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py
@@ -4147,3 +4147,143 @@ def test_streaming_final_chunk_carries_provider_metadata():
     assert chunks[-1]["content_filters"] == content_filters
     assert "background" not in chunks[-1]
     assert all("service_tier" not in chunk for chunk in chunks[:-1])
+
+
+def _system_input_item(text: str) -> dict[str, object]:
+    return {"type": "message", "role": "system", "content": [{"type": "input_text", "text": text}]}
+
+
+def test_mid_conversation_system_string_stays_in_input_after_a_user_turn():
+    handler: Final = LiteLLMResponsesTransformationHandler()
+
+    input_items, instructions = handler.convert_chat_completion_messages_to_responses_api(
+        [
+            {"role": "system", "content": "You are a helpful assistant."},
+            {"role": "user", "content": "Read the file."},
+            {"role": "system", "content": "<total_tokens>14982391 tokens left</total_tokens>"},
+            {"role": "user", "content": "Now summarize it."},
+        ]
+    )
+
+    assert instructions == "You are a helpful assistant."
+    assert input_items == [
+        {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Read the file."}]},
+        _system_input_item("<total_tokens>14982391 tokens left</total_tokens>"),
+        {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "Now summarize it."}]},
+    ]
+
+
+def test_leading_system_strings_still_join_instructions_without_a_following_turn():
+    handler: Final = LiteLLMResponsesTransformationHandler()
+
+    input_items, instructions = handler.convert_chat_completion_messages_to_responses_api(
+        [
+            {"role": "system", "content": "Be brief."},
+            {"role": "system", "content": "Answer in French."},
+        ]
+    )
+
+    assert instructions == "Be brief. Answer in French."
+    assert input_items == []
+
+
+def test_mid_conversation_system_reminder_as_string_and_as_text_block_produce_identical_input_items():
+    handler: Final = LiteLLMResponsesTransformationHandler()
+    reminder: Final = "<total_tokens>14982391 tokens left</total_tokens>"
+
+    as_string, string_instructions = handler.convert_chat_completion_messages_to_responses_api(
+        [{"role": "user", "content": "Read the file."}, {"role": "system", "content": reminder}]
+    )
+    as_block, block_instructions = handler.convert_chat_completion_messages_to_responses_api(
+        [
+            {"role": "user", "content": "Read the file."},
+            {
+                "role": "system",
+                "content": [{"type": "text", "text": reminder, "cache_control": {"type": "ephemeral"}}],
+            },
+        ]
+    )
+
+    assert string_instructions is None
+    assert block_instructions is None
+    assert json.dumps(as_string) == json.dumps(as_block)
+    assert as_string[1] == _system_input_item(reminder)
+
+
+def test_claude_code_shaped_history_keeps_a_byte_stable_input_prefix_across_requests():
+    handler: Final = LiteLLMResponsesTransformationHandler()
+    top_level_system: Final = [{"type": "text", "text": "You are Claude Code.", "cache_control": {"type": "ephemeral"}}]
+    first_reminder: Final = "<system-reminder>27k chars of deferred tools</system-reminder>"
+    second_reminder: Final = "<total_tokens>14982391 tokens left</total_tokens>"
+    first_request_messages: Final = [
+        {"role": "system", "content": top_level_system},
+        {"role": "user", "content": "Read inventory.py."},
+        {
+            "role": "system",
+            "content": [{"type": "text", "text": first_reminder, "cache_control": {"type": "ephemeral"}}],
+        },
+    ]
+    second_request_messages: Final = [
+        {"role": "system", "content": top_level_system},
+        {"role": "user", "content": "Read inventory.py."},
+        {"role": "system", "content": first_reminder},
+        {
+            "role": "assistant",
+            "content": None,
+            "tool_calls": [
+                {
+                    "id": "call_1",
+                    "type": "function",
+                    "function": {"name": "Read", "arguments": '{"file_path": "inventory.py"}'},
+                }
+            ],
+        },
+        {"role": "tool", "tool_call_id": "call_1", "content": "ITEMS = []"},
+        {
+            "role": "system",
+            "content": [{"type": "text", "text": second_reminder, "cache_control": {"type": "ephemeral"}}],
+        },
+    ]
+
+    first_request: Final = handler.transform_request(
+        model="gpt-5.6-luna",
+        messages=first_request_messages,
+        optional_params={},
+        litellm_params={},
+        headers={},
+        litellm_logging_obj=Mock(),
+    )
+    second_request: Final = handler.transform_request(
+        model="gpt-5.6-luna",
+        messages=second_request_messages,
+        optional_params={},
+        litellm_params={},
+        headers={},
+        litellm_logging_obj=Mock(),
+    )
+
+    assert "instructions" not in first_request
+    assert "instructions" not in second_request
+    assert first_request["input"][0] == _system_input_item("You are Claude Code.")
+    assert json.dumps(second_request["input"][: len(first_request["input"])]) == json.dumps(first_request["input"])
+    assert second_request["input"][len(first_request["input"]) :] == [
+        {"type": "function_call", "call_id": "call_1", "name": "Read", "arguments": '{"file_path": "inventory.py"}'},
+        {"type": "function_call_output", "call_id": "call_1", "output": [{"type": "input_text", "text": "ITEMS = []"}]},
+        _system_input_item(second_reminder),
+    ]
+
+
+def test_system_string_after_a_developer_message_stays_in_input_in_client_order():
+    handler: Final = LiteLLMResponsesTransformationHandler()
+
+    input_items, instructions = handler.convert_chat_completion_messages_to_responses_api(
+        [
+            {"role": "developer", "content": "Always answer in French."},
+            {"role": "system", "content": "Be brief."},
+            {"role": "user", "content": "Bonjour"},
+        ]
+    )
+
+    assert instructions is None
+    assert [item["role"] for item in input_items] == ["developer", "system", "user"]
+    assert input_items[1] == _system_input_item("Be brief.")

From ee1a6407cb5b901660e62d757f83cdd6a99e5a34 Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 8 Sep 2026 11:45:02 -0700
Subject: [PATCH 63/85] fix(guardrails): keep guardrail telemetry when a policy
 pipeline blocks or modifies the response (#40211)

* fix(guardrails): keep guardrail telemetry when a policy pipeline blocks or modifies the response

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(guardrails): count every raw-snapshot guardrail evaluation and type the telemetry carry helpers

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(policy_engine): type the recording guardrail hooks and telemetry test parameters

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: yucheng <yucheng@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../proxy/policy_engine/pipeline_executor.py  | 104 +++++++++----
 .../policy_engine/test_pipeline_executor.py   | 141 ++++++++++++++++++
 2 files changed, 216 insertions(+), 29 deletions(-)

diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py
index 4be0f556ed7..1bcda38657b 100644
--- a/litellm/proxy/policy_engine/pipeline_executor.py
+++ b/litellm/proxy/policy_engine/pipeline_executor.py
@@ -6,7 +6,7 @@ pass/fail actions (allow, block, next, modify_response) and data forwarding.
 """
 
 import time
-from collections.abc import Sequence
+from collections.abc import Mapping, Sequence
 from typing import Any, Final, Literal
 
 import litellm
@@ -16,7 +16,11 @@ from litellm.integrations.custom_guardrail import (
     ModifyResponseException,
 )
 from litellm.integrations.custom_logger import CustomLogger
-from litellm.litellm_core_utils.core_helpers import independent_snapshot
+from litellm.litellm_core_utils.core_helpers import (
+    get_metadata_variable_name_from_kwargs,
+    get_or_create_metadata_bucket,
+    independent_snapshot,
+)
 from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import (
     UnifiedLLMGuardrails,
 )
@@ -25,6 +29,7 @@ from litellm.types.proxy.policy_engine.pipeline_types import (
     PipelineStep,
     PipelineStepResult,
 )
+from litellm.types.utils import StandardLoggingGuardrailInformation
 
 try:
     from fastapi.exceptions import HTTPException
@@ -118,6 +123,7 @@ class PipelineExecutor:
                 return _allow_result(step_results=step_results, working_data=working_data, request_data=data)
 
             if action == "block":
+                _carry_working_guardrail_information(working_data=working_data, request_data=data)
                 return PipelineExecutionResult(
                     terminal_action="block",
                     step_results=step_results,
@@ -126,6 +132,7 @@ class PipelineExecutor:
                 )
 
             if action == "modify_response":
+                _carry_working_guardrail_information(working_data=working_data, request_data=data)
                 return PipelineExecutionResult(
                     terminal_action="modify_response",
                     step_results=step_results,
@@ -168,34 +175,33 @@ class PipelineExecutor:
             verbose_proxy_logger.warning("Pipeline: guardrail '%s' not found in callbacks", step.guardrail)
             return ("error", None, f"Guardrail '{step.guardrail}' not found", None)
 
+        # Inject guardrail name into metadata so should_run_guardrail() allows it
+        if "metadata" not in data:
+            data["metadata"] = {}
+        data["metadata"]["guardrails"] = [step.guardrail]
+
+        # A scan_raw_request step evaluates the pristine pre-pipeline
+        # snapshot instead of `data` (which earlier pass_data steps in
+        # this same pipeline may have already rewritten), same reason
+        # the normal sequential/parallel guardrail loops do this.
+        scans_raw_request: Final = callback.scan_raw_request
+        hook_input: Final[dict] = (  # mutable-ok: same request-payload shape as data
+            independent_snapshot(raw_request_snapshot)
+            if scans_raw_request and raw_request_snapshot is not None
+            else data
+        )
+        if hook_input is not data:
+            hook_input.setdefault("metadata", {})["guardrails"] = [step.guardrail]
+        snapshot_entries_before: Final = len(_recorded_guardrail_information(hook_input))
+
+        # Use unified_guardrail path if callback implements apply_guardrail
+        target: CustomLogger = callback
+        use_unified: Final = "apply_guardrail" in type(callback).__dict__ and not callback.use_native_lifecycle_hooks
+        if use_unified:
+            hook_input["guardrail_to_apply"] = callback
+            target = UnifiedLLMGuardrails()
+
         try:
-            # Inject guardrail name into metadata so should_run_guardrail() allows it
-            if "metadata" not in data:
-                data["metadata"] = {}
-            data["metadata"]["guardrails"] = [step.guardrail]
-
-            # A scan_raw_request step evaluates the pristine pre-pipeline
-            # snapshot instead of `data` (which earlier pass_data steps in
-            # this same pipeline may have already rewritten), same reason
-            # the normal sequential/parallel guardrail loops do this.
-            scans_raw_request: Final = callback.scan_raw_request
-            hook_input: Final[dict] = (  # mutable-ok: same request-payload shape as data
-                independent_snapshot(raw_request_snapshot)
-                if scans_raw_request and raw_request_snapshot is not None
-                else data
-            )
-            if hook_input is not data:
-                hook_input.setdefault("metadata", {})["guardrails"] = [step.guardrail]
-
-            # Use unified_guardrail path if callback implements apply_guardrail
-            target: CustomLogger = callback
-            use_unified: Final = (
-                "apply_guardrail" in type(callback).__dict__ and not callback.use_native_lifecycle_hooks
-            )
-            if use_unified:
-                hook_input["guardrail_to_apply"] = callback
-                target = UnifiedLLMGuardrails()
-
             if mode == "pre_call":
                 response = await target.async_pre_call_hook(
                     user_api_key_dict=user_api_key_dict,
@@ -233,6 +239,12 @@ class PipelineExecutor:
             else:
                 verbose_proxy_logger.error("Pipeline: unexpected error from guardrail '%s': %s", step.guardrail, e)
                 return ("error", None, str(e), e)
+        finally:
+            if hook_input is not data:
+                _append_guardrail_information(
+                    request_data=data,
+                    entries=_recorded_guardrail_information(hook_input)[snapshot_entries_before:],
+                )
 
     @staticmethod
     def find_guardrail_callback(guardrail_name: str) -> CustomGuardrail | None:
@@ -283,6 +295,40 @@ def _restore_request_guardrails(
     return {**working_data, "metadata": stripped}  # mutable-ok: request dict
 
 
+_GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information"
+
+
+def _recorded_guardrail_information(source: Mapping[str, object]) -> list[StandardLoggingGuardrailInformation]:
+    bucket: Final = source.get(get_metadata_variable_name_from_kwargs(source))
+    recorded: Final = bucket.get(_GUARDRAIL_INFORMATION_KEY) if isinstance(bucket, dict) else None
+    return recorded if isinstance(recorded, list) else []
+
+
+def _append_guardrail_information(
+    request_data: dict[str, object],  # mutable-ok: same request-payload shape as execute_steps' data
+    entries: Sequence[StandardLoggingGuardrailInformation],
+) -> None:
+    if not entries:
+        return
+    _, request_bucket = get_or_create_metadata_bucket(request_data)
+    existing: Final = request_bucket.get(_GUARDRAIL_INFORMATION_KEY)
+    if isinstance(existing, list):
+        existing.extend(entries)
+        return
+    request_bucket[_GUARDRAIL_INFORMATION_KEY] = list(entries)
+
+
+def _carry_working_guardrail_information(
+    working_data: Mapping[str, object],
+    request_data: dict[str, object],  # mutable-ok: same request-payload shape as execute_steps' data
+) -> None:
+    recorded: Final = _recorded_guardrail_information(working_data)
+    existing: Final = _recorded_guardrail_information(request_data)
+    if recorded is existing:
+        return
+    _append_guardrail_information(request_data=request_data, entries=[e for e in recorded if e not in existing])
+
+
 def _pipeline_action_for_outcome(step: PipelineStep, outcome: str) -> str:
     """
     Map pipeline step outcome to the configured action.
diff --git a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py
index 4fcb7d22588..e95d001bb12 100644
--- a/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py
+++ b/tests/test_litellm/proxy/policy_engine/test_pipeline_executor.py
@@ -4,20 +4,26 @@ Tests for the pipeline executor.
 Uses mock guardrails to validate pipeline execution without external services.
 """
 
+import copy
+from typing import Literal
 from unittest.mock import MagicMock
 
 import pytest
 
 import litellm
+from litellm.caching.dual_cache import DualCache
 from litellm.integrations.custom_guardrail import CustomGuardrail
+from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.guardrails.guardrail_hooks.custom_code.custom_code_guardrail import (
     CustomCodeGuardrail,
 )
 from litellm.proxy.policy_engine.pipeline_executor import PipelineExecutor
+from litellm.types.guardrails import GuardrailEventHooks
 from litellm.types.proxy.policy_engine.pipeline_types import (
     GuardrailPipeline,
     PipelineStep,
 )
+from litellm.types.utils import CallTypesLiteral
 
 try:
     from fastapi.exceptions import HTTPException
@@ -158,11 +164,146 @@ class ContentCheckGuardrail(CustomGuardrail):
         return None
 
 
+class RecordingGuardrail(CustomGuardrail):
+    def __init__(self, guardrail_name: str, scan_raw_request: bool = False, block: bool = True):
+        super().__init__(
+            guardrail_name=guardrail_name,
+            event_hook="pre_call",
+            default_on=True,
+            scan_raw_request=scan_raw_request,
+        )
+        self.block = block
+
+    def should_run_guardrail(self, data: dict[str, object], event_type: GuardrailEventHooks) -> bool:
+        return True
+
+    async def async_pre_call_hook(
+        self,
+        user_api_key_dict: UserAPIKeyAuth,
+        cache: DualCache,
+        data: dict[str, object],
+        call_type: CallTypesLiteral,
+    ) -> dict[str, object]:
+        self.add_standard_logging_guardrail_information_to_request_data(
+            guardrail_json_response={"detected": ["aws_access_key"]},
+            request_data=data,
+            guardrail_status="guardrail_intervened" if self.block else "success",
+        )
+        if self.block:
+            raise HTTPException(status_code=400, detail="Content policy violation")
+        return copy.deepcopy(data)
+
+
 # ─────────────────────────────────────────────────────────────────────────────
 # Tests
 # ─────────────────────────────────────────────────────────────────────────────
 
 
+@pytest.mark.skipif(HTTPException is None, reason="fastapi not installed")
+@pytest.mark.asyncio
+@pytest.mark.parametrize("scan_raw_request", [False, True])
+@pytest.mark.parametrize("on_fail", ["block", "modify_response"])
+async def test_terminal_block_carries_guardrail_information_to_request(
+    monkeypatch: pytest.MonkeyPatch, scan_raw_request: bool, on_fail: Literal["block", "modify_response"]
+):
+    """
+    Spend logging and the Guardrails Monitor read standard_logging_guardrail_information
+    off the caller's request dict. A blocking step records it on the executor's
+    working copy (or the raw-request snapshot), so the terminal result must carry it
+    back onto the request or the block is never counted.
+    """
+    guard = RecordingGuardrail(guardrail_name="credentials-api-keys", scan_raw_request=scan_raw_request)
+    monkeypatch.setattr(litellm, "callbacks", [guard])
+    data = {
+        "messages": [{"role": "user", "content": "key AKIAIOSFODNN7EXAMPLE"}],
+        "metadata": {"user_api_key_hash": "abc"},
+    }
+
+    result = await PipelineExecutor.execute_steps(
+        steps=[PipelineStep(guardrail="credentials-api-keys", on_fail=on_fail, on_pass="next")],
+        mode="pre_call",
+        data=data,
+        user_api_key_dict=MagicMock(),
+        call_type="completion",
+        policy_name="baseline-pii-protection",
+        raw_request_snapshot={"messages": data["messages"], "metadata": {"user_api_key_hash": "abc"}},
+    )
+
+    assert result.terminal_action == on_fail
+    recorded = data["metadata"]["standard_logging_guardrail_information"]
+    assert [entry["guardrail_name"] for entry in recorded] == ["credentials-api-keys"]
+    assert recorded[0]["guardrail_status"] == "guardrail_intervened"
+    assert data["metadata"]["user_api_key_hash"] == "abc"
+    assert "guardrails" not in data["metadata"]
+
+
+@pytest.mark.skipif(HTTPException is None, reason="fastapi not installed")
+@pytest.mark.asyncio
+async def test_terminal_block_merges_guardrail_information_without_duplicates(monkeypatch: pytest.MonkeyPatch):
+    """A pass_data step that returns a rewritten copy of the request, and a scan_raw_request step
+    that evaluates a deep copy taken before the pipeline ran, both leave earlier entries in two
+    dicts at once. Those must be carried back once while every step's own entry is kept."""
+    first = RecordingGuardrail(guardrail_name="pii-scan", block=False)
+    second = RecordingGuardrail(guardrail_name="credentials-api-keys", scan_raw_request=True)
+    monkeypatch.setattr(litellm, "callbacks", [first, second])
+    earlier = {"guardrail_name": "earlier-guard", "guardrail_status": "success"}
+    data = {"messages": [{"role": "user", "content": "hi"}], "metadata": {}}
+    data["metadata"]["standard_logging_guardrail_information"] = [earlier]
+
+    result = await PipelineExecutor.execute_steps(
+        steps=[
+            PipelineStep(guardrail="pii-scan", on_fail="block", on_pass="next", pass_data=True),
+            PipelineStep(guardrail="credentials-api-keys", on_fail="block", on_pass="next"),
+        ],
+        mode="pre_call",
+        data=data,
+        user_api_key_dict=MagicMock(),
+        call_type="completion",
+        policy_name="baseline-pii-protection",
+        raw_request_snapshot={
+            "messages": data["messages"],
+            "metadata": {"standard_logging_guardrail_information": [dict(earlier)]},
+        },
+    )
+
+    assert result.terminal_action == "block"
+    recorded = data["metadata"]["standard_logging_guardrail_information"]
+    assert [entry["guardrail_name"] for entry in recorded] == ["earlier-guard", "pii-scan", "credentials-api-keys"]
+
+
+@pytest.mark.skipif(HTTPException is None, reason="fastapi not installed")
+@pytest.mark.asyncio
+async def test_repeated_scan_raw_request_step_is_counted_once_per_evaluation(monkeypatch: pytest.MonkeyPatch):
+    """Running the same raw-scan guardrail twice yields two identical entries; both must reach the caller,
+    while the entries the raw snapshot already held before the pipeline ran are not copied again."""
+    guard = RecordingGuardrail(guardrail_name="credentials-raw", scan_raw_request=True, block=False)
+    monkeypatch.setattr(litellm, "callbacks", [guard])
+    earlier = {"guardrail_name": "earlier-guard", "guardrail_status": "success"}
+    data = {"messages": [{"role": "user", "content": "hi"}], "metadata": {}}
+    data["metadata"]["standard_logging_guardrail_information"] = [earlier]
+
+    result = await PipelineExecutor.execute_steps(
+        steps=[
+            PipelineStep(guardrail="credentials-raw", on_fail="block", on_pass="next"),
+            PipelineStep(guardrail="credentials-raw", on_fail="block", on_pass="next"),
+        ],
+        mode="pre_call",
+        data=data,
+        user_api_key_dict=MagicMock(),
+        call_type="completion",
+        policy_name="raw-scan-policy",
+        raw_request_snapshot={
+            "messages": data["messages"],
+            "metadata": {"standard_logging_guardrail_information": [dict(earlier)]},
+        },
+    )
+
+    assert result.terminal_action == "allow"
+    assert result.modified_data is not None
+    recorded = result.modified_data["metadata"]["standard_logging_guardrail_information"]
+    assert [entry["guardrail_name"] for entry in recorded] == ["earlier-guard", "credentials-raw", "credentials-raw"]
+
+
 @pytest.mark.skipif(HTTPException is None, reason="fastapi not installed")
 @pytest.mark.asyncio
 async def test_escalation_step1_fails_step2_blocks(monkeypatch):

From 35d1d40a6784c222714e880c7d07729bc90018b2 Mon Sep 17 00:00:00 2001
From: yujonglee <yujong@berri.ai>
Date: Tue, 8 Sep 2026 12:07:25 -0700
Subject: [PATCH 64/85] fix(ocr): run post-call logging hooks (#40154)

---
 litellm/integrations/custom_guardrail.py      | 10 +--
 litellm/llms/custom_httpx/llm_http_handler.py | 12 ++++
 .../test_secret_detection.py                  |  2 +-
 .../integrations/test_custom_guardrail.py     | 33 ++++++++++
 .../custom_httpx/test_llm_http_handler.py     | 64 +++++++++++++++++++
 .../test_proxy_logging_hook_detection.py      | 10 +--
 6 files changed, 121 insertions(+), 10 deletions(-)

diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py
index 2d66a280663..e76d4b05bb2 100644
--- a/litellm/integrations/custom_guardrail.py
+++ b/litellm/integrations/custom_guardrail.py
@@ -773,7 +773,7 @@ class CustomGuardrail(CustomLogger):
     def uses_apply_guardrail_interface(self) -> bool:
         return type(self).apply_guardrail is not CustomGuardrail.apply_guardrail
 
-    def _deployment_pre_call_target(self) -> "CustomLogger":
+    def _deployment_hook_target(self) -> "CustomLogger":
         if not self.uses_apply_guardrail_interface() or self.use_native_lifecycle_hooks:
             return self
         try:
@@ -802,7 +802,7 @@ class CustomGuardrail(CustomLogger):
 
         # CHECK IF GUARDRAIL REJECTS THE REQUEST
         if call_type == CallTypes.completion or call_type == CallTypes.acompletion:
-            target: Final = self._deployment_pre_call_target()
+            target: Final = self._deployment_hook_target()
             if target is not self:
                 kwargs["guardrail_to_apply"] = self
             result: Final = await target.async_pre_call_hook(
@@ -845,7 +845,9 @@ class CustomGuardrail(CustomLogger):
             return None
 
         # CHECK IF GUARDRAIL REJECTS THE REQUEST
-        result: Final = await self.async_post_call_success_hook(
+        target: Final = self._deployment_hook_target()
+        hook_request_data: Final = {**request_data, "guardrail_to_apply": self} if target is not self else request_data
+        result: Final = await target.async_post_call_success_hook(
             user_api_key_dict=UserAPIKeyAuth(
                 user_id=request_data.get("user_api_key_user_id"),
                 team_id=request_data.get("user_api_key_team_id"),
@@ -853,7 +855,7 @@ class CustomGuardrail(CustomLogger):
                 api_key=request_data.get("user_api_key_hash"),
                 request_route=request_data.get("user_api_key_request_route"),
             ),
-            data=request_data,
+            data=hook_request_data,
             response=response,
         )
 
diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py
index 2f561809940..8b6d3a0ceb4 100644
--- a/litellm/llms/custom_httpx/llm_http_handler.py
+++ b/litellm/llms/custom_httpx/llm_http_handler.py
@@ -1741,6 +1741,12 @@ class BaseLLMHTTPHandler:
         except Exception as e:
             raise self._handle_error(e=e, provider_config=provider_config)
 
+        logging_obj.post_call(
+            api_key=api_key,
+            original_response=response.text,
+            additional_args={"complete_input_dict": data},
+        )
+
         return self._transform_ocr_response(
             provider_config=provider_config,
             model=model,
@@ -1804,6 +1810,12 @@ class BaseLLMHTTPHandler:
         except Exception as e:
             raise self._handle_error(e=e, provider_config=provider_config)
 
+        logging_obj.post_call(
+            api_key=api_key,
+            original_response=response.text,
+            additional_args={"complete_input_dict": data},
+        )
+
         # Use async response transform for async operations
         return await provider_config.async_transform_ocr_response(
             model=model,
diff --git a/tests/test_litellm/enterprise/enterprise_callbacks/test_secret_detection.py b/tests/test_litellm/enterprise/enterprise_callbacks/test_secret_detection.py
index f46df5baadf..c1c3569c0e2 100644
--- a/tests/test_litellm/enterprise/enterprise_callbacks/test_secret_detection.py
+++ b/tests/test_litellm/enterprise/enterprise_callbacks/test_secret_detection.py
@@ -262,7 +262,7 @@ def test_proxied_traffic_stays_on_native_hooks():
     never sees ``data["prompt"]``."""
     guardrail = _guardrail()
     assert guardrail.uses_apply_guardrail_interface() is True
-    assert guardrail._deployment_pre_call_target() is guardrail
+    assert guardrail._deployment_hook_target() is guardrail
 
 
 @pytest.mark.asyncio
diff --git a/tests/test_litellm/integrations/test_custom_guardrail.py b/tests/test_litellm/integrations/test_custom_guardrail.py
index 885bd1d4d72..359635335b3 100644
--- a/tests/test_litellm/integrations/test_custom_guardrail.py
+++ b/tests/test_litellm/integrations/test_custom_guardrail.py
@@ -2610,3 +2610,36 @@ class TestCustomGuardrailPostCallSuccessDeploymentHook:
         )
 
         assert result is replacement
+
+    @pytest.mark.asyncio
+    async def test_apply_guardrail_interface_modifies_deployment_response(self):
+        from litellm.types.guardrails import GuardrailEventHooks
+        from litellm.types.utils import ModelResponse
+
+        class ReplacingGuardrail(CustomGuardrail):
+            async def apply_guardrail(
+                self,
+                inputs: GenericGuardrailAPIInputs,
+                request_data: dict[str, object],
+                input_type: Literal["request", "response"],
+                logging_obj: Optional["LiteLLMLoggingObj"] = None,
+            ) -> GenericGuardrailAPIInputs:
+                assert input_type == "response"
+                return {**inputs, "texts": ["filtered response"]}
+
+        guardrail = ReplacingGuardrail(
+            guardrail_name="test-guardrail",
+            event_hook=GuardrailEventHooks.post_call,
+        )
+        response = ModelResponse(choices=[{"message": {"role": "assistant", "content": "original response"}}])
+        request_data = {"guardrails": ["test-guardrail"]}
+
+        result = await guardrail.async_post_call_success_deployment_hook(
+            request_data=request_data,
+            response=response,
+            call_type=CallTypes.acompletion,
+        )
+
+        assert result is response
+        assert response.choices[0].message.content == "filtered response"
+        assert request_data == {"guardrails": ["test-guardrail"]}
diff --git a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py
index e16855da8cb..98a5f4b2db5 100644
--- a/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py
+++ b/tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py
@@ -29,6 +29,7 @@ from litellm.llms.custom_httpx.llm_http_handler import (
     _rust_responses_websocket_enabled,
 )
 from litellm.llms.azure.videos.transformation import AzureVideoConfig
+from litellm.llms.mistral.ocr.transformation import MistralOCRConfig
 from litellm.llms.openai.videos.transformation import OpenAIVideoConfig
 from litellm.types.llms.openai import ResponsesAPIResponse
 from litellm.types.router import GenericLiteLLMParams
@@ -37,6 +38,69 @@ from litellm.types.utils import ImageObject, ImageResponse, ModelResponse, Trans
 _ACTIVE_KEY = "_code_interpreter_interception_active"
 _SANDBOX_KEY = "_code_interpreter_interception_sandbox_key"
 
+OCR_RESPONSE = {
+    "pages": [{"index": 0, "markdown": "OCR output", "images": []}],
+    "model": "mistral-ocr-latest",
+    "usage_info": {"pages_processed": 1},
+}
+
+
+def _ocr_sync_client() -> HTTPHandler:
+    client = HTTPHandler()
+    client.client = httpx.Client(transport=httpx.MockTransport(lambda _request: httpx.Response(200, json=OCR_RESPONSE)))
+    return client
+
+
+def _ocr_async_client() -> AsyncHTTPHandler:
+    client = AsyncHTTPHandler()
+    client.client = httpx.AsyncClient(
+        transport=httpx.MockTransport(lambda _request: httpx.Response(200, json=OCR_RESPONSE))
+    )
+    return client
+
+
+def test_ocr_calls_post_call_with_raw_provider_response():
+    logging_obj = Mock()
+
+    response = BaseLLMHTTPHandler().ocr(
+        model="mistral-ocr-latest",
+        document={"type": "document_url", "document_url": "https://example.com/document.pdf"},
+        optional_params={},
+        timeout=5,
+        logging_obj=logging_obj,
+        api_key="test-key",
+        api_base="https://api.mistral.ai/v1/ocr",
+        custom_llm_provider="mistral",
+        client=_ocr_sync_client(),
+        provider_config=MistralOCRConfig(),
+    )
+
+    assert response.pages[0].markdown == "OCR output"
+    logging_obj.post_call.assert_called_once()
+    assert json.loads(logging_obj.post_call.call_args.kwargs["original_response"]) == OCR_RESPONSE
+
+
+@pytest.mark.asyncio
+async def test_async_ocr_calls_post_call_with_raw_provider_response():
+    logging_obj = Mock()
+
+    response = await BaseLLMHTTPHandler().async_ocr(
+        model="mistral-ocr-latest",
+        document={"type": "document_url", "document_url": "https://example.com/document.pdf"},
+        optional_params={},
+        timeout=5,
+        logging_obj=logging_obj,
+        api_key="test-key",
+        api_base="https://api.mistral.ai/v1/ocr",
+        custom_llm_provider="mistral",
+        client=_ocr_async_client(),
+        provider_config=MistralOCRConfig(),
+    )
+
+    assert response.pages[0].markdown == "OCR output"
+    logging_obj.post_call.assert_called_once()
+    assert json.loads(logging_obj.post_call.call_args.kwargs["original_response"]) == OCR_RESPONSE
+
 
 def test_prepare_fake_stream_request():
     # Initialize the BaseLLMHTTPHandler
diff --git a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py
index 9f1321aec2c..22930a26974 100644
--- a/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py
+++ b/tests/test_litellm/proxy/test_proxy_logging_hook_detection.py
@@ -637,14 +637,14 @@ def test_callback_capabilities_excludes_opted_out_guardrail_from_iterator_overri
     assert [cb for cb, _ in caps.iterator_overrides if cb is opted_out] == []
 
 
-def test_deployment_pre_call_target_stays_native_when_opted_out():
+def test_deployment_hook_target_stays_native_when_opted_out():
     """Model-level guardrails resolve their target here rather than through ProxyLogging."""
-    assert _KeepsNativeHooks()._deployment_pre_call_target() is not None
+    assert _KeepsNativeHooks()._deployment_hook_target() is not None
     opted_out = _KeepsNativeHooks()
-    assert opted_out._deployment_pre_call_target() is opted_out
-    assert _AppliesGuardrail()._deployment_pre_call_target() is not None
+    assert opted_out._deployment_hook_target() is opted_out
+    assert _AppliesGuardrail()._deployment_hook_target() is not None
     routed = _AppliesGuardrail()
-    assert routed._deployment_pre_call_target() is not routed
+    assert routed._deployment_hook_target() is not routed
 
 
 @pytest.mark.asyncio

From 01c68c199bc0afddf5a4d806cb03eca5e7745295 Mon Sep 17 00:00:00 2001
From: yujonglee <yujong@berri.ai>
Date: Tue, 8 Sep 2026 12:18:09 -0700
Subject: [PATCH 65/85] fix(guardrails): allow framework-supported logging-only
 mode (#40267)

---
 litellm/integrations/custom_guardrail.py      |  10 +-
 .../integrations/test_custom_guardrail.py     | 100 +++++++++++++++++-
 2 files changed, 107 insertions(+), 3 deletions(-)

diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py
index e76d4b05bb2..37d6a7e793d 100644
--- a/litellm/integrations/custom_guardrail.py
+++ b/litellm/integrations/custom_guardrail.py
@@ -601,6 +601,12 @@ class CustomGuardrail(CustomLogger):
         event_hook: GuardrailEventHooks | list[GuardrailEventHooks] | Mode | None,
         supported_event_hooks: list[GuardrailEventHooks],
     ) -> None:
+        allowed_hooks: Final = frozenset(supported_event_hooks) | (
+            frozenset((GuardrailEventHooks.logging_only,))
+            if self.uses_apply_guardrail_interface() and not self.use_native_lifecycle_hooks
+            else frozenset()
+        )
+
         def _validate_event_hook_list_is_in_supported_event_hooks(
             event_hook: list[GuardrailEventHooks] | list[str],
             supported_event_hooks: list[GuardrailEventHooks],
@@ -608,7 +614,7 @@ class CustomGuardrail(CustomLogger):
             for hook in event_hook:
                 if isinstance(hook, str):
                     hook = GuardrailEventHooks(hook)
-                if hook not in supported_event_hooks:
+                if hook not in allowed_hooks:
                     raise ValueError(f"Event hook {hook} is not in the supported event hooks {supported_event_hooks}")
 
         if event_hook is None:
@@ -629,7 +635,7 @@ class CustomGuardrail(CustomLogger):
                 default_list = event_hook.default if isinstance(event_hook.default, list) else [event_hook.default]
                 _validate_event_hook_list_is_in_supported_event_hooks(default_list, supported_event_hooks)
         elif isinstance(event_hook, GuardrailEventHooks):
-            if event_hook not in supported_event_hooks:
+            if event_hook not in allowed_hooks:
                 raise ValueError(f"Event hook {event_hook} is not in the supported event hooks {supported_event_hooks}")
 
     @staticmethod
diff --git a/tests/test_litellm/integrations/test_custom_guardrail.py b/tests/test_litellm/integrations/test_custom_guardrail.py
index 359635335b3..cd8d609cf71 100644
--- a/tests/test_litellm/integrations/test_custom_guardrail.py
+++ b/tests/test_litellm/integrations/test_custom_guardrail.py
@@ -1,5 +1,5 @@
 import asyncio
-from typing import TYPE_CHECKING, Literal, Optional
+from typing import TYPE_CHECKING, ClassVar, Final, Literal, Optional
 from unittest.mock import AsyncMock
 
 import pytest
@@ -10,6 +10,7 @@ from litellm.integrations.custom_guardrail import (
     log_guardrail_information,
 )
 from litellm.proxy._types import CallTypes, UserAPIKeyAuth
+from litellm.types.guardrails import GuardrailEventHooks, Mode
 from litellm.types.utils import GenericGuardrailAPIInputs, GuardrailTracingDetail
 
 if TYPE_CHECKING:
@@ -2378,11 +2379,108 @@ def _logged_call(messages: list | str) -> tuple[dict, object]:
     return kwargs, response
 
 
+class _NativeApplyGuardrail(_InheritedApplyGuardrail):
+    use_native_lifecycle_hooks: ClassVar[bool] = True
+
+
+@pytest.mark.parametrize("guardrail_type", (CustomGuardrail, _NativeApplyGuardrail, _InheritedApplyGuardrail))
+@pytest.mark.parametrize(
+    "event_hook",
+    (
+        GuardrailEventHooks.logging_only,
+        "logging_only",
+        [GuardrailEventHooks.pre_call, GuardrailEventHooks.logging_only],
+        ["pre_call", "logging_only"],
+        Mode(tags={"audit": "logging_only"}, default="pre_call"),
+        Mode(tags={"audit": ["pre_call", "logging_only"]}),
+        Mode(tags={"enforce": "pre_call"}, default="logging_only"),
+        Mode(tags={}, default=["pre_call", "logging_only"]),
+    ),
+)
+def test_logging_only_requires_framework_support_or_explicit_declaration(
+    guardrail_type: type[CustomGuardrail],
+    event_hook: GuardrailEventHooks | str | list[GuardrailEventHooks] | list[str] | Mode,
+) -> None:
+    supported: Final = [GuardrailEventHooks.pre_call]
+    if guardrail_type is _InheritedApplyGuardrail:
+        guardrail: Final = guardrail_type(event_hook=event_hook, supported_event_hooks=supported)
+        assert guardrail.event_hook == event_hook
+        assert supported == [GuardrailEventHooks.pre_call]
+    else:
+        with pytest.raises(ValueError, match=r"logging_only.*not in the supported event hooks"):
+            guardrail_type(event_hook=event_hook, supported_event_hooks=supported)
+
+    explicitly_supported: Final = guardrail_type(
+        event_hook=event_hook,
+        supported_event_hooks=[GuardrailEventHooks.pre_call, GuardrailEventHooks.logging_only],
+    )
+    assert explicitly_supported.event_hook == event_hook
+
+
+@pytest.mark.parametrize(
+    "event_hook",
+    (
+        GuardrailEventHooks.post_call,
+        "post_call",
+        [GuardrailEventHooks.logging_only, GuardrailEventHooks.post_call],
+        ["logging_only", "post_call"],
+        Mode(tags={"enforce": "post_call"}, default="logging_only"),
+        Mode(tags={"enforce": ["logging_only", "post_call"]}),
+        Mode(tags={"audit": "logging_only"}, default="post_call"),
+        Mode(tags={}, default=["logging_only", "post_call"]),
+    ),
+)
+def test_framework_logging_only_does_not_allow_other_unsupported_modes(
+    event_hook: GuardrailEventHooks | str | list[GuardrailEventHooks] | list[str] | Mode,
+) -> None:
+    with pytest.raises(ValueError, match=r"post_call.*not in the supported event hooks"):
+        _InheritedApplyGuardrail(event_hook=event_hook, supported_event_hooks=[GuardrailEventHooks.pre_call])
+
+
 class TestLoggingOnlyApplyGuardrail:
     """LIT-4876 regression: a guardrail in mode logging_only that implements only
     apply_guardrail must still run against the logged request and response and
     record guardrail_information, instead of inheriting the CustomLogger no-op."""
 
+    @pytest.mark.parametrize(
+        "event_hook",
+        (
+            GuardrailEventHooks.logging_only,
+            "logging_only",
+            [GuardrailEventHooks.pre_call, GuardrailEventHooks.logging_only],
+            ["pre_call", "logging_only"],
+            Mode(tags={"audit": "logging_only"}, default="pre_call"),
+            Mode(tags={"audit": ["pre_call", "logging_only"]}),
+            Mode(tags={"enforce": "pre_call"}, default="logging_only"),
+            Mode(tags={}, default=["pre_call", "logging_only"]),
+        ),
+    )
+    @pytest.mark.asyncio
+    async def test_content_filter_accepts_logging_only_and_records_detection(
+        self, event_hook: GuardrailEventHooks | str | list[GuardrailEventHooks] | list[str] | Mode
+    ) -> None:
+        from litellm.proxy.guardrails.guardrail_hooks.litellm_content_filter.content_filter import (
+            ContentFilterGuardrail,
+        )
+        from litellm.types.guardrails import BlockedWord, ContentFilterAction, GuardrailEventHooks
+
+        guardrail: Final = ContentFilterGuardrail(
+            guardrail_name="content-review",
+            event_hook=event_hook,
+            default_on=True,
+            blocked_words=[BlockedWord(keyword="hello", action=ContentFilterAction.BLOCK)],
+        )
+        kwargs, response = _logged_call([{"role": "user", "content": "hello there"}])
+
+        out_kwargs, out_response = await guardrail.async_logging_hook(kwargs, response, CallTypes.acompletion.value)
+
+        assert out_response is response
+        assert out_kwargs["messages"] == kwargs["messages"]
+        assert (
+            out_kwargs["standard_logging_object"]["guardrail_information"][0]["guardrail_status"]
+            == "guardrail_intervened"
+        )
+
     @pytest.mark.asyncio
     async def test_runs_apply_guardrail_observe_only_and_records_verdict(self):
         guardrail = _ApplyOnlyObserver()

From d36e032241490a5a75063425a99da4ccdc85548a Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 8 Sep 2026 13:00:14 -0700
Subject: [PATCH 66/85] fix(proxy): initialize string success/failure callbacks
 at startup after config load (#38226)

* fix(proxy): eagerly initialize string callbacks

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(proxy): materialize string callbacks after load_config so later litellm_settings keys are applied

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(proxy): drop casts when snapshotting string callbacks so LIT006 stays at base

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: yucheng <yucheng@berri.ai>
---
 litellm/proxy/utils.py                        | 11 +++++
 tests/test_litellm/proxy/test_proxy_server.py | 41 +++++++++++++++++++
 2 files changed, 52 insertions(+)

diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index ddf31cb1d8a..75d7972c621 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -177,6 +177,9 @@ from litellm.types.mcp import (
 )
 from litellm.types.proxy.policy_engine.pipeline_types import PipelineExecutionResult
 from litellm.types.utils import LLMResponseTypes, LoggedLiteLLMParams
+from litellm.utils import (
+    _add_custom_logger_callback_to_specific_event,  # pyright: ignore[reportPrivateUsage]  # only string-to-logger helper
+)
 
 if TYPE_CHECKING:
     from mcp.types import CallToolResult
@@ -857,6 +860,14 @@ class ProxyLogging:
                 litellm.logging_callback_manager.add_litellm_async_success_callback(callback)
                 litellm.logging_callback_manager.add_litellm_async_failure_callback(callback)
 
+        # Runs after load_config applied every litellm_settings key: logger __init__s read e.g. s3_callback_params
+        success_callbacks: Final = tuple(cb for cb in litellm.success_callback if isinstance(cb, str))
+        failure_callbacks: Final = tuple(cb for cb in litellm.failure_callback if isinstance(cb, str))
+        for callback in success_callbacks:
+            _add_custom_logger_callback_to_specific_event(callback, "success")
+        for callback in failure_callbacks:
+            _add_custom_logger_callback_to_specific_event(callback, "failure")
+
     async def update_request_status(self, litellm_call_id: str, status: Literal["success", "fail"]):
         # only use this if slack alerting is being used
         if self.alerting is None:
diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py
index 937e4f15741..6a901c5e7de 100644
--- a/tests/test_litellm/proxy/test_proxy_server.py
+++ b/tests/test_litellm/proxy/test_proxy_server.py
@@ -3166,6 +3166,47 @@ async def test_custom_ui_sso_sign_in_handler_config_loading():
         os.unlink(config_file_path)
 
 
+@pytest.mark.asyncio
+async def test_startup_initializes_string_callbacks_after_all_litellm_settings_load(tmp_path, monkeypatch):
+    from litellm.integrations.s3_v2 import S3Logger
+    from litellm.litellm_core_utils import litellm_logging
+    from litellm.proxy.proxy_server import ProxyConfig
+    from litellm.proxy.utils import ProxyLogging
+
+    config_file = tmp_path / "config.yaml"
+    config_file.write_text(
+        "model_list: []\n"
+        "litellm_settings:\n"
+        "  success_callback:\n"
+        "    - s3_v2\n"
+        "  failure_callback:\n"
+        "    - s3_v2\n"
+        "  s3_callback_params:\n"
+        "    s3_bucket_name: ordering-regression-bucket\n"
+        "    s3_region_name: us-west-2\n"
+    )
+
+    monkeypatch.setattr(litellm, "success_callback", [])
+    monkeypatch.setattr(litellm, "_async_success_callback", [])
+    monkeypatch.setattr(litellm, "failure_callback", [])
+    monkeypatch.setattr(litellm, "_async_failure_callback", [])
+    monkeypatch.setattr(litellm, "callbacks", [])
+    monkeypatch.setattr(litellm, "s3_callback_params", None)
+    monkeypatch.setattr(litellm_logging, "_in_memory_loggers", [])
+
+    await ProxyConfig().load_config(router=MagicMock(), config_file_path=str(config_file))
+    ProxyLogging(user_api_key_cache=MagicMock())._init_litellm_callbacks(llm_router=None)
+
+    success_loggers = [cb for cb in litellm._async_success_callback if isinstance(cb, S3Logger)]
+    failure_loggers = [cb for cb in litellm._async_failure_callback if isinstance(cb, S3Logger)]
+    assert len(success_loggers) == 1
+    assert len(failure_loggers) == 1
+    assert success_loggers[0].s3_bucket_name == "ordering-regression-bucket"
+    assert success_loggers[0].s3_region_name == "us-west-2"
+    assert "s3_v2" not in litellm.success_callback
+    assert "s3_v2" not in litellm.failure_callback
+
+
 @pytest.mark.asyncio
 async def test_load_config_max_budget_env_var_coerced_to_float(tmp_path, monkeypatch):
     """

From 6a425a5cc59ce67f3860ac04f752cf61340e1028 Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 8 Sep 2026 13:08:26 -0700
Subject: [PATCH 67/85] fix(responses): record spend for native Responses API
 WebSocket sessions (#38856)

* fix(responses): record spend for native Responses API WebSocket sessions

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(responses): bill usage from response.incomplete WebSocket turns

A turn cut short by max_output_tokens ends in response.incomplete, which
OpenAI bills but the processor only read response.completed, so those
sessions still logged zero spend

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(responses): hoist websocket usage test imports to module scope

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(responses): price websocket sessions through the standard cost path

The realtime completion_cost branch skips cost_discount_config and cost_margin_config, so a native
Responses WebSocket session was priced differently from the same usage over HTTP /v1/responses.
Drop the explicit widening so the LiteLLMRealtimeStreamLoggingObject built by
normalize_logging_result flows through the generic usage path, and pin WS == HTTP cost under a
50% provider discount in the regression test

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* ci: rerun proxy-infra after flaky test_check_migration process tree test

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Co-authored-by: yucheng <yucheng@berri.ai>
---
 litellm/cost_calculator.py                    | 40 ++++++++
 litellm/litellm_core_utils/litellm_logging.py | 12 +++
 litellm/types/utils.py                        |  1 +
 .../test_litellm_logging.py                   | 92 ++++++++++++++++++-
 4 files changed, 144 insertions(+), 1 deletion(-)

diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py
index 9a9d2ceda03..c9c7df4d75e 100644
--- a/litellm/cost_calculator.py
+++ b/litellm/cost_calculator.py
@@ -2414,6 +2414,46 @@ class RealtimeAPITokenUsageProcessor(BaseTokenUsageProcessor):
         )
 
 
+_RESPONSES_WS_BILLABLE_EVENT_TYPES: Final = frozenset({"response.completed", "response.incomplete"})
+
+
+class _ResponsesWsEventResponse(BaseModel):
+    usage: Mapping[str, object] | None = None
+
+
+class _ResponsesWsEvent(BaseModel):
+    type: str = ""
+    response: _ResponsesWsEventResponse | None = None
+
+
+class ResponsesWebSocketTokenUsageProcessor(BaseTokenUsageProcessor):
+    @staticmethod
+    def collect_usage_from_responses_ws_results(
+        results: Sequence[Mapping[str, object]],
+    ) -> tuple[Usage, ...]:
+        events: Final = tuple(_ResponsesWsEvent.model_validate(result) for result in results)
+        return tuple(
+            ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(  # pyright: ignore[reportPrivateUsage]  # same shared transform the realtime processor uses
+                event.response.usage
+            )
+            for event in events
+            if event.type in _RESPONSES_WS_BILLABLE_EVENT_TYPES
+            and event.response is not None
+            and event.response.usage is not None
+        )
+
+    @staticmethod
+    def collect_and_combine_usage_from_responses_ws_results(
+        results: Sequence[Mapping[str, object]],
+    ) -> Usage:
+        collected_usage_objects: Final = ResponsesWebSocketTokenUsageProcessor.collect_usage_from_responses_ws_results(
+            results
+        )
+        return ResponsesWebSocketTokenUsageProcessor.combine_usage_objects(
+            list(collected_usage_objects)  # mutable-ok: combine_usage_objects requires a list parameter
+        )
+
+
 _TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed"
 
 
diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py
index ca2cca5360f..e3c94866985 100644
--- a/litellm/litellm_core_utils/litellm_logging.py
+++ b/litellm/litellm_core_utils/litellm_logging.py
@@ -48,6 +48,7 @@ from litellm.constants import (
 )
 from litellm.cost_calculator import (
     RealtimeAPITokenUsageProcessor,
+    ResponsesWebSocketTokenUsageProcessor,
     _select_model_name_for_cost_calc,
 )
 from litellm.exceptions import (
@@ -2028,6 +2029,17 @@ class Logging(LiteLLMLoggingBaseClass):
                 results=result,
             )
 
+        elif self.call_type == CallTypes.aresponses_websocket.value and isinstance(result, list):  # pyright: ignore[reportUnknownMemberType]  # Logging.call_type is untyped
+            combined_ws_usage: Final = (
+                ResponsesWebSocketTokenUsageProcessor.collect_and_combine_usage_from_responses_ws_results(
+                    results=result  # pyright: ignore[reportUnknownArgumentType]  # raw event dicts from the WS stream
+                )
+            )
+            logging_result = LiteLLMRealtimeStreamLoggingObject(
+                usage=combined_ws_usage,
+                results=result,  # pyright: ignore[reportUnknownArgumentType]  # raw event dicts from the WS stream
+            )
+
         elif (
             self.call_type == CallTypes.llm_passthrough_route.value
             or self.call_type == CallTypes.allm_passthrough_route.value
diff --git a/litellm/types/utils.py b/litellm/types/utils.py
index 9b5fb08a45f..40a6d77482f 100644
--- a/litellm/types/utils.py
+++ b/litellm/types/utils.py
@@ -580,6 +580,7 @@ CallTypesLiteral = Literal[
     "search",
     "asearch",
     "_arealtime",
+    "_aresponses_websocket",
     "create_batch",
     "acreate_batch",
     "create_file",
diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
index 0fdca755685..b6366bc803e 100644
--- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
+++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
@@ -22,7 +22,13 @@ from litellm.litellm_core_utils.litellm_logging import (
     _get_status_fields,
     set_callbacks,
 )
-from litellm.types.utils import ModelResponse, TextCompletionResponse
+from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse
+from litellm.types.utils import (
+    CallTypes,
+    LiteLLMRealtimeStreamLoggingObject,
+    ModelResponse,
+    TextCompletionResponse,
+)
 
 
 @pytest.fixture
@@ -6393,6 +6399,90 @@ async def test_prompt_hook_injection_marker_recorded_for_every_surface(logging_o
     assert pre_choice["metadata"]["litellm_gateway_injected_cache"] == ""
 
 
+def _responses_ws_logging_obj() -> LitellmLogging:
+    return LitellmLogging(
+        model="gpt-4o",
+        messages=[],
+        stream=False,
+        call_type=CallTypes.aresponses_websocket.value,
+        start_time=time.time(),
+        litellm_call_id="responses-ws-usage-test",
+        function_id="responses-ws-usage-test",
+    )
+
+
+def test_normalize_logging_result_extracts_usage_for_responses_websocket(monkeypatch):
+    """LIT-6512: native /v1/responses WebSocket sessions logged $0 spend because the usage
+    carried by stored response.completed events was never extracted. The session must cost
+    exactly what the same usage costs over HTTP /v1/responses, discounts included."""
+    monkeypatch.setattr(litellm, "cost_discount_config", {"openai": 0.5})
+    logging_obj = _responses_ws_logging_obj()
+    events = [
+        {"type": "response.created", "response": {}},
+        {
+            "type": "response.completed",
+            "response": {"usage": {"input_tokens": 100, "output_tokens": 40, "total_tokens": 140}},
+        },
+        {
+            "type": "response.completed",
+            "response": {"usage": {"input_tokens": 60, "output_tokens": 10, "total_tokens": 70}},
+        },
+    ]
+
+    normalized = logging_obj.normalize_logging_result(result=events)
+
+    assert isinstance(normalized, LiteLLMRealtimeStreamLoggingObject)
+    assert normalized.usage.prompt_tokens == 160
+    assert normalized.usage.completion_tokens == 50
+
+    ws_cost = litellm.completion_cost(
+        completion_response=normalized,
+        model="gpt-4o",
+        call_type=CallTypes.aresponses_websocket.value,
+        custom_llm_provider="openai",
+    )
+    http_cost = litellm.completion_cost(
+        completion_response=ResponsesAPIResponse(
+            id="resp-6512",
+            created_at=1700000000,
+            output=[],
+            usage=ResponseAPIUsage(input_tokens=160, output_tokens=50, total_tokens=210),
+        ),
+        model="gpt-4o",
+        call_type=CallTypes.aresponses.value,
+        custom_llm_provider="openai",
+    )
+    assert ws_cost > 0
+    assert ws_cost == http_cost
+
+
+def test_normalize_logging_result_bills_incomplete_responses_websocket_turns():
+    """LIT-6512: a turn cut short by max_output_tokens ends in response.incomplete, which
+    OpenAI bills, so its usage counts toward the session like a completed turn."""
+    events = [
+        {
+            "type": "response.created",
+            "response": {"usage": {"input_tokens": 999, "output_tokens": 999, "total_tokens": 1998}},
+        },
+        {
+            "type": "response.incomplete",
+            "response": {"usage": {"input_tokens": 15, "output_tokens": 16, "total_tokens": 31}},
+        },
+        {
+            "type": "response.completed",
+            "response": {"usage": {"input_tokens": 40, "output_tokens": 4, "total_tokens": 44}},
+        },
+        {"type": "response.failed", "response": {"usage": None}},
+    ]
+
+    normalized = _responses_ws_logging_obj().normalize_logging_result(result=events)
+
+    assert isinstance(normalized, LiteLLMRealtimeStreamLoggingObject)
+    assert normalized.usage.prompt_tokens == 55
+    assert normalized.usage.completion_tokens == 20
+    assert normalized.usage.total_tokens == 75
+
+
 def test_get_standard_logging_object_payload_reads_overhead_from_logging_obj_for_dict_results(logging_obj):
     """LIT-5466: /v1/messages returns a plain dict with no _hidden_params, so the overhead
     recorded on the logging object must reach hidden_params.litellm_overhead_time_ms (SpendLogs)."""

From 9d164fa34150ac7dc0551d9efec16cdb27848c5c Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Tue, 8 Sep 2026 13:24:15 -0700
Subject: [PATCH 68/85] bump: litellm-proxy-extras 0.4.94 -> 0.4.95, litellm
 1.101.0 -> 1.102.0

---
 litellm-proxy-extras/pyproject.toml | 4 ++--
 pyproject.toml                      | 6 +++---
 uv.lock                             | 6 +++---
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml
index 82d31fec373..91b4e4a7ba1 100644
--- a/litellm-proxy-extras/pyproject.toml
+++ b/litellm-proxy-extras/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.94"
+version = "0.4.95"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.94"
+version = "0.4.95"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",
diff --git a/pyproject.toml b/pyproject.toml
index af35c77d259..706b14b8a04 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "litellm"
-version = "1.101.0"
+version = "1.102.0"
 description = "Library to easily interface with LLM API providers"
 readme = "README.md"
 requires-python = ">=3.10, <3.15"
@@ -67,7 +67,7 @@ proxy = [
     "azure-identity>=1.25.2,<2.0",
     "azure-storage-blob>=12.28.0,<13.0",
     "mcp>=1.28.1,<2.0",
-    "litellm-proxy-extras==0.4.94",
+    "litellm-proxy-extras==0.4.95",
     "litellm-enterprise==0.1.65",
     "RestrictedPython>=8.5,<9.0",
     "rich>=13.9.4,<14.0",
@@ -328,7 +328,7 @@ members = ["enterprise", "litellm-proxy-extras"]
 profile = "black"
 
 [tool.commitizen]
-version = "1.101.0"
+version = "1.102.0"
 version_files = [
     "pyproject.toml:^version",
 ]
diff --git a/uv.lock b/uv.lock
index 89205cd9527..12dc2c69d23 100644
--- a/uv.lock
+++ b/uv.lock
@@ -10,7 +10,7 @@ resolution-markers = [
 ]
 
 [options]
-exclude-newer = "2026-09-02T16:58:34.594994Z"
+exclude-newer = "2026-09-05T20:24:07.535116Z"
 exclude-newer-span = "P3D"
 
 [manifest]
@@ -4358,7 +4358,7 @@ wheels = [
 
 [[package]]
 name = "litellm"
-version = "1.101.0"
+version = "1.102.0"
 source = { editable = "." }
 dependencies = [
     { name = "aiohttp" },
@@ -4776,7 +4776,7 @@ source = { editable = "enterprise" }
 
 [[package]]
 name = "litellm-proxy-extras"
-version = "0.4.94"
+version = "0.4.95"
 source = { editable = "litellm-proxy-extras" }
 
 [[package]]

From 4a3a78c256bab366597a76d44930b323bce6335b Mon Sep 17 00:00:00 2001
From: tin-berri <tin@berri.ai>
Date: Tue, 8 Sep 2026 13:28:57 -0700
Subject: [PATCH 69/85] feat(complexity_router): rebalance heuristic weights in
 the dashboard and grade custom dimensions by match count (#40205)

The Advanced scoring editor now lists built-in and custom dimensions together. Editing any weight holds it and rescales the others proportionally so the vector totals 1.00, and Save stores those explicit values. The backend scores exactly what is stored, with no runtime normalization, so routers nobody edits keep their weights.

CustomDimension gains an opt-in scoring_mode. match_count scores 0, 0.5 or 1 by distinct matcher hits; the default stays binary. The tuning fingerprint omits a binary scoring_mode, so routers written before this change keep their recorded baseline and the upgrade does not consume the free heuristic-v1 tuning slot.
---
 .../complexity_router/README.md               |  12 +-
 .../complexity_router/complexity_router.py    |  38 ++++-
 .../complexity_router/config.py               |  13 +-
 .../auto_router_tuning_baseline.py            |  12 +-
 .../router_strategy/test_complexity_router.py | 148 +++++++++++++-----
 .../test_auto_router_tuning_baseline.py       |  73 ++++++++-
 .../add_model/ClassificationMethodConfig.tsx  |   5 +-
 .../add_model/ComplexityRouterConfig.test.tsx |   6 +-
 .../add_model/ComplexityRouterConfig.tsx      |   6 +
 .../add_model/CustomDimensionRows.tsx         | 136 ++++++++++++++++
 ...euristicScoringConfig.integration.test.tsx |  90 +++++++++++
 .../add_model/HeuristicScoringConfig.test.tsx |  14 +-
 .../add_model/HeuristicScoringConfig.tsx      |  85 ++++++++--
 .../add_model/add_auto_router_tab.tsx         |   4 +
 .../build_complexity_router_config.test.ts    |  47 +++++-
 .../build_complexity_router_config.ts         |  17 +-
 .../add_model/custom_dimensions.test.ts       |  41 +++++
 .../components/add_model/custom_dimensions.ts |  54 +++++++
 .../add_model/heuristic_scoring_knobs.test.ts | 119 ++++++++++++++
 .../add_model/heuristic_scoring_knobs.ts      |  97 +++++++++++-
 .../src/components/add_model/tier_rows.ts     |   1 +
 ...d_updated_complexity_router_config.test.ts |  49 ++++++
 .../edit_auto_router_modal.tsx                |  17 +-
 .../src/lib/autorouter_presets.test.ts        |  28 ++++
 .../src/lib/autorouter_presets.ts             |  12 ++
 ui/litellm-dashboard/src/lib/http/schema.d.ts |   9 +-
 26 files changed, 1047 insertions(+), 86 deletions(-)
 create mode 100644 ui/litellm-dashboard/src/components/add_model/CustomDimensionRows.tsx
 create mode 100644 ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.integration.test.tsx
 create mode 100644 ui/litellm-dashboard/src/components/add_model/custom_dimensions.test.ts
 create mode 100644 ui/litellm-dashboard/src/components/add_model/custom_dimensions.ts

diff --git a/litellm/router_strategy/complexity_router/README.md b/litellm/router_strategy/complexity_router/README.md
index 88ed374dd3f..d605b43e42a 100644
--- a/litellm/router_strategy/complexity_router/README.md
+++ b/litellm/router_strategy/complexity_router/README.md
@@ -207,17 +207,27 @@ custom_dimensions:
   - name: sqlMigration
     weight: 0.7
     patterns: ['\b(create|alter|drop)\s{1,4}table\b']
+  - name: dataPipeline
+    weight: 0.4
+    scoring_mode: match_count
+    keywords: [airflow, dbt, snowflake]
 ```
 
 Each dimension contributes its weight once when any matcher hits the current ask. Repeated matches do not increase it. The built-in score and tier boundaries are unchanged, and the total score is not renormalized. Keywords use the existing case-insensitive word-boundary and CJK rules. Regexes search the first 2048 characters case-insensitively and compile during configuration validation and router initialization, never per request
 
+`scoring_mode` is optional and defaults to `binary`, the behavior above. `match_count` grades the dimension by how many distinct matchers hit: none scores 0 and emits no signal, one scores half the weight, two or more score the full weight. Repeated occurrences of one matcher never raise the count, keywords are distinct case-insensitively, patterns are distinct by source, and a keyword and a pattern are always distinct from each other. Matching stops as soon as the selected mode's maximum is reached, so a binary dimension still stops at its first hit. Existing configurations without the field keep binary scoring and the same tuning fingerprint, so the field only counts as a tuning change when set to `match_count`
+
+### Weights through the API versus the dashboard
+
+The API and YAML store exactly the weights written. A `dimension_weights` map and inline custom weights are read literally, missing recognized built-in names score zero, and nothing renormalizes the vector, so a total other than 1 is legal and scores accordingly. The dashboard's heuristic scoring editor is the one place that rebalances: editing one weight there holds it and redistributes the remainder across the other active dimensions in the draft, then Save sends the resulting explicit values, which the backend stores and scores as written. Opening a router, applying a preset, editing matchers, changing `scoring_mode`, or saving unrelated fields never normalizes existing weights
+
 Only `heuristic`, `heuristic_first` and `hybrid` accept custom dimensions. Each name must be a unique ASCII identifier starting with a letter, at most 64 characters, and cannot reuse a built-in dimension name or a key in `dimension_weights`. Set its weight inline, greater than zero and at most one
 
 Patterns are checked at configuration time against a grammar whose worst case stays a few milliseconds on 2048 characters. Every quantifier needs an explicit upper bound of at most 64 and must repeat a single character or character class, so `\s{1,4}` is accepted while `\s+`, `(a|aa){0,12}` and `(?:ab){0,64}` are refused. Backreferences, lookarounds, atomic groups and possessive quantifiers are refused as well. Each pattern is then costed: alternation branches and repeat lengths multiply the ways the engine can retry, and every later piece of the pattern is charged once per path that can reach it, so `a?a?a?a?a?a?a?a?` followed by a long fixed tail is refused even though each quantifier is small. The budget is 2048 work units per pattern and 8192 across the router. An invalid or over-budget pattern fails the write with a message naming the pattern and the rule it broke
 
 Limits are 16 dimensions, 32 combined keywords/patterns per dimension, 256 characters per matcher and 4096 matcher characters per dimension. Matching runs inline on the request path with no timeout and no worker thread, because the grammar is what bounds the cost. These are routing hints, not security enforcement rules
 
-The existing heuristic-v1 tuning quota covers custom dimensions and their weights: one changed router without an auto-router license, unlimited with the entitlement. Omitting `custom_dimensions` preserves existing scoring. Routing decisions and spend logs include signals such as `custom (sqlMigration)` without recording the configured pattern or matched text. The field is configured through YAML or the model API; this change adds no dashboard editor
+The existing heuristic-v1 tuning quota covers custom dimensions, their weights and their scoring mode: one changed router without an auto-router license, unlimited with the entitlement. Omitting `custom_dimensions` preserves existing scoring. Routing decisions and spend logs include signals such as `custom (sqlMigration)` without recording the configured pattern or matched text. The field is configured through YAML, the model API, or the dashboard's heuristic scoring editor
 
 ## Usage
 
diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py
index c8644f52c57..ae1f82fc40b 100644
--- a/litellm/router_strategy/complexity_router/complexity_router.py
+++ b/litellm/router_strategy/complexity_router/complexity_router.py
@@ -20,7 +20,7 @@ import random
 import re
 import time
 from collections.abc import Callable, Iterator, Mapping, Sequence
-from itertools import accumulate, islice, takewhile
+from itertools import accumulate, chain, islice, takewhile
 from threading import Lock
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast
@@ -82,6 +82,7 @@ from .config import (
     ClassificationRubric,
     ComplexityRouterConfig,
     ComplexityTier,
+    CustomDimension,
     TierDefinition,
 )
 from .stall_detector import detect_stalled_task
@@ -879,6 +880,15 @@ class DimensionScore:
         self.signal = signal
 
 
+class _CustomDimensionMatchers(NamedTuple):
+    """One custom dimension's distinct matchers and the number of hits that saturates its score."""
+
+    dimension: CustomDimension
+    keywords: tuple[str, ...]
+    patterns: tuple[re.Pattern[str], ...]
+    saturation: int
+
+
 class KeywordOverride(NamedTuple):
     """A keyword_tier_rules match: the winning tier and, on the lexical path, the keyword that fired."""
 
@@ -1121,7 +1131,12 @@ class ComplexityRouter(CustomLogger):
         )
         self.simple_keywords = self.config.simple_keywords or DEFAULT_SIMPLE_KEYWORDS
         self._custom_dimensions = tuple(
-            (dimension, tuple(re.compile(pattern, re.IGNORECASE) for pattern in dimension.patterns))
+            _CustomDimensionMatchers(
+                dimension,
+                tuple(dict.fromkeys(keyword.lower() for keyword in dimension.keywords)),
+                tuple(re.compile(pattern, re.IGNORECASE) for pattern in dict.fromkeys(dimension.patterns)),
+                2 if dimension.scoring_mode == "match_count" else 1,
+            )
             for dimension in self.config.custom_dimensions
         )
         if self.config.has_custom_tiers:
@@ -1325,15 +1340,26 @@ class ComplexityRouter(CustomLogger):
         score: Final = score_high if match_count >= high_threshold else score_low
         return DimensionScore(name, score, f"{signal_label} ({detail})"), match_count
 
+    def _count_custom_hits(self, matchers: _CustomDimensionMatchers, user_text: str, scanned: str) -> int:
+        hits: Final = chain(
+            (self._keyword_matches(user_text, keyword) for keyword in matchers.keywords),
+            (pattern.search(scanned) is not None for pattern in matchers.patterns),
+        )
+        return sum(islice((1 for hit in hits if hit), matchers.saturation))
+
     def _score_custom_dimensions(self, prompt: str, user_text: str) -> tuple[tuple[DimensionScore, float], ...]:
         if not self._custom_dimensions:
             return ()
         scanned: Final = prompt[:CUSTOM_PATTERN_SCAN_CHARS]
         return tuple(
-            (DimensionScore(dimension.name, 1.0, f"custom ({dimension.name})"), dimension.weight)
-            for dimension, patterns in self._custom_dimensions
-            if any(self._keyword_matches(user_text, keyword) for keyword in dimension.keywords)
-            or any(pattern.search(scanned) is not None for pattern in patterns)
+            (
+                DimensionScore(
+                    matchers.dimension.name, hits / matchers.saturation, f"custom ({matchers.dimension.name})"
+                ),
+                matchers.dimension.weight,
+            )
+            for matchers in self._custom_dimensions
+            if (hits := self._count_custom_hits(matchers, user_text, scanned))
         )
 
     def _score_multi_step(self, text: str) -> DimensionScore:
diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py
index 3ec9f9b5394..37375d9727d 100644
--- a/litellm/router_strategy/complexity_router/config.py
+++ b/litellm/router_strategy/complexity_router/config.py
@@ -667,6 +667,14 @@ class CustomDimension(BaseModel):
     weight: float = Field(gt=0, le=1, allow_inf_nan=False)
     keywords: tuple[Annotated[str, Field(min_length=1, max_length=256)], ...] = Field(default=(), max_length=32)
     patterns: tuple[Annotated[str, Field(min_length=1, max_length=256)], ...] = Field(default=(), max_length=32)
+    scoring_mode: Literal["binary", "match_count"] = Field(
+        default="binary",
+        description=(
+            "'binary' scores 1 when any matcher hits. 'match_count' scores 0.5 when one distinct matcher hits and 1 "
+            "when two or more do; repeated occurrences of one matcher never raise it. Keywords are distinct "
+            "case-insensitively, patterns by source, and a keyword and a pattern are always distinct from each other."
+        ),
+    )
 
     @model_validator(mode="after")
     def _validate_matchers(self) -> "CustomDimension":
@@ -794,8 +802,9 @@ class ComplexityRouterConfig(BaseModel):
         default=(),
         max_length=16,
         description=(
-            "Named binary dimensions added to the heuristic-v1 score. Each contributes its inline weight once "
-            "when any keyword matches the current ask or a case-insensitive regex matches its first 2048 characters. "
+            "Named dimensions added to the heuristic-v1 score. Each contributes its inline weight once "
+            "when any keyword matches the current ask or a case-insensitive regex matches its first 2048 characters; "
+            "scoring_mode 'match_count' instead grades half weight for one distinct matcher and full for two or more. "
             "Regex quantifiers repeat one character or class at most 64 times. Unbounded quantifiers, repeated groups, "
             "backreferences and lookarounds are rejected. Conservative work limits include alternation paths, "
             "repeat lengths and subsequent matching: 2048 units per pattern, 8192 across the router. "
diff --git a/litellm/router_utils/auto_router_tuning_baseline.py b/litellm/router_utils/auto_router_tuning_baseline.py
index 74f7b82389a..9699ab886b9 100644
--- a/litellm/router_utils/auto_router_tuning_baseline.py
+++ b/litellm/router_utils/auto_router_tuning_baseline.py
@@ -52,7 +52,17 @@ def tuning_fingerprint(complexity_router_config: object) -> str | None:
     supplied: Final = ((_TUNING_FIELD_SET - frozenset(("tier_model_configs",))) & frozenset(raw)) | (
         frozenset(("tier_model_configs",)) if validated.tier_model_configs else frozenset()
     )
-    payload: Final = validated.model_dump(mode="json", include=supplied)
+    payload: Final = validated.model_dump(
+        mode="json",
+        include=supplied,
+        exclude={
+            "custom_dimensions": {
+                index: {"scoring_mode"}
+                for index, dimension in enumerate(validated.custom_dimensions)
+                if dimension.scoring_mode == "binary"
+            }
+        },
+    )
     return hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
 
 
diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py
index 5b1d8562abd..b0f2a65bb9b 100644
--- a/tests/test_litellm/router_strategy/test_complexity_router.py
+++ b/tests/test_litellm/router_strategy/test_complexity_router.py
@@ -826,6 +826,8 @@ class TestCustomDimensions:
             pytest.param({"keywords": ["x"] * 32, "patterns": ["y"]}, {}, id="combined-matcher-count"),
             pytest.param({"keywords": ["x" * 256] * 17}, {}, id="matcher-character-budget"),
             pytest.param({"unknown": True}, {}, id="extra-field"),
+            pytest.param({"scoring_mode": "graded"}, {}, id="unknown-scoring-mode"),
+            pytest.param({"scoring_mode": None}, {}, id="null-scoring-mode"),
         ],
     )
     def test_custom_dimension_invalid_configuration_rejected(
@@ -878,43 +880,125 @@ class TestCustomDimensions:
             )
 
     @pytest.mark.asyncio
-    @pytest.mark.parametrize("current_ask", ("Hello!", "orbitmesh"))
+    @pytest.mark.parametrize("scoring_mode", ("binary", "match_count"))
+    @pytest.mark.parametrize("current_ask", ("Hello!", "orbitmesh", "orbitmesh fluxgate"))
     async def test_custom_dimensions_public_hook_scores_only_current_ask(
-        self, mock_router_instance: MagicMock, current_ask: str
+        self, mock_router_instance: MagicMock, current_ask: str, scoring_mode: str
     ) -> None:
         router: Final = ComplexityRouter(
             "test-router",
             mock_router_instance,
             {
                 "tiers": {"SIMPLE": "cheap", "MEDIUM": "mid", "COMPLEX": "strong", "REASONING": "top"},
-                "custom_dimensions": [{"name": "internalFrameworks", "weight": 0.7, "keywords": ["orbitmesh"]}],
+                "dimension_weights": {},
+                "custom_dimensions": [
+                    {
+                        "name": "internalFrameworks",
+                        "weight": 0.8,
+                        "keywords": ["orbitmesh", "fluxgate"],
+                        "scoring_mode": scoring_mode,
+                    }
+                ],
             },
         )
         result: Final = await router.async_pre_routing_hook(
             model="test-router",
             request_kwargs={},
             messages=[
-                {"role": "system", "content": "orbitmesh"},
-                {"role": "user", "content": "orbitmesh"},
-                {"role": "assistant", "content": "orbitmesh is ready"},
+                {"role": "system", "content": "orbitmesh fluxgate"},
+                {"role": "user", "content": "orbitmesh fluxgate"},
+                {"role": "assistant", "content": "orbitmesh fluxgate is ready"},
                 {"role": "user", "content": current_ask},
-                {"role": "tool", "tool_call_id": "previous", "content": "orbitmesh"},
+                {"role": "tool", "tool_call_id": "previous", "content": "orbitmesh fluxgate"},
             ],
         )
         assert result is not None
         assert result.routing_decision is not None
-        assert ("custom (internalFrameworks)" in result.routing_decision["signals"]) is (current_ask == "orbitmesh")
-        assert result.model == ("top" if current_ask == "orbitmesh" else "cheap")
+        expected_score: Final = (
+            0.0
+            if current_ask == "Hello!"
+            else 0.4
+            if scoring_mode == "match_count" and current_ask == "orbitmesh"
+            else 0.8
+        )
+        assert result.routing_decision["score"] == expected_score
+        assert ("custom (internalFrameworks)" in result.routing_decision["signals"]) is (expected_score > 0)
+        assert result.model == ("cheap" if expected_score == 0 else "strong" if expected_score == 0.4 else "top")
         assert "orbitmesh" not in " ".join(result.routing_decision["signals"])
 
-    def test_custom_patterns_scan_only_the_first_2048_characters(self, mock_router_instance: MagicMock) -> None:
+    @pytest.mark.parametrize("scoring_mode", ("binary", "match_count"))
+    def test_custom_patterns_scan_only_the_first_2048_characters(
+        self, mock_router_instance: MagicMock, scoring_mode: str
+    ) -> None:
         router: Final = ComplexityRouter(
             "test-router",
             mock_router_instance,
-            {"custom_dimensions": [{"name": "late", "weight": 0.7, "patterns": [r"zzz{1,3}"]}]},
+            {
+                "custom_dimensions": [
+                    {
+                        "name": "late",
+                        "weight": 0.7,
+                        "patterns": [r"zzz{1,3}", r"yyy{1,3}"],
+                        "scoring_mode": scoring_mode,
+                    }
+                ]
+            },
         )
+        baseline: Final = ComplexityRouter("test-router", mock_router_instance)
         assert "custom (late)" in router.classify("a" * 2040 + " zzz")[2]
         assert "custom (late)" not in router.classify("a" * 2048 + " zzz")[2]
+        second_hit_past_the_bound: Final = "yyy " + "a" * 2044 + " zzz"
+        contribution: Final = (
+            router.classify(second_hit_past_the_bound)[1] - baseline.classify(second_hit_past_the_bound)[1]
+        )
+        assert contribution == pytest.approx(0.7 if scoring_mode == "binary" else 0.35)
+
+    @pytest.mark.parametrize(
+        "prompt,expected_score",
+        [
+            pytest.param("Hello!", 0.0, id="no-hit"),
+            pytest.param("orbitmesh orbitmesh ORBITMESH again", 0.5, id="one-keyword-repeated"),
+            pytest.param("create table a; CREATE TABLE b; create  table c", 0.5, id="one-pattern-repeated"),
+            pytest.param("orbitmesh and fluxgate", 1.0, id="two-keywords"),
+            pytest.param("orbitmesh then create table t", 1.0, id="keyword-plus-pattern"),
+            pytest.param("create table a; alter table b", 1.0, id="two-patterns"),
+            pytest.param("orbitmesh fluxgate create table a alter table b", 1.0, id="all-matchers"),
+        ],
+    )
+    def test_match_count_grades_distinct_matchers(
+        self, mock_router_instance: MagicMock, prompt: str, expected_score: float
+    ) -> None:
+        dimension: Final = {
+            "name": "graded",
+            "weight": 0.6,
+            "keywords": ["orbitmesh", "ORBITMESH", "fluxgate"],
+            "patterns": [r"\bcreate\s{1,4}table\b", r"\bcreate\s{1,4}table\b", r"\balter\s{1,4}table\b"],
+        }
+        baseline: Final = ComplexityRouter("test-router", mock_router_instance)
+        binary: Final = ComplexityRouter("test-router", mock_router_instance, {"custom_dimensions": [dimension]})
+        graded: Final = ComplexityRouter(
+            "test-router",
+            mock_router_instance,
+            {"custom_dimensions": [{**dimension, "scoring_mode": "match_count"}]},
+        )
+        _, baseline_score, baseline_signals = baseline.classify(prompt)
+        _, binary_score, binary_signals = binary.classify(prompt)
+        _, graded_score, graded_signals = graded.classify(prompt)
+        assert graded_score == pytest.approx(baseline_score + 0.6 * expected_score)
+        assert binary_score == pytest.approx(baseline_score + (0.6 if expected_score else 0.0))
+        expected_signals: Final = [*baseline_signals, *(["custom (graded)"] if expected_score else [])]
+        assert graded_signals == expected_signals
+        assert binary_signals == expected_signals
+
+    def test_scoring_mode_round_trips_and_defaults_to_binary(self) -> None:
+        dimension: Final = {"name": "graded", "weight": 0.6, "keywords": ["orbitmesh"]}
+        legacy: Final = ComplexityRouterConfig.model_validate({"custom_dimensions": [dimension]})
+        graded: Final = ComplexityRouterConfig.model_validate(
+            {"custom_dimensions": [{**dimension, "scoring_mode": "match_count"}]}
+        )
+        assert legacy.custom_dimensions[0].scoring_mode == "binary"
+        assert graded.model_dump(mode="json")["custom_dimensions"][0]["scoring_mode"] == "match_count"
+        assert ComplexityRouterConfig.model_validate(graded.model_dump(mode="json")) == graded
 
     def test_custom_dimensions_router_wide_regex_work_is_capped(self) -> None:
         heavy: Final = {"weight": 0.5, "patterns": ["a?" * 8 + "z"]}
@@ -1511,6 +1595,7 @@ class TestRouterComplexityDeploymentMethods:
     def test_the_shipped_rubric_and_default_prompt_stay_free(self) -> None:
         """Only an operator-written prompt is gated: picking a shipped rubric preset, or writing no
         prompt at all, leaves a router unmetered, so several of them register under a ceiling of one."""
+
         def rubric(model_name: str, model_id: str, preset: str | None) -> dict[str, object]:
             llm_config: dict[str, object] = {"model": "gpt-4o-mini"}
             if preset is not None:
@@ -1647,6 +1732,7 @@ class TestRouterComplexityDeploymentMethods:
     def test_renaming_built_in_tiers_is_not_a_custom_tier_set(self) -> None:
         """tier_labels renames the built-in ladder without defining one, so it stays ungated: two such
         routers register under a ceiling of one."""
+
         def labeled(model_name: str, model_id: str) -> dict[str, object]:
             row = self._router_row(model_name, model_id, "heuristic")
             row["litellm_params"]["complexity_router_config"]["tier_labels"] = {"SIMPLE": "Cheap", "MEDIUM": "Standard"}
@@ -2520,9 +2606,7 @@ class TestLLMClassifier:
         assert outcome.classifier_cost == pytest.approx(1.35e-05)
 
     @pytest.mark.asyncio
-    async def test_aclassify_timeout_does_not_inherit_router_retries_or_fallbacks(
-        self, llm_classifier_config
-    ):
+    async def test_aclassify_timeout_does_not_inherit_router_retries_or_fallbacks(self, llm_classifier_config):
         real_router = Router(
             model_list=[
                 {
@@ -2565,9 +2649,7 @@ class TestLLMClassifier:
         assert real_router.total_calls["openai/mock-backup-classifier"] == 0
 
     @pytest.mark.asyncio
-    async def test_aclassify_enforces_total_classifier_deadline(
-        self, mock_router_instance, llm_classifier_config
-    ):
+    async def test_aclassify_enforces_total_classifier_deadline(self, mock_router_instance, llm_classifier_config):
         cancelled = asyncio.Event()
 
         async def slow_classifier(**_kwargs: object) -> None:
@@ -12414,9 +12496,7 @@ class TestTierHealthFailover:
                     llm_provider="",
                 )
             filtered = (*cooling, *blocked, *excluded)
-            healthy = [
-                {"model_name": model, "model_info": {"id": i}} for i in ids_by_model[model] if i not in filtered
-            ]
+            healthy = [{"model_name": model, "model_info": {"id": i}} for i in ids_by_model[model] if i not in filtered]
             if not healthy:
                 raise RouterRateLimitError(
                     model=model, cooldown_time=60.0, enable_pre_call_checks=False, cooldown_list=[]
@@ -12845,9 +12925,7 @@ class TestTierHealthFailover:
         assert all(probed is not request_kwargs for probed in router.litellm_router_instance.probed_kwargs)
 
     @pytest.mark.asyncio
-    async def test_a_peer_whose_every_deployment_is_over_its_rpm_is_not_a_failover_target(
-        self, mock_router_instance
-    ):
+    async def test_a_peer_whose_every_deployment_is_over_its_rpm_is_not_a_failover_target(self, mock_router_instance):
         """RPM exhaustion is its own verdict from the owner (RouterRateLimitErrorBasic). A peer
         in that state would be rejected downstream, so it cannot be the substitute."""
         from litellm.types.router import RouterRateLimitErrorBasic
@@ -12880,9 +12958,7 @@ class TestTierHealthFailover:
         assert {r.model for r in results} == {"live-c"}
 
     @pytest.mark.asyncio
-    async def test_the_probe_forwards_input_so_window_checks_run_on_input_only_surfaces(
-        self, mock_router_instance
-    ):
+    async def test_the_probe_forwards_input_so_window_checks_run_on_input_only_surfaces(self, mock_router_instance):
         """The Responses API carries its prompt as `input`, never as messages. The owner only
         runs its context-window pre-call check when one of them is present, so dropping `input`
         would silently skip window filtering on that whole surface."""
@@ -12908,9 +12984,7 @@ class TestTierHealthFailover:
         ), "the eligibility probe must forward `input` to the owner"
 
     @pytest.mark.asyncio
-    async def test_a_group_the_router_has_no_deployment_for_is_not_a_failover_target(
-        self, mock_router_instance
-    ):
+    async def test_a_group_the_router_has_no_deployment_for_is_not_a_failover_target(self, mock_router_instance):
         """The owner answers an unconfigured group with BadRequestError. Reading that as live
         would both skip failover off it and let it be chosen as a substitute."""
         router = self._router(
@@ -13099,9 +13173,7 @@ class TestClassifierVision:
         routed as default_fallback on text the request never contained.
         """
         router = self._router(mock_router_instance, vision={"enabled": True})
-        response = await router.async_pre_routing_hook(
-            model="m", request_kwargs={}, messages=self._turn(IMG_PART)
-        )
+        response = await router.async_pre_routing_hook(model="m", request_kwargs={}, messages=self._turn(IMG_PART))
         assert response.routing_decision["cause"] == "llm_classifier"
         assert response.model == "t-complex"
         assert [block["type"] for block in self._classifier_user_content(mock_router_instance)] == [
@@ -13112,9 +13184,7 @@ class TestClassifierVision:
     @pytest.mark.asyncio
     async def test_image_only_turn_still_falls_back_when_vision_is_off(self, mock_router_instance):
         router = self._router(mock_router_instance, vision={"enabled": False})
-        response = await router.async_pre_routing_hook(
-            model="m", request_kwargs={}, messages=self._turn(IMG_PART)
-        )
+        response = await router.async_pre_routing_hook(model="m", request_kwargs={}, messages=self._turn(IMG_PART))
         assert response.routing_decision["cause"] == "default_fallback"
         mock_router_instance.acompletion.assert_not_awaited()
 
@@ -13184,9 +13254,7 @@ class TestClassifierVision:
         makes the image the only variable; a margin loose enough to leave the score undecided
         would pass whether or not the guard exists.
         """
-        router = self._router(
-            mock_router_instance, vision={"enabled": True}, classifier_type=classifier_type, **extra
-        )
+        router = self._router(mock_router_instance, vision={"enabled": True}, classifier_type=classifier_type, **extra)
         response = await router.async_pre_routing_hook(
             model="m", request_kwargs={}, messages=self._turn({"type": "text", "text": "what is this"}, IMG_PART)
         )
@@ -13200,9 +13268,7 @@ class TestClassifierVision:
         self, mock_router_instance, classifier_type, extra, short_circuit_cause
     ):
         """The negative class: same router, same text, no image, and the scorer still decides."""
-        router = self._router(
-            mock_router_instance, vision={"enabled": True}, classifier_type=classifier_type, **extra
-        )
+        router = self._router(mock_router_instance, vision={"enabled": True}, classifier_type=classifier_type, **extra)
         response = await router.async_pre_routing_hook(
             model="m", request_kwargs={}, messages=[{"role": "user", "content": "what is this"}]
         )
diff --git a/tests/test_litellm/router_utils/test_auto_router_tuning_baseline.py b/tests/test_litellm/router_utils/test_auto_router_tuning_baseline.py
index f686a62db76..75c115cd3ab 100644
--- a/tests/test_litellm/router_utils/test_auto_router_tuning_baseline.py
+++ b/tests/test_litellm/router_utils/test_auto_router_tuning_baseline.py
@@ -7,6 +7,7 @@ from typing import Final
 
 import pytest
 
+from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig
 from litellm.router_utils.auto_router_tuning_baseline import (
     DEFAULT_TUNING_FINGERPRINT,
     HEURISTIC_V1_TUNING_FIELDS,
@@ -21,6 +22,33 @@ from litellm.router_utils.auto_router_tuning_baseline import (
 
 _TIERS = {"SIMPLE": "cheap", "MEDIUM": "mid", "COMPLEX": "strong", "REASONING": "top"}
 _ALT_TIERS = {**_TIERS, "COMPLEX": "other-strong"}
+_KEYWORD_DIMENSION: Final = {"name": "internalFrameworks", "weight": 0.7, "keywords": ["orbitmesh"]}
+_HISTORICAL_FINGERPRINTS: Final = (
+    ({}, "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a"),
+    (
+        {"custom_dimensions": [_KEYWORD_DIMENSION]},
+        "b5c3c3f3be6341a8a16148d68d9067e03f94ed01a0bbfcde7955763042744372",
+    ),
+    (
+        {"custom_dimensions": [{"name": "sqlDdl", "weight": 0.4, "patterns": [r"\bCREATE\s{1,4}TABLE\b"]}]},
+        "814ce0017fc7f60a160b262f658d910e9bdf784e6139a4ba4f1e2657aa203950",
+    ),
+    (
+        {
+            "tiers": _TIERS,
+            "dimension_weights": {"codePresence": 0.3},
+            "custom_dimensions": [
+                {
+                    "name": "internalFrameworks",
+                    "weight": 0.2,
+                    "keywords": ["orbitmesh", "fluxgate"],
+                    "patterns": [r"\bALTER\s{1,4}TABLE\b"],
+                }
+            ],
+        },
+        "38970dc9224e265ab38c89674563d8d0537822591f9239b45251db6f5ca6cc39",
+    ),
+)
 
 
 def _router(
@@ -77,6 +105,27 @@ class TestTuningFingerprint:
     def test_explicit_empty_tier_model_configs_follow_omission(self) -> None:
         assert tuning_fingerprint({"tier_model_configs": {}}) == DEFAULT_TUNING_FINGERPRINT
 
+    @pytest.mark.parametrize(("config", "fingerprint"), _HISTORICAL_FINGERPRINTS)
+    def test_fingerprints_recorded_before_scoring_mode_existed_are_preserved(
+        self, config: Mapping[str, object], fingerprint: str
+    ) -> None:
+        """Literal hashes captured from the merged implementation at 9bc9104102, before CustomDimension.scoring_mode."""
+        assert tuning_fingerprint(config) == fingerprint
+
+    def test_binary_scoring_mode_hashes_like_its_absence(self) -> None:
+        historical: Final = tuning_fingerprint({"custom_dimensions": [_KEYWORD_DIMENSION]})
+        explicit: Final = tuning_fingerprint({"custom_dimensions": [{**_KEYWORD_DIMENSION, "scoring_mode": "binary"}]})
+        reserialized: Final = ComplexityRouterConfig.model_validate(
+            {"custom_dimensions": [_KEYWORD_DIMENSION]}
+        ).model_dump(mode="json", include={"custom_dimensions"})
+        assert reserialized["custom_dimensions"][0]["scoring_mode"] == "binary"
+        assert reserialized["custom_dimensions"][0]["patterns"] == []
+        assert historical == explicit == tuning_fingerprint(reserialized)
+        assert (
+            tuning_fingerprint({"custom_dimensions": [{**_KEYWORD_DIMENSION, "scoring_mode": "match_count"}]})
+            != historical
+        )
+
     def test_tier_model_overrides_change_the_fingerprint(self) -> None:
         plain = tuning_fingerprint({"tiers": {"SIMPLE": "x"}})
         with_override = tuning_fingerprint(
@@ -218,15 +267,18 @@ class TestQuota:
             is None
         )
 
-    def test_custom_dimension_add_edit_and_revert_share_one_quota_slot(self) -> None:
+    @pytest.mark.parametrize(
+        "edit",
+        [
+            pytest.param({"weight": 0.9}, id="weight"),
+            pytest.param({"scoring_mode": "match_count"}, id="scoring-mode"),
+        ],
+    )
+    def test_custom_dimension_add_edit_and_revert_share_one_quota_slot(self, edit: Mapping[str, object]) -> None:
         baselines: Final = snapshot_tuning_baselines(())
         original: Final = _router("a", {})
-        config: Final = {
-            "custom_dimensions": [{"name": "internalFrameworks", "weight": 0.7, "keywords": ["orbitmesh"]}]
-        }
-        edited_config: Final = {
-            "custom_dimensions": [{"name": "internalFrameworks", "weight": 0.9, "keywords": ["orbitmesh"]}]
-        }
+        config: Final = {"custom_dimensions": [_KEYWORD_DIMENSION]}
+        edited_config: Final = {"custom_dimensions": [{**_KEYWORD_DIMENSION, **edit}]}
         added: Final = _router("a", config)
         edited: Final = _router("a", edited_config)
         second: Final = _router("b", config)
@@ -240,6 +292,13 @@ class TestQuota:
         assert mutable_tuned_identities((original,), baselines) == frozenset()
         assert tuning_quota_violation(candidate=second, others=(original,), baselines=baselines, limit=1) is None
 
+    def test_graded_dimension_recorded_at_snapshot_is_its_own_baseline(self) -> None:
+        graded: Final = _router("a", {"custom_dimensions": [{**_KEYWORD_DIMENSION, "scoring_mode": "match_count"}]})
+        baselines: Final = snapshot_tuning_baselines((graded,))
+        assert mutable_tuned_identities((graded,), baselines) == frozenset()
+        reverted_to_binary: Final = _router("a", {"custom_dimensions": [_KEYWORD_DIMENSION]})
+        assert mutable_tuned_identities((reverted_to_binary,), baselines) == {router_identity(graded)}
+
     def test_violation_message_names_the_limit_and_remedy(self) -> None:
         message = tuning_limit_violation(held=2, limit=1)
         assert message is not None
diff --git a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx
index 15cfff01766..111eeebe5a3 100644
--- a/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/ClassificationMethodConfig.tsx
@@ -44,8 +44,9 @@ import {
 } from "./ComplexityRouterConfig";
 
 const DEFAULT_SCORING_EXPLANATION =
-  "The router scores each request across 7 dimensions: token count, code presence, reasoning markers, technical " +
-  "terms, simple indicators, multi-step patterns, and question complexity. The weighted score determines the tier:";
+  "The router scores each request across 7 built-in dimensions: token count, code presence, reasoning markers, technical " +
+  "terms, simple indicators, multi-step patterns, and question complexity, plus any custom dimensions you add. " +
+  "The weighted score determines the tier:";
 
 const HEURISTIC_V2_EXPLANATION =
   "The router estimates success probability for all four tiers with the bundled calibrated model, then selects " +
diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx
index 2970e14b335..0c12cc0ba1a 100644
--- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.test.tsx
@@ -1521,14 +1521,16 @@ describe("ComplexityRouterConfig tier editing", () => {
     renderWithProviders(<ComplexityRouterConfig {...baseProps} value={customValue} onEditingTiersChange={vi.fn()} />);
     fireEvent.click(screen.getByText("Advanced: Classification Method"));
     expect(screen.queryByText("How Classification Works")).not.toBeInTheDocument();
-    expect(screen.queryByText("scores each request across 7 dimensions", { exact: false })).not.toBeInTheDocument();
+    expect(
+      screen.queryByText("scores each request across 7 built-in dimensions", { exact: false }),
+    ).not.toBeInTheDocument();
   });
 
   it("keeps the scorer card on a built-in router, whose tiers the score still decides", () => {
     renderWithProviders(<ComplexityRouterConfig {...baseProps} onEditingTiersChange={vi.fn()} />);
     fireEvent.click(screen.getByText("Advanced: Classification Method"));
     expect(screen.getByText("How Classification Works")).toBeInTheDocument();
-    expect(screen.getByText("scores each request across 7 dimensions", { exact: false })).toBeInTheDocument();
+    expect(screen.getByText("scores each request across 7 built-in dimensions", { exact: false })).toBeInTheDocument();
   });
 
   it("says why a custom row is blocked instead of only reddening its border", () => {
diff --git a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
index d7df6ce33bb..de80e714e66 100644
--- a/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/ComplexityRouterConfig.tsx
@@ -50,6 +50,7 @@ import EscalationKeywords from "./EscalationKeywords";
 import KeywordTierRules, { KeywordTierRule } from "./KeywordTierRules";
 import SemanticKeywordMatching from "./SemanticKeywordMatching";
 import { type DimensionWeights, type TierBoundaries, type TokenThresholds } from "./heuristic_scoring_knobs";
+import { type CustomDimensionRow } from "./custom_dimensions";
 import CompressionControls from "./CompressionControls";
 import { type AutoRouterCompressionState, DEFAULT_AUTO_ROUTER_COMPRESSION } from "./buildAutoRouterCompression";
 
@@ -432,6 +433,11 @@ export interface ComplexityRouterConfigValue {
   tier_boundaries?: TierBoundaries;
   token_thresholds?: TokenThresholds;
   dimension_weights?: DimensionWeights;
+  /**
+   * Operator-added scoring dimensions, each carrying its own inline weight. Undefined means the router has
+   * none and keeps the key out of the payload; an empty array is a real "the last row was removed" state.
+   */
+  custom_dimensions?: CustomDimensionRow[];
   /**
    * Score floor the reasoning-marker override must clear. Undefined keeps the key out of the payload, so the
    * floor tracks tier_boundaries.simple_medium; an explicit 0 is a real floor that promotes on the markers alone.
diff --git a/ui/litellm-dashboard/src/components/add_model/CustomDimensionRows.tsx b/ui/litellm-dashboard/src/components/add_model/CustomDimensionRows.tsx
new file mode 100644
index 00000000000..34d8370aafb
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/add_model/CustomDimensionRows.tsx
@@ -0,0 +1,136 @@
+import { Trash2 } from "lucide-react";
+import { useState } from "react";
+import { Button } from "@/components/ui/button";
+import { Input } from "@/components/ui/input";
+import { Label } from "@/components/ui/label";
+import { Textarea } from "@/components/ui/textarea";
+import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
+import { Slider } from "@/components/ui/slider";
+import type { CustomDimensionRow } from "./custom_dimensions";
+
+const SCORING_MODES = [
+  { value: "binary", label: "Binary" },
+  { value: "match_count", label: "Match count" },
+] as const;
+
+interface Props {
+  rows: CustomDimensionRow[];
+  disabled: boolean;
+  onChange: (rows: CustomDimensionRow[]) => void;
+  onWeight: (id: string, weight: number) => void;
+  onAdd: () => void;
+  onRemove: (id: string) => void;
+}
+
+export default function CustomDimensionRows({ rows, disabled, onChange, onWeight, onAdd, onRemove }: Props) {
+  const [draft, setDraft] = useState<{ id: string; raw: string } | null>(null);
+  const update = (id: string, patch: Partial<CustomDimensionRow>) =>
+    onChange(rows.map((row) => (row.id === id ? { ...row, ...patch } : row)));
+  const editWeight = (id: string, raw: string) => {
+    setDraft({ id, raw });
+    if (raw.trim() && Number.isFinite(Number(raw))) onWeight(id, Number(raw));
+  };
+  return (
+    <div className="space-y-4">
+      {rows.map((row, index) => (
+        <fieldset key={row.id} className="min-w-0 space-y-3 rounded-md border p-3">
+          <legend className="float-left text-sm font-semibold">Custom dimension {index + 1}</legend>
+          <div className="flex items-center gap-2">
+            <Button
+              type="button"
+              variant="ghost"
+              size="sm"
+              className="text-destructive hover:text-destructive/80"
+              aria-label={`Remove custom dimension ${index + 1}`}
+              disabled={disabled}
+              onClick={() => onRemove(row.id)}
+            >
+              <Trash2 />
+              Remove
+            </Button>
+          </div>
+          <div className="space-y-1">
+            <Label htmlFor={`${row.id}-name`}>Name</Label>
+            <Input
+              id={`${row.id}-name`}
+              value={row.name}
+              maxLength={64}
+              onChange={(event) => update(row.id, { name: event.target.value })}
+            />
+          </div>
+          <div className="flex flex-wrap items-center gap-3">
+            <Label htmlFor={`${row.id}-weight`}>Weight</Label>
+            <Slider
+              min={0}
+              max={1}
+              step={0.01}
+              disabled={disabled}
+              value={[row.weight]}
+              className="min-w-24 flex-1"
+              aria-label={`${row.name || `Custom dimension ${index + 1}`} weight`}
+              onValueChange={(value) => onWeight(row.id, Array.isArray(value) ? value[0] : value)}
+            />
+            <Input
+              id={`${row.id}-weight`}
+              className="w-24"
+              inputMode="decimal"
+              disabled={disabled}
+              value={draft?.id === row.id ? draft.raw : Number(row.weight.toPrecision(6)).toString()}
+              onBlur={() => setDraft(null)}
+              onChange={(event) => editWeight(row.id, event.target.value)}
+            />
+          </div>
+          <div className="grid grid-cols-1 gap-3 sm:grid-cols-2">
+            {(["keywords", "patterns"] as const).map((field) => (
+              <div key={field} className="min-w-0 space-y-1">
+                <Label htmlFor={`${row.id}-${field}`}>
+                  {field === "keywords" ? "Keywords" : "Regex patterns"} (one per line)
+                </Label>
+                <Textarea
+                  id={`${row.id}-${field}`}
+                  rows={2}
+                  value={row[field]?.join("\n") ?? ""}
+                  onChange={(event) =>
+                    update(row.id, { [field]: event.target.value ? event.target.value.split("\n") : [] })
+                  }
+                />
+              </div>
+            ))}
+          </div>
+          <div className="space-y-1">
+            <Label htmlFor={`${row.id}-scoring`}>Scoring</Label>
+            <Select
+              items={SCORING_MODES}
+              value={row.scoring_mode ?? "binary"}
+              onValueChange={(mode) => {
+                if (mode === "binary" || mode === "match_count") update(row.id, { scoring_mode: mode });
+              }}
+            >
+              <SelectTrigger id={`${row.id}-scoring`}>
+                <SelectValue />
+              </SelectTrigger>
+              <SelectContent>
+                {SCORING_MODES.map((mode) => (
+                  <SelectItem key={mode.value} value={mode.value}>
+                    {mode.label}
+                  </SelectItem>
+                ))}
+              </SelectContent>
+            </Select>
+            <p className="text-xs text-muted-foreground">
+              Binary uses the full weight for any hit. Match count uses half for one distinct matcher and full weight
+              for two or more.
+            </p>
+          </div>
+        </fieldset>
+      ))}
+      <Button type="button" variant="outline" size="sm" disabled={disabled || rows.length >= 16} onClick={onAdd}>
+        Add custom dimension
+      </Button>
+      <p className="text-xs text-muted-foreground">
+        Keywords match the current ask. Regex scans its first 2,048 characters and permits bounded single-character
+        repeats up to 64. The proxy validates patterns on save.
+      </p>
+    </div>
+  );
+}
diff --git a/ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.integration.test.tsx
new file mode 100644
index 00000000000..9c4600e3b97
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.integration.test.tsx
@@ -0,0 +1,90 @@
+import { useState } from "react";
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import userEvent from "@testing-library/user-event";
+import { fireEvent, renderWithProviders, screen, testQueryClient, chooseSelectOption } from "../../../tests/test-utils";
+import { SHIPPED_SCORER_DEFAULTS } from "../../../tests/mocks/complexityScorerDefaults";
+import HeuristicScoringConfig from "./HeuristicScoringConfig";
+import type { ComplexityRouterConfigValue } from "./ComplexityRouterConfig";
+import { getComplexityScorerDefaults } from "@/components/networking";
+
+vi.mock("@/components/networking", () => ({ getComplexityScorerDefaults: vi.fn() }));
+
+const base: ComplexityRouterConfigValue = {
+  classifier_type: "heuristic",
+  tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] },
+};
+function Editor({ initial = base }: { initial?: ComplexityRouterConfigValue }) {
+  const [value, setValue] = useState(initial);
+  return (
+    <>
+      <HeuristicScoringConfig value={value} onChange={setValue} />
+      <output aria-label="Draft config">{JSON.stringify(value)}</output>
+    </>
+  );
+}
+const draft = (): ComplexityRouterConfigValue => JSON.parse(screen.getByLabelText("Draft config").textContent!);
+
+beforeEach(() => {
+  testQueryClient.clear();
+  vi.mocked(getComplexityScorerDefaults).mockResolvedValue(SHIPPED_SCORER_DEFAULTS);
+});
+
+describe("combined heuristic editor", () => {
+  it("adds and edits a graded dimension, rebalances builtins, preserves matcher-only edits, and removes it", async () => {
+    const user = userEvent.setup();
+    renderWithProviders(<Editor />);
+    await user.click(screen.getByText("Advanced scoring"));
+    expect(await screen.findByRole("button", { name: "Restore default weights" })).toBeDisabled();
+    await user.click(screen.getByRole("button", { name: "Add custom dimension" }));
+    expect(screen.getByRole("button", { name: "Restore default weights" })).toBeEnabled();
+    expect(draft().dimension_weights?.codePresence).toBeCloseTo(0.27, 12);
+    expect(draft().custom_dimensions?.[0].scoring_mode).toBe("match_count");
+    expect(screen.getByRole("group", { name: "Custom dimension 1" })).toContainElement(
+      screen.getByLabelText("Name", { exact: true }),
+    );
+    fireEvent.change(screen.getByLabelText("Name", { exact: true }), { target: { value: "domain" } });
+    fireEvent.change(screen.getByLabelText("Keywords (one per line)"), { target: { value: "orbitmesh\nfluxgate" } });
+    fireEvent.change(screen.getByLabelText("Weight", { exact: true }), { target: { value: "0.2" } });
+    expect(screen.getByLabelText("Code presence", { exact: true })).toHaveValue("0.24");
+    expect(screen.getByTestId("dimension-weight-total")).toHaveTextContent("total 1.00");
+    const beforeMatchers = draft().dimension_weights;
+    fireEvent.change(screen.getByLabelText("Regex patterns (one per line)"), { target: { value: "abc" } });
+    await chooseSelectOption(user, screen.getByLabelText("Scoring"), "Binary");
+    expect(draft().dimension_weights).toEqual(beforeMatchers);
+    expect(draft().custom_dimensions?.[0].scoring_mode).toBe("binary");
+    await user.click(screen.getByRole("button", { name: "Remove custom dimension 1" }));
+    expect(draft().custom_dimensions).toBeUndefined();
+    expect(draft().dimension_weights?.codePresence).toBeCloseTo(0.3, 12);
+  });
+
+  it("preserves a legacy vector on load and resets both fields only when requested", async () => {
+    const legacy = {
+      ...base,
+      dimension_weights: { codePresence: 0.4 },
+      custom_dimensions: [{ id: "stored-0", name: "domain", weight: 0.7, keywords: ["a"] }],
+    };
+    renderWithProviders(<Editor initial={legacy} />);
+    await userEvent.click(screen.getByText("Advanced scoring"));
+    expect(await screen.findByTestId("dimension-weight-total")).toHaveTextContent("total 1.10");
+    expect(draft()).toEqual(legacy);
+    expect(screen.getByLabelText("Token count", { exact: true })).toHaveValue("0");
+    await userEvent.click(screen.getByRole("button", { name: "Restore default weights" }));
+    expect(draft().custom_dimensions).toBeUndefined();
+    expect(draft().dimension_weights).toBeUndefined();
+  });
+
+  it("drops hidden custom drafts on a fallback-only weight edit so switching back cannot exceed the budget", async () => {
+    const initial: ComplexityRouterConfigValue = {
+      ...base,
+      classifier_type: "llm",
+      classifier_fallback: "heuristic",
+      custom_dimensions: [{ id: "a", name: "domain", weight: 0.7, keywords: ["a"] }],
+    };
+    renderWithProviders(<Editor initial={initial} />);
+    await userEvent.click(screen.getByText("Advanced scoring"));
+    fireEvent.change(await screen.findByLabelText("Code presence", { exact: true }), { target: { value: "0.5" } });
+    expect(draft().custom_dimensions).toBeUndefined();
+    expect(Object.values(draft().dimension_weights!).reduce((sum, weight) => sum + weight, 0)).toBeCloseTo(1, 12);
+    expect(screen.queryByRole("button", { name: "Add custom dimension" })).not.toBeInTheDocument();
+  });
+});
diff --git a/ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.test.tsx b/ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.test.tsx
index f5fb32bb654..61779318fff 100644
--- a/ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.test.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.test.tsx
@@ -69,13 +69,12 @@ describe("HeuristicScoringConfig", () => {
     expect(onChange).not.toHaveBeenCalled();
   });
 
-  // min and max are inert attributes on a text input, so without the explicit clamp these would persist
-  // a weight of 999, or an infinite boundary, into the router config.
-  it.each([
-    ["Code presence", "999", 1],
-    ["Code presence", "-2", 0],
-    ["Long above", "100000", 100000],
-  ])("clamps %s = %s to %s", async (label, raw, expected) => {
+  it.each(["999", "-2"])("rejects an invalid weight %s instead of silently clamping", async (raw) => {
+    expect(await commit("Code presence", raw)).toBeUndefined();
+    expect(screen.getByRole("alert")).toHaveTextContent("Use a weight from 0 to 1");
+  });
+
+  it.each([["Long above", "100000", 100000]])("clamps %s = %s to %s", async (label, raw, expected) => {
     const next = await commit(label, raw);
     expect(
       { ...next?.dimension_weights, ...next?.token_thresholds }[label === "Long above" ? "complex" : "codePresence"],
@@ -264,6 +263,7 @@ describe("HeuristicScoringConfig degraded states", () => {
     await userEvent.click(screen.getByText("Advanced scoring"));
 
     expect(screen.getByLabelText("Code presence")).toHaveValue("0.5");
+    expect(screen.getByLabelText("Code presence")).toBeDisabled();
     expect(screen.queryByTestId("dimension-weight-total")).not.toBeInTheDocument();
   });
 });
diff --git a/ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.tsx b/ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.tsx
index 540beff8177..4d59ac5bd33 100644
--- a/ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/HeuristicScoringConfig.tsx
@@ -8,7 +8,14 @@ import { Input } from "@/components/ui/input";
 import { Label } from "@/components/ui/label";
 import { Slider } from "@/components/ui/slider";
 import { type ComplexityRouterConfigValue, heuristicScoringRole } from "./ComplexityRouterConfig";
-import { dimensionLabel, weightTotal } from "./heuristic_scoring_knobs";
+import {
+  dimensionLabel,
+  effectiveDimensionWeights,
+  rebalanceDimensionWeights,
+  type WeightEdit,
+} from "./heuristic_scoring_knobs";
+import CustomDimensionRows from "./CustomDimensionRows";
+import { customDimensionsError } from "./custom_dimensions";
 
 export type KnobGroup = "tier_boundaries" | "token_thresholds" | "dimension_weights";
 
@@ -54,7 +61,8 @@ const GROUPS: GroupSpec[] = [
   {
     group: "dimension_weights",
     title: "Dimension weights",
-    blurb: "How much each signal contributes to the score. Absolute multipliers, so the total need not be 1.00.",
+    blurb:
+      "Changing a weight rebalances the other built-in and custom weights to total 1.00. Save stores those values. Untouched routers keep their existing weights.",
     min: 0,
     max: 1,
     step: 0.01,
@@ -89,6 +97,19 @@ const HeuristicScoringConfig: React.FC<HeuristicScoringConfigProps> = ({ value,
   // The panel owns its own visibility: the scorer does not run at all when an LLM classifier
   // falls back to the default model, so there is nothing here to configure.
   const scorerRuns = heuristicScoringRole(value) !== "never";
+  const customEnabled = heuristicScoringRole(value) === "decides";
+  const customRows = customEnabled ? value.custom_dimensions : undefined;
+  const [weightError, setWeightError] = useState<string | null>(null);
+  const rowError = customDimensionsError(customRows);
+  const changeWeights = (edit: WeightEdit) => {
+    const result = rebalanceDimensionWeights(defaults?.dimension_weights, value.dimension_weights, customRows, edit);
+    if (!result.ok) {
+      setWeightError(result.error);
+      return;
+    }
+    setWeightError(null);
+    onChange({ ...value, dimension_weights: result.dimension_weights, custom_dimensions: result.custom_dimensions });
+  };
 
   // What an untouched override floor follows: the boundary in effect, override included, not the shipped one.
   const trackedFloor: number | undefined = { ...defaults?.tier_boundaries, ...value.tier_boundaries }.simple_medium;
@@ -102,6 +123,10 @@ const HeuristicScoringConfig: React.FC<HeuristicScoringConfigProps> = ({ value,
   const commit = (spec: GroupSpec, effective: Record<string, number>, key: string, raw: string) => {
     const parsed = Number(raw);
     if (raw.trim() === "" || !Number.isFinite(parsed)) return;
+    if (spec.group === "dimension_weights") {
+      changeWeights({ type: "set", target: { kind: "builtin", id: key }, weight: parsed });
+      return;
+    }
     const clamped = Math.min(spec.max ?? Infinity, Math.max(spec.min, parsed));
     onChange({
       ...value,
@@ -154,9 +179,19 @@ const HeuristicScoringConfig: React.FC<HeuristicScoringConfigProps> = ({ value,
                 </div>
               )}
               {GROUPS.map((spec) => {
-                const shipped = defaults?.[spec.group] ?? {};
-                const effective: Record<string, number> = { ...shipped, ...value[spec.group] };
+                const shipped = defaults?.[spec.group] ?? value[spec.group] ?? {};
+                const effective: Record<string, number> =
+                  spec.group === "dimension_weights"
+                    ? effectiveDimensionWeights(shipped, value.dimension_weights)
+                    : { ...shipped, ...value[spec.group] };
+                const total =
+                  Object.values(effective).reduce((sum, weight) => sum + weight, 0) +
+                  (customRows ?? []).reduce((sum, row) => sum + row.weight, 0);
                 const problem = warn(spec.group, effective);
+                const overridden =
+                  value[spec.group] !== undefined || (spec.withSlider && value.custom_dimensions !== undefined);
+                const resettable = spec.withSlider || overridden;
+                const scoringError = spec.withSlider ? weightError || rowError : null;
                 return (
                   <section key={spec.group} className="space-y-2">
                     <div className="flex items-center justify-between">
@@ -166,18 +201,26 @@ const HeuristicScoringConfig: React.FC<HeuristicScoringConfigProps> = ({ value,
                             alone would state a total that is not the router's. */}
                         {spec.withSlider && defaults !== undefined && (
                           <span className="text-xs text-muted-foreground" data-testid="dimension-weight-total">
-                            total {weightTotal(effective).toFixed(2)}
+                            total {total.toFixed(2)}
                           </span>
                         )}
                       </div>
-                      {value[spec.group] !== undefined && (
+                      {resettable && (
                         <Button
                           type="button"
                           variant="link"
                           size="xs"
-                          onClick={() => onChange({ ...value, [spec.group]: undefined })}
+                          disabled={!overridden}
+                          onClick={() => {
+                            setWeightError(null);
+                            onChange({
+                              ...value,
+                              [spec.group]: undefined,
+                              ...(spec.withSlider && { custom_dimensions: undefined }),
+                            });
+                          }}
                         >
-                          Reset to defaults
+                          {spec.withSlider ? "Restore default weights" : "Reset to defaults"}
                         </Button>
                       )}
                     </div>
@@ -196,6 +239,7 @@ const HeuristicScoringConfig: React.FC<HeuristicScoringConfigProps> = ({ value,
                               min={spec.min}
                               max={spec.max}
                               step={spec.step}
+                              disabled={defaults === undefined}
                               value={[effective[key]]}
                               onValueChange={(next) =>
                                 commit(spec, effective, key, String(Array.isArray(next) ? next[0] : next))
@@ -209,7 +253,8 @@ const HeuristicScoringConfig: React.FC<HeuristicScoringConfigProps> = ({ value,
                             type="text"
                             inputMode="decimal"
                             className={spec.withSlider ? "w-24" : "w-28"}
-                            value={draft?.id === id ? draft.raw : String(effective[key])}
+                            disabled={spec.withSlider && defaults === undefined}
+                            value={draft?.id === id ? draft.raw : Number(effective[key].toPrecision(6)).toString()}
                             onChange={(event) => {
                               setDraft({ id, raw: event.target.value });
                               commit(spec, effective, key, event.target.value);
@@ -220,6 +265,28 @@ const HeuristicScoringConfig: React.FC<HeuristicScoringConfigProps> = ({ value,
                       );
                     })}
 
+                    {spec.withSlider && customEnabled && (
+                      <CustomDimensionRows
+                        rows={customRows ?? []}
+                        disabled={defaults === undefined}
+                        onChange={(rows) => onChange({ ...value, custom_dimensions: rows })}
+                        onWeight={(id, weight) =>
+                          changeWeights({ type: "set", target: { kind: "custom", id }, weight })
+                        }
+                        onAdd={() =>
+                          changeWeights({
+                            type: "add",
+                            row: { id: crypto.randomUUID(), name: "", weight: 0.1, scoring_mode: "match_count" },
+                          })
+                        }
+                        onRemove={(id) => changeWeights({ type: "remove", id })}
+                      />
+                    )}
+                    {scoringError && (
+                      <p className="text-xs text-destructive" role="alert">
+                        {scoringError}
+                      </p>
+                    )}
                     {problem && (
                       <p className="text-xs font-medium text-destructive" role="alert">
                         {problem}
diff --git a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx
index 2d0d6bc2fd8..cf65a093dfb 100644
--- a/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx
+++ b/ui/litellm-dashboard/src/components/add_model/add_auto_router_tab.tsx
@@ -25,12 +25,14 @@ import ComplexityRouterConfig, {
   ComplexityRouterConfigValue,
   effectiveClassifierType,
   usesLlmClassifier,
+  heuristicScoringRole,
   DEFAULT_ADAPTIVE_WEIGHTS,
   DEFAULT_SESSION_AFFINITY,
   DEFAULT_DEPLOYMENT_AFFINITY,
   DEFAULT_TIER_DISTANCE_PENALTY,
 } from "./ComplexityRouterConfig";
 import { KeywordTierRule } from "./KeywordTierRules";
+import { customDimensionsError } from "./custom_dimensions";
 import { DEFAULT_ESCALATION_KEYWORDS } from "./EscalationKeywords";
 import {
   type AutoRouterCompressionState,
@@ -139,6 +141,7 @@ export const getSubmitBlockedReason = (
     getPlanModeTierError(config.plan_mode_min_tier, activeTierRows(config)) ??
     getKeywordTierRulesError(keywordTierRules, activeTierRows(config)) ??
     getClassifierModelError(config) ??
+    (heuristicScoringRole(config) === "decides" ? customDimensionsError(config.custom_dimensions) : null) ??
     getClassifierReasoningEffortError(config, modelInfo) ??
     getReferencedModelsError(referencedModelsParams, availability)
   );
@@ -411,6 +414,7 @@ const AddAutoRouterTab: React.FC<AddAutoRouterTabProps> = ({
     tierBoundaries: complexityRouterConfig.tier_boundaries,
     tokenThresholds: complexityRouterConfig.token_thresholds,
     dimensionWeights: complexityRouterConfig.dimension_weights,
+    customDimensions: complexityRouterConfig.custom_dimensions,
     reasoningOverrideMinScore: complexityRouterConfig.reasoning_override_min_score,
     enableContextWindowEscalation: complexityRouterConfig.enable_context_window_escalation,
     contextWindowEscalationBuffer: complexityRouterConfig.context_window_escalation_buffer,
diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts
index baf04822e4f..9973aec7616 100644
--- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts
+++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts
@@ -704,6 +704,49 @@ describe("buildComplexityRouterConfig scorer knobs", () => {
     expect(buildComplexityRouterConfig(tuned).tier_boundaries).toEqual(BOUNDARIES);
   });
 
+  it("serializes custom rows without changing weights, matcher order, or optional-field absence", () => {
+    const weights = { codePresence: 0.12345678901234568, unknownStoredWeight: 9 };
+    const dimension = { name: "internalFrameworks", weight: 0.3765432109876543, keywords: ["ORBITMESH", "fluxgate"] };
+    const graded = { name: "sqlDdl", weight: 0.5, patterns: ["create table"], scoring_mode: "match_count" as const };
+    const payload = buildComplexityRouterConfig({
+      ...baseParams,
+      dimensionWeights: weights,
+      customDimensions: [
+        { id: "row-1", ...dimension },
+        { id: "row-2", ...graded },
+      ],
+    });
+    expect(payload.dimension_weights).toEqual(weights);
+    expect(payload.custom_dimensions).toEqual([dimension, graded]);
+    expect(buildComplexityRouterConfig(baseParams)).not.toHaveProperty("custom_dimensions");
+    expect(buildComplexityRouterConfig({ ...baseParams, customDimensions: [] }).custom_dimensions).toEqual([]);
+  });
+
+  it.each([
+    ["heuristic", undefined, true],
+    ["heuristic_first", "heuristic", true],
+    ["hybrid", "heuristic", true],
+    ["llm", "heuristic", false],
+    ["custom", "heuristic", false],
+    ["llm", "default_model", false],
+    ["custom", "default_model", false],
+    ["heuristic_v2", undefined, false],
+  ] as const)(
+    "%s with fallback %s only emits custom dimensions when its scorer decides",
+    (classifierType, classifierFallback, emits) => {
+      const dimension = { name: "d", weight: 0.4, keywords: ["orbitmesh"] };
+      const params = {
+        ...baseParams,
+        classifierType,
+        classifierFallback,
+        customDimensions: [{ id: "row", ...dimension }],
+      };
+      const payload = buildComplexityRouterConfig(params);
+      if (emits) expect(payload.custom_dimensions).toEqual([dimension]);
+      else expect(payload).not.toHaveProperty("custom_dimensions");
+    },
+  );
+
   it("drops them when the classifier falls back to the default model and nothing is scored", () => {
     expect(buildComplexityRouterConfig(llmWithDefaultFallback)).not.toHaveProperty("tier_boundaries");
   });
@@ -1060,6 +1103,7 @@ describe("buildComplexityRouterConfig with an edited tier set", () => {
       tierBoundaries: { simple_medium: 0.1, medium_complex: 0.25, complex_reasoning: 0.5 },
       tokenThresholds: { short: 1, long: 2 },
       dimensionWeights: { length: 1 },
+      customDimensions: [{ id: "row-1", name: "sqlDdl", weight: 0.4, keywords: ["orbitmesh"] }],
       reasoningOverrideMinScore: 0.5,
       heuristicFirstMaxTier: "SIMPLE",
       hybridBoundaryMargin: 0.03,
@@ -1068,7 +1112,8 @@ describe("buildComplexityRouterConfig with an edited tier set", () => {
       stallEscalationWindow: 6,
       stallEscalationRepeatThreshold: 3,
     };
-    const emittingType = key === "heuristic_first_max_tier" ? "heuristic_first" : "llm";
+    // custom_dimensions only ever ship when the scorer decides, so "llm" cannot prove it emits.
+    const emittingType = key === "heuristic_first_max_tier" || key === "custom_dimensions" ? "heuristic_first" : "llm";
     const typeForKey = key === "hybrid_boundary_margin" ? "hybrid" : emittingType;
     expect(buildComplexityRouterConfig({ ...baseParams, ...loaded, classifierType: typeForKey })).toHaveProperty(key);
     expect(build(loaded)).not.toHaveProperty(key);
diff --git a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts
index 7769fb832fe..d04d48198a8 100644
--- a/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts
+++ b/ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts
@@ -11,6 +11,7 @@ import {
   tierRowByName,
 } from "./tier_rows";
 import { emptyKeywordTierRuleIndexes, serializeKeywordTierRules } from "./complexity_router_keywords";
+import { type CustomDimension, type CustomDimensionRow, serializeCustomDimensions } from "./custom_dimensions";
 import {
   TierModelParams,
   TierModelParamsByTier,
@@ -92,6 +93,7 @@ interface ScorerKnobInputs {
   tierBoundaries: TierBoundaries | undefined;
   tokenThresholds: TokenThresholds | undefined;
   dimensionWeights: DimensionWeights | undefined;
+  customDimensions: CustomDimensionRow[] | undefined;
   reasoningOverrideMinScore: number | undefined;
 }
 
@@ -106,16 +108,23 @@ const scorerKnobPayload = ({
   tierBoundaries,
   tokenThresholds,
   dimensionWeights,
+  customDimensions,
   reasoningOverrideMinScore,
-}: ScorerKnobInputs) =>
-  heuristicScoringRoleFor(classifierType, classifierFallback) === "never"
+}: ScorerKnobInputs) => {
+  const role = heuristicScoringRoleFor(classifierType, classifierFallback);
+  return role === "never"
     ? {}
     : {
         ...(tierBoundaries && { tier_boundaries: tierBoundaries }),
         ...(tokenThresholds && { token_thresholds: tokenThresholds }),
         ...(dimensionWeights && { dimension_weights: dimensionWeights }),
+        // Only a scorer that decides accepts these; the backend rejects them on every other
+        // classifier, so a fallback-only router must not carry rows a switch left behind.
+        ...(role === "decides" &&
+          customDimensions !== undefined && { custom_dimensions: serializeCustomDimensions(customDimensions) }),
         ...(reasoningOverrideMinScore !== undefined && { reasoning_override_min_score: reasoningOverrideMinScore }),
       };
+};
 
 export interface BuildComplexityRouterConfigParams {
   tiers: ComplexityTiers;
@@ -155,6 +164,7 @@ export interface BuildComplexityRouterConfigParams {
   tierBoundaries?: TierBoundaries;
   tokenThresholds?: TokenThresholds;
   dimensionWeights?: DimensionWeights;
+  customDimensions?: CustomDimensionRow[];
   reasoningOverrideMinScore?: number;
   tierModelParams?: TierModelParamsByTier;
   enableContextWindowEscalation?: boolean;
@@ -219,6 +229,7 @@ export interface ComplexityRouterConfigPayload {
   tier_boundaries?: TierBoundaries;
   token_thresholds?: TokenThresholds;
   dimension_weights?: DimensionWeights;
+  custom_dimensions?: CustomDimension[];
   reasoning_override_min_score?: number;
   enable_context_window_escalation?: boolean;
   context_window_escalation_buffer?: number;
@@ -496,6 +507,7 @@ export const buildComplexityRouterConfig = ({
   tierBoundaries,
   tokenThresholds,
   dimensionWeights,
+  customDimensions,
   reasoningOverrideMinScore,
   tierModelParams,
   enableContextWindowEscalation,
@@ -517,6 +529,7 @@ export const buildComplexityRouterConfig = ({
     tierBoundaries,
     tokenThresholds,
     dimensionWeights,
+    customDimensions,
     reasoningOverrideMinScore,
   };
   const scorerKnobs = scorerKnobPayload(scorerInputs);
diff --git a/ui/litellm-dashboard/src/components/add_model/custom_dimensions.test.ts b/ui/litellm-dashboard/src/components/add_model/custom_dimensions.test.ts
new file mode 100644
index 00000000000..3c6060fd083
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/add_model/custom_dimensions.test.ts
@@ -0,0 +1,41 @@
+import { describe, expect, it } from "vitest";
+import { customDimensionsError, hydrateCustomDimensions, serializeCustomDimensions } from "./custom_dimensions";
+
+describe("custom dimension drafts", () => {
+  it.each([
+    [[{ name: "domain", weight: 0.2, keywords: [" orbitmesh "] }]],
+    [[{ name: "domain", weight: 0.2, patterns: ["abc"], keywords: [], scoring_mode: "binary" }]],
+    [[{ name: "domain", weight: 0.2, keywords: ["a", "b"], scoring_mode: "match_count" }]],
+    [[]],
+  ])("round-trips stored optional fields and matcher text without normalization: %j", (raw) => {
+    const rows = hydrateCustomDimensions(raw);
+    expect(rows).toBeDefined();
+    expect(serializeCustomDimensions(rows!)).toEqual(raw);
+  });
+
+  it("does not insert a default mode or custom list on load", () => {
+    expect(hydrateCustomDimensions(undefined)).toBeUndefined();
+    const rows = hydrateCustomDimensions([{ name: "domain", weight: 0.2, keywords: ["a"] }])!;
+    expect(rows[0].scoring_mode).toBeUndefined();
+    expect(customDimensionsError(rows)).toBeNull();
+  });
+
+  it.each([
+    { name: "", weight: 0.2, keywords: ["a"] },
+    { name: "codePresence", weight: 0.2, keywords: ["a"] },
+    { name: "bad name", weight: 0.2, keywords: ["a"] },
+    { name: "domain", weight: 0, keywords: ["a"] },
+    { name: "domain", weight: 0.2, keywords: [] },
+    { name: "domain", weight: 0.2, keywords: [" "] },
+    { name: "domain", weight: 0.2, keywords: ["a".repeat(257)] },
+  ])("rejects an invalid draft %j", (row) => {
+    expect(customDimensionsError([{ ...row, id: "draft" }])).not.toBeNull();
+  });
+
+  it("rejects duplicate names and aggregate matcher limits", () => {
+    const row = { id: "a", name: "domain", weight: 0.2, keywords: ["a"] };
+    expect(customDimensionsError([row, { ...row, id: "b", name: "DOMAIN" }])).not.toBeNull();
+    expect(customDimensionsError([{ ...row, keywords: Array(33).fill("a") }])).not.toBeNull();
+    expect(customDimensionsError([{ ...row, keywords: Array(32).fill("a".repeat(256)) }])).not.toBeNull();
+  });
+});
diff --git a/ui/litellm-dashboard/src/components/add_model/custom_dimensions.ts b/ui/litellm-dashboard/src/components/add_model/custom_dimensions.ts
new file mode 100644
index 00000000000..8cf5a16c303
--- /dev/null
+++ b/ui/litellm-dashboard/src/components/add_model/custom_dimensions.ts
@@ -0,0 +1,54 @@
+import { z } from "zod";
+import { DIMENSION_LABELS } from "./heuristic_scoring_knobs";
+
+const customDimensionShape = {
+  name: z.string(),
+  weight: z.number(),
+  keywords: z.array(z.string()).optional(),
+  patterns: z.array(z.string()).optional(),
+  scoring_mode: z.enum(["binary", "match_count"]).optional(),
+};
+const customDimensionSchema = z.object(customDimensionShape);
+
+export type CustomDimension = z.infer<typeof customDimensionSchema>;
+export type CustomDimensionRow = CustomDimension & { id: string };
+
+export const hydrateCustomDimensions = (raw: unknown): CustomDimensionRow[] | undefined => {
+  if (raw === undefined) return undefined;
+  const parsed = z.array(customDimensionSchema).safeParse(raw);
+  return parsed.success ? parsed.data.map((row, index) => ({ ...row, id: `stored-${index}` })) : undefined;
+};
+
+export const serializeCustomDimensions = (rows: CustomDimensionRow[]): CustomDimension[] =>
+  rows.map(({ id: _id, ...dimension }) => dimension);
+
+export const customDimensionsError = (
+  rows: CustomDimensionRow[] | undefined,
+  builtinNames: string[] = Object.keys(DIMENSION_LABELS),
+): string | null => {
+  if (!rows) return null;
+  if (rows.length > 16) return "A router can have at most 16 custom dimensions";
+  const names = rows.map((row) => row.name.toLowerCase());
+  for (const [index, row] of rows.entries()) {
+    const prefix = `Custom dimension ${index + 1}: `;
+    if (!/^[A-Za-z][A-Za-z0-9_]{0,63}$/.test(row.name))
+      return (
+        prefix + "use a name starting with a letter, followed by letters, numbers or underscores (64 characters max)"
+      );
+    if (builtinNames.some((name) => name.toLowerCase() === row.name.toLowerCase()))
+      return prefix + "choose a name that is not already a built-in weight";
+    if (names.indexOf(row.name.toLowerCase()) !== index) return prefix + "names must be unique";
+    if (!Number.isFinite(row.weight) || row.weight <= 0 || row.weight > 1)
+      return prefix + "weight must be greater than 0 and at most 1";
+    const matchers = [...(row.keywords ?? []), ...(row.patterns ?? [])];
+    if (!matchers.length || matchers.some((matcher) => !matcher.trim()))
+      return prefix + "add at least one nonblank keyword or pattern";
+    if (
+      matchers.length > 32 ||
+      matchers.some((matcher) => [...matcher].length > 256) ||
+      matchers.reduce((total, matcher) => total + [...matcher].length, 0) > 4096
+    )
+      return prefix + "use at most 32 matchers, 256 characters each and 4096 characters combined";
+  }
+  return null;
+};
diff --git a/ui/litellm-dashboard/src/components/add_model/heuristic_scoring_knobs.test.ts b/ui/litellm-dashboard/src/components/add_model/heuristic_scoring_knobs.test.ts
index ee1d819872b..ea785ddb594 100644
--- a/ui/litellm-dashboard/src/components/add_model/heuristic_scoring_knobs.test.ts
+++ b/ui/litellm-dashboard/src/components/add_model/heuristic_scoring_knobs.test.ts
@@ -3,6 +3,8 @@ import { describe, expect, it } from "vitest";
 import { heuristicScoringRoleFor } from "./ComplexityRouterConfig";
 import {
   dimensionLabel,
+  effectiveDimensionWeights,
+  rebalanceDimensionWeights,
   hydrateDimensionWeights,
   hydrateReasoningOverrideMinScore,
   hydrateTierBoundaries,
@@ -68,6 +70,123 @@ describe("hydrating the scorer knobs", () => {
   });
 });
 
+describe("rebalancing the complete weight vector", () => {
+  const defaults = {
+    codePresence: 0.3,
+    reasoningMarkers: 0.25,
+    technicalTerms: 0.25,
+    tokenCount: 0.1,
+    simpleIndicators: 0.05,
+    multiStepPatterns: 0.03,
+    questionComplexity: 0.02,
+  };
+  const row = { id: "domain", name: "domainMarkers", weight: 0.2, keywords: ["orbitmesh"] };
+  const success = (result: ReturnType<typeof rebalanceDimensionWeights>) => {
+    if (!result.ok) throw new Error(result.error);
+    const total =
+      Object.keys(defaults).reduce((sum, name) => sum + result.dimension_weights[name], 0) +
+      (result.custom_dimensions ?? []).reduce((sum, dimension) => sum + dimension.weight, 0);
+    expect(total).toBeCloseTo(1, 12);
+    return result;
+  };
+
+  it("adds a dimension, edits either kind, and redistributes its share on removal", () => {
+    const added = success(rebalanceDimensionWeights(defaults, undefined, undefined, { type: "add", row }));
+    expect(added.dimension_weights.codePresence).toBeCloseTo(0.24, 12);
+    expect(added.custom_dimensions?.[0].weight).toBe(0.2);
+    const edited = success(
+      rebalanceDimensionWeights(defaults, added.dimension_weights, added.custom_dimensions, {
+        type: "set",
+        target: { kind: "custom", id: row.id },
+        weight: 0.4,
+      }),
+    );
+    expect(edited.dimension_weights.codePresence).toBeCloseTo(0.18, 12);
+    const builtin = success(
+      rebalanceDimensionWeights(defaults, edited.dimension_weights, edited.custom_dimensions, {
+        type: "set",
+        target: { kind: "builtin", id: "codePresence" },
+        weight: 0.5,
+      }),
+    );
+    expect(builtin.dimension_weights.codePresence).toBe(0.5);
+    expect(builtin.custom_dimensions?.[0].weight).toBeCloseTo((0.4 * 0.5) / 0.82, 12);
+    const removed = success(
+      rebalanceDimensionWeights(defaults, added.dimension_weights, added.custom_dimensions, {
+        type: "remove",
+        id: row.id,
+      }),
+    );
+    expect(removed.custom_dimensions).toBeUndefined();
+    expect(removed.dimension_weights.codePresence).toBeCloseTo(defaults.codePresence, 12);
+  });
+
+  it("uses zero for omitted keys of an explicit map and preserves ignored keys", () => {
+    expect(effectiveDimensionWeights(defaults, { codePresence: 0.2 }).technicalTerms).toBe(0);
+    expect(effectiveDimensionWeights(defaults, undefined)).toEqual(defaults);
+    const result = success(
+      rebalanceDimensionWeights(defaults, { codePresence: 0.2, unknown: 7 }, undefined, { type: "add", row }),
+    );
+    expect(result.dimension_weights.unknown).toBe(7);
+    expect(result.dimension_weights.codePresence).toBeCloseTo(0.8, 12);
+    expect(result.dimension_weights.technicalTerms).toBe(0);
+  });
+
+  it("distributes an all-zero vector without inventing custom matchers", () => {
+    const result = success(rebalanceDimensionWeights(defaults, {}, undefined, { type: "add", row }));
+    expect(result.dimension_weights.codePresence).toBeCloseTo(0.8 / 7, 12);
+    expect(result.custom_dimensions).toEqual([row]);
+  });
+
+  it("keeps small positive custom weights and full precision", () => {
+    const tiny = { ...row, weight: 1e-8 };
+    const result = success(
+      rebalanceDimensionWeights(defaults, defaults, [tiny], {
+        type: "set",
+        target: { kind: "builtin", id: "codePresence" },
+        weight: 0.123456789,
+      }),
+    );
+    expect(result.dimension_weights.codePresence).toBe(0.123456789);
+    expect(result.custom_dimensions?.[0].weight).toBeGreaterThan(0);
+    expect(result.custom_dimensions?.[0].weight).toBeLessThan(0.01);
+  });
+
+  it.each([0, 1, Number.NaN, -0.1, 1.1])(
+    "rejects invalid or impossible custom weight %s without changing the input",
+    (weight) => {
+      const original = structuredClone(row);
+      const result = rebalanceDimensionWeights(defaults, defaults, [row, { ...row, id: "other" }], {
+        type: "set",
+        target: { kind: "custom", id: row.id },
+        weight,
+      });
+      expect(result.ok).toBe(false);
+      expect(row).toEqual(original);
+    },
+  );
+
+  it("allows one dimension to take the whole budget when no custom sibling needs a share", () => {
+    const result = success(
+      rebalanceDimensionWeights(defaults, defaults, [row], {
+        type: "set",
+        target: { kind: "custom", id: row.id },
+        weight: 1,
+      }),
+    );
+    expect(Object.values(result.dimension_weights).every((weight) => weight === 0)).toBe(true);
+    expect(result.custom_dimensions?.[0].weight).toBe(1);
+  });
+
+  it("refuses missing defaults and invalid stored weights, preserving absence on built-in edits", () => {
+    const edit = { type: "set", target: { kind: "builtin", id: "codePresence" }, weight: 0.2 } as const;
+    expect(rebalanceDimensionWeights(undefined, defaults, undefined, edit).ok).toBe(false);
+    expect(rebalanceDimensionWeights(defaults, { codePresence: -1 }, undefined, edit).ok).toBe(false);
+    expect(success(rebalanceDimensionWeights(defaults, undefined, undefined, edit)).custom_dimensions).toBeUndefined();
+    expect(success(rebalanceDimensionWeights(defaults, undefined, [], edit)).custom_dimensions).toEqual([]);
+  });
+});
+
 describe("heuristicScoringRoleFor", () => {
   it.each([
     ["heuristic", undefined, "decides"],
diff --git a/ui/litellm-dashboard/src/components/add_model/heuristic_scoring_knobs.ts b/ui/litellm-dashboard/src/components/add_model/heuristic_scoring_knobs.ts
index 1f44a479c60..3de4af92b9c 100644
--- a/ui/litellm-dashboard/src/components/add_model/heuristic_scoring_knobs.ts
+++ b/ui/litellm-dashboard/src/components/add_model/heuristic_scoring_knobs.ts
@@ -1,3 +1,5 @@
+import type { CustomDimensionRow } from "./custom_dimensions";
+
 export type TierBoundaries = Record<string, number>;
 
 export type TokenThresholds = Record<string, number>;
@@ -27,8 +29,8 @@ const asRecord = (raw: unknown): Record<string, unknown> | undefined =>
 /**
  * Absent means the router is tracking the shipped defaults, so it must hydrate to undefined rather than to
  * a copy of them: hydrating defaults would make an untouched save write them out and pin the router to
- * whatever they were the day the modal was opened. A stored dict is kept exactly as stored, since the
- * backend fills in any key it omits at scoring time.
+ * whatever they were the day the modal was opened. A stored weight map replaces the defaults;
+ * missing dimension names score zero.
  */
 const hydrateNumericMap = (raw: unknown): Record<string, number> | undefined => {
   const stored = asRecord(raw);
@@ -53,3 +55,94 @@ export const hydrateReasoningOverrideMinScore = (raw: unknown): number | undefin
 
 export const weightTotal = (weights: DimensionWeights): number =>
   Math.round(Object.values(weights).reduce((total, weight) => total + weight, 0) * 100) / 100;
+
+export const effectiveDimensionWeights = (
+  defaults: DimensionWeights,
+  stored: DimensionWeights | undefined,
+): DimensionWeights =>
+  Object.fromEntries(
+    Object.keys(defaults).map((name) => [name, stored === undefined ? defaults[name] : stored[name] ?? 0]),
+  );
+
+type WeightTarget = { kind: "builtin" | "custom"; id: string };
+const weightValid = ({ kind, weight }: { kind: WeightTarget["kind"]; weight: number }): boolean => {
+  const inRange = Number.isFinite(weight) && weight >= 0 && weight <= 1;
+  return inRange && (kind === "builtin" || weight > 0);
+};
+export type WeightEdit =
+  | { type: "set"; target: WeightTarget; weight: number }
+  | { type: "add"; row: CustomDimensionRow }
+  | { type: "remove"; id: string };
+type WeightResult =
+  | { ok: true; dimension_weights: DimensionWeights; custom_dimensions: CustomDimensionRow[] | undefined }
+  | { ok: false; error: string };
+
+export const rebalanceDimensionWeights = (
+  defaults: DimensionWeights | undefined,
+  stored: DimensionWeights | undefined,
+  custom: CustomDimensionRow[] | undefined,
+  edit: WeightEdit,
+): WeightResult => {
+  if (!defaults || !Object.keys(defaults).length)
+    return { ok: false, error: "Load the shipped defaults before changing weights" };
+  const builtin = effectiveDimensionWeights(defaults, stored);
+  const rows =
+    edit.type === "add"
+      ? [...(custom ?? []), edit.row]
+      : (custom ?? []).filter((row) => edit.type !== "remove" || row.id !== edit.id);
+  const vector = [
+    ...Object.entries(builtin).map(([id, weight]) => ({ kind: "builtin" as const, id, weight })),
+    ...rows.map(({ id, weight }) => ({ kind: "custom" as const, id, weight })),
+  ];
+  if (!vector.every(weightValid))
+    return {
+      ok: false,
+      error: "Existing weights must be finite and nonnegative; custom weights must be greater than 0 and at most 1",
+    };
+  const target = edit.type === "set" ? edit.target : undefined;
+  const pinned = (entry: WeightTarget) =>
+    edit.type === "add"
+      ? entry.kind === "custom" && entry.id === edit.row.id
+      : entry.kind === target?.kind && entry.id === target.id;
+  if (edit.type === "set" && !vector.some(pinned)) return { ok: false, error: "The dimension is no longer available" };
+  const requestedWeight = (): number => {
+    if (edit.type === "set") return edit.weight;
+    return edit.type === "add" ? edit.row.weight : 0;
+  };
+  const weight = requestedWeight();
+  if (!weightValid({ kind: target?.kind ?? "builtin", weight }))
+    return { ok: false, error: "Use a weight from 0 to 1; custom dimensions must stay greater than 0" };
+  const others = vector.filter((entry) => !pinned(entry));
+  const total = others.reduce((sum, entry) => sum + entry.weight, 0);
+  if (!Number.isFinite(total)) return { ok: false, error: "Existing weights are too large to rebalance" };
+  const remainder = 1 - weight;
+  const builtinCount = others.filter((entry) => entry.kind === "builtin").length;
+  const redistributed = (entry: WeightTarget & { weight: number }): number => {
+    if (pinned(entry)) return weight;
+    if (total > 0) return remainder * (entry.weight / total);
+    return entry.kind === "builtin" ? remainder / builtinCount : 0;
+  };
+  const balanced = vector.map((entry) => ({ ...entry, weight: redistributed(entry) }));
+  const residual = 1 - balanced.reduce((sum, entry) => sum + entry.weight, 0);
+  const receiver = balanced
+    .filter((entry) => entry.kind === "builtin" && !pinned(entry) && entry.weight > 0)
+    .sort((a, b) => b.weight - a.weight)[0];
+  const corrected = balanced.map((entry) =>
+    entry === receiver ? { ...entry, weight: entry.weight + residual } : entry,
+  );
+  const offBudget = Math.abs(corrected.reduce((sum, entry) => sum + entry.weight, 0) - 1) > 1e-12;
+  if (!corrected.every(weightValid) || offBudget)
+    return { ok: false, error: "Leave a positive share for every custom dimension, or remove it first" };
+  const weights = Object.fromEntries(
+    corrected.filter((entry) => entry.kind === "builtin").map(({ id, weight }) => [id, weight]),
+  );
+  const customWeights = new Map(
+    corrected.filter((entry) => entry.kind === "custom").map(({ id, weight }) => [id, weight]),
+  );
+  const emptyRows = edit.type === "remove" ? undefined : custom;
+  return {
+    ok: true,
+    dimension_weights: { ...stored, ...weights },
+    custom_dimensions: rows.length ? rows.map((row) => ({ ...row, weight: customWeights.get(row.id)! })) : emptyRows,
+  };
+};
diff --git a/ui/litellm-dashboard/src/components/add_model/tier_rows.ts b/ui/litellm-dashboard/src/components/add_model/tier_rows.ts
index 5e2a32addee..d2cec15d75b 100644
--- a/ui/litellm-dashboard/src/components/add_model/tier_rows.ts
+++ b/ui/litellm-dashboard/src/components/add_model/tier_rows.ts
@@ -136,6 +136,7 @@ export const CUSTOM_TIER_RESTRICTIONS = {
       "tier_boundaries",
       "token_thresholds",
       "dimension_weights",
+      "custom_dimensions",
       "reasoning_override_min_score",
       "custom_technical_keywords",
     ],
diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts
index 0144dff3498..4a57e9d5e13 100644
--- a/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts
+++ b/ui/litellm-dashboard/src/components/edit_auto_router/build_updated_complexity_router_config.test.ts
@@ -63,6 +63,54 @@ describe("buildUpdatedComplexityRouterConfig keyword matching", () => {
     expect(result.some_future_backend_key).toEqual({ nested: true });
   });
 
+  describe("custom dimensions", () => {
+    const stored = [
+      { name: "internalFrameworks", weight: 0.7, keywords: ["orbitmesh"] },
+      { name: "sqlDdl", weight: 0.3, patterns: ["create table"], scoring_mode: "match_count" },
+    ];
+    const withDimensions = { ...STORED, custom_dimensions: stored };
+
+    it("round-trips stored rows through hydration and save without dropping or reshaping one", () => {
+      const hydrated = hydrateComplexityRouterConfig(withDimensions, null);
+      expect(hydrated.custom_dimensions).toEqual([
+        { id: "stored-0", ...stored[0] },
+        { id: "stored-1", ...stored[1] },
+      ]);
+      const saved = buildUpdatedComplexityRouterConfig(withDimensions, {
+        ...FORM_VALUE,
+        custom_dimensions: hydrated.custom_dimensions,
+      });
+      expect(saved.custom_dimensions).toEqual(stored);
+    });
+
+    it("is a managed key, so removing the last row does not resurrect the stored dimensions", () => {
+      expect(MANAGED_COMPLEXITY_ROUTER_KEYS.has("custom_dimensions")).toBe(true);
+      const saved = buildUpdatedComplexityRouterConfig(withDimensions, { ...FORM_VALUE, custom_dimensions: [] });
+      expect(saved.custom_dimensions).toEqual([]);
+    });
+
+    it("omits the key entirely when the editor never held rows, so an untouched router gains nothing", () => {
+      expect(hydrateComplexityRouterConfig(STORED, null).custom_dimensions).toBeUndefined();
+      expect(buildUpdatedComplexityRouterConfig(STORED, FORM_VALUE)).not.toHaveProperty("custom_dimensions");
+    });
+
+    it("drops rows a custom tier set forbids rather than sending them to a scorer that never runs", () => {
+      const customTierStored = storedCustomConfig({ custom_dimensions: stored });
+      const saved = buildUpdatedComplexityRouterConfig(customTierStored, {
+        tiers: { SIMPLE: [], MEDIUM: [], COMPLEX: [], REASONING: [] },
+        classifier_type: "llm" as const,
+        custom_tier_set: {
+          tiers: [
+            { id: "a", name: "CASUAL", definition: "small talk", models: ["gpt-4o-mini"] },
+            { id: "b", name: "AUDIT", definition: "security review", models: ["o1"] },
+          ],
+          fallback_tier_id: "a",
+        },
+      });
+      expect(saved).not.toHaveProperty("custom_dimensions");
+    });
+  });
+
   it("persists an edited keyword rule", () => {
     const result = buildUpdatedComplexityRouterConfig(STORED, FORM_VALUE, undefined, {
       ...hydratedState,
@@ -580,6 +628,7 @@ describe("managed keys survive an untouched open-and-save", () => {
     tier_boundaries: { simple_medium: 0.2, medium_complex: 0.4, complex_reasoning: 0.7 },
     token_thresholds: { simple: 20, complex: 500 },
     dimension_weights: { tokenCount: 0.1 },
+    custom_dimensions: [{ name: "domain", weight: 0.9, keywords: ["orbitmesh"] }],
     reasoning_override_min_score: 0.3,
     enable_context_window_escalation: false,
     context_window_escalation_buffer: 0.9,
diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx
index 3c0013e267e..b23040ab80c 100644
--- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx
+++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.tsx
@@ -48,6 +48,7 @@ import {
   hydrateAutoRouterCompression,
 } from "../add_model/buildAutoRouterCompression";
 import { hydrateKeywordTierRules } from "../add_model/complexity_router_keywords";
+import { customDimensionsError, hydrateCustomDimensions } from "../add_model/custom_dimensions";
 import {
   hydrateDimensionWeights,
   hydrateReasoningOverrideMinScore,
@@ -61,6 +62,7 @@ import ComplexityRouterConfig, {
   ClassifierType,
   ComplexityRouterConfigValue,
   ComplexityTiers,
+  heuristicScoringRole,
   DEFAULT_ADAPTIVE_WEIGHTS,
   DEFAULT_SESSION_AFFINITY,
   DEFAULT_DEPLOYMENT_AFFINITY,
@@ -109,6 +111,7 @@ export interface StoredComplexityRouterConfig {
   tier_boundaries?: unknown;
   token_thresholds?: unknown;
   dimension_weights?: unknown;
+  custom_dimensions?: unknown;
   reasoning_override_min_score?: unknown;
   session_affinity?: unknown;
   session_affinity_ttl_seconds?: unknown;
@@ -194,6 +197,7 @@ export const hydrateComplexityRouterConfig = (
     tier_boundaries: hydrateTierBoundaries(parsedConfig.tier_boundaries),
     token_thresholds: hydrateTokenThresholds(parsedConfig.token_thresholds),
     dimension_weights: hydrateDimensionWeights(parsedConfig.dimension_weights),
+    custom_dimensions: hydrateCustomDimensions(parsedConfig.custom_dimensions),
     reasoning_override_min_score: hydrateReasoningOverrideMinScore(parsedConfig.reasoning_override_min_score),
     session_affinity:
       typeof parsedConfig.session_affinity === "boolean" ? parsedConfig.session_affinity : DEFAULT_SESSION_AFFINITY,
@@ -264,6 +268,7 @@ export const MANAGED_COMPLEXITY_ROUTER_KEYS = new Set([
   "tier_boundaries",
   "token_thresholds",
   "dimension_weights",
+  "custom_dimensions",
   "reasoning_override_min_score",
   "enable_context_window_escalation",
   "context_window_escalation_buffer",
@@ -373,6 +378,7 @@ export const buildUpdatedComplexityRouterConfig = (
     tierBoundaries: value.tier_boundaries,
     tokenThresholds: value.token_thresholds,
     dimensionWeights: value.dimension_weights,
+    customDimensions: value.custom_dimensions,
     reasoningOverrideMinScore: value.reasoning_override_min_score,
     tierModelParams: value.tier_model_params,
     enableContextWindowEscalation: value.enable_context_window_escalation,
@@ -481,7 +487,10 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
             : null) ?? getTierLabelsError(complexityRouterConfig.tier_labels)) ??
       getPlanModeTierError(complexityRouterConfig.plan_mode_min_tier, activeTierRows(complexityRouterConfig)) ??
       getKeywordTierRulesError(keywordTierRules, activeTierRows(complexityRouterConfig)) ??
-      getClassifierModelError(complexityRouterConfig);
+      getClassifierModelError(complexityRouterConfig) ??
+      (heuristicScoringRole(complexityRouterConfig) === "decides"
+        ? customDimensionsError(complexityRouterConfig.custom_dimensions)
+        : null);
 
   useEffect(() => {
     if (isVisible && modelData) {
@@ -601,7 +610,11 @@ const EditAutoRouterModal: React.FC<EditAutoRouterModalProps> = ({
         toast.fromError(tierSetError);
         return;
       }
-      const classifierError = getClassifierModelError(complexityRouterConfig);
+      const classifierError =
+        getClassifierModelError(complexityRouterConfig) ??
+        (heuristicScoringRole(complexityRouterConfig) === "decides"
+          ? customDimensionsError(complexityRouterConfig.custom_dimensions)
+          : null);
       if (classifierError) {
         setShowValidationErrors(true);
         toast.fromError(classifierError);
diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts
index 344964c4434..5d1c7f73c71 100644
--- a/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts
+++ b/ui/litellm-dashboard/src/lib/autorouter_presets.test.ts
@@ -87,6 +87,19 @@ describe("autorouter_presets", () => {
     }
   });
 
+  it("keeps every preset free of custom dimensions, so applying one never adds scoring rows", () => {
+    for (const { complexity_router_config: config } of getAllPresets()) {
+      expect(config.custom_dimensions).toBeUndefined();
+    }
+    const config = getPresetByKey("anthropic_family")!.complexity_router_config;
+    expect(buildPresetPrefill(config, groupsOnly([])).complexityRouterConfig.custom_dimensions).toBeUndefined();
+  });
+
+  it("resets both scoring overrides when the form falls back to an empty prefill", () => {
+    expect(buildEmptyPrefill().complexityRouterConfig.custom_dimensions).toBeUndefined();
+    expect(buildEmptyPrefill().complexityRouterConfig.dimension_weights).toBeUndefined();
+  });
+
   it("carries a preset's session affinity idle window into the prefilled form state", () => {
     const config = getPresetByKey("anthropic_family")!.complexity_router_config;
     const prefill = buildPresetPrefill({ ...config, session_affinity_ttl_seconds: 300 }, groupsOnly([]));
@@ -96,6 +109,21 @@ describe("autorouter_presets", () => {
     ).toBeUndefined();
   });
 
+  it("carries a preset's stored weights and custom dimensions into the prefill without rebalancing them", () => {
+    const config = getPresetByKey("anthropic_family")!.complexity_router_config;
+    const weights = { codePresence: 0.4 };
+    const dimension = { name: "domain", weight: 0.9, keywords: ["orbitmesh"] };
+    const prefill = buildPresetPrefill(
+      { ...config, dimension_weights: weights, custom_dimensions: [dimension] },
+      groupsOnly([]),
+    ).complexityRouterConfig;
+    expect(prefill.dimension_weights).toEqual(weights);
+    expect(prefill.custom_dimensions).toEqual([{ ...dimension, id: "stored-0" }]);
+    const plain = buildPresetPrefill(config, groupsOnly([])).complexityRouterConfig;
+    expect(plain.dimension_weights).toBeUndefined();
+    expect(plain.custom_dimensions).toBeUndefined();
+  });
+
   it("keeps the model-family presets on the heuristic classifier", () => {
     for (const key of ["anthropic_family", "gemini_family", "openai_family"]) {
       expect(getPresetByKey(key)!.complexity_router_config.classifier_type).toBe("heuristic");
diff --git a/ui/litellm-dashboard/src/lib/autorouter_presets.ts b/ui/litellm-dashboard/src/lib/autorouter_presets.ts
index 2aafbfcfcd9..f2df55fc310 100644
--- a/ui/litellm-dashboard/src/lib/autorouter_presets.ts
+++ b/ui/litellm-dashboard/src/lib/autorouter_presets.ts
@@ -13,6 +13,13 @@ import {
 } from "@/components/add_model/ComplexityRouterConfig";
 import { KeywordTierRule } from "@/components/add_model/KeywordTierRules";
 import { hydrateKeywordTierRules } from "@/components/add_model/complexity_router_keywords";
+import { hydrateCustomDimensions } from "@/components/add_model/custom_dimensions";
+import {
+  hydrateDimensionWeights,
+  hydrateTierBoundaries,
+  hydrateTokenThresholds,
+  hydrateReasoningOverrideMinScore,
+} from "@/components/add_model/heuristic_scoring_knobs";
 import {
   TierModelParams,
   TierModelParamsByTier,
@@ -293,6 +300,11 @@ export const buildPresetPrefill = (
       tier_distance_penalty: config.tier_distance_penalty,
       adaptive_eligible: config.adaptive_eligible,
       return_raw_model_name: config.return_raw_model_name,
+      dimension_weights: hydrateDimensionWeights(config.dimension_weights),
+      custom_dimensions: hydrateCustomDimensions(config.custom_dimensions),
+      tier_boundaries: hydrateTierBoundaries(config.tier_boundaries),
+      token_thresholds: hydrateTokenThresholds(config.token_thresholds),
+      reasoning_override_min_score: hydrateReasoningOverrideMinScore(config.reasoning_override_min_score),
       enable_context_window_escalation: config.enable_context_window_escalation,
       context_window_escalation_buffer: config.context_window_escalation_buffer,
     },
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index 232b366d8ce..d1431d8956f 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -26591,6 +26591,13 @@ export interface components {
              * @default []
              */
             patterns: string[];
+            /**
+             * Scoring Mode
+             * @description 'binary' scores 1 when any matcher hits. 'match_count' scores 0.5 when one distinct matcher hits and 1 when two or more do; repeated occurrences of one matcher never raise it. Keywords are distinct case-insensitively, patterns by source, and a keyword and a pattern are always distinct from each other.
+             * @default binary
+             * @enum {string}
+             */
+            scoring_mode: "binary" | "match_count";
             /** Weight */
             weight: number;
         };
@@ -34909,7 +34916,7 @@ export interface components {
             context_window_escalation_buffer: number;
             /**
              * Custom Dimensions
-             * @description Named binary dimensions added to the heuristic-v1 score. Each contributes its inline weight once when any keyword matches the current ask or a case-insensitive regex matches its first 2048 characters. Regex quantifiers repeat one character or class at most 64 times. Unbounded quantifiers, repeated groups, backreferences and lookarounds are rejected. Conservative work limits include alternation paths, repeat lengths and subsequent matching: 2048 units per pattern, 8192 across the router. Only heuristic, heuristic_first and hybrid accept this field. Uses the existing heuristic tuning quota.
+             * @description Named dimensions added to the heuristic-v1 score. Each contributes its inline weight once when any keyword matches the current ask or a case-insensitive regex matches its first 2048 characters; scoring_mode 'match_count' instead grades half weight for one distinct matcher and full for two or more. Regex quantifiers repeat one character or class at most 64 times. Unbounded quantifiers, repeated groups, backreferences and lookarounds are rejected. Conservative work limits include alternation paths, repeat lengths and subsequent matching: 2048 units per pattern, 8192 across the router. Only heuristic, heuristic_first and hybrid accept this field. Uses the existing heuristic tuning quota.
              * @default []
              */
             custom_dimensions: components["schemas"]["CustomDimension"][];

From 9e18526887c847415b8333742603daf2bb90bb5f Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 8 Sep 2026 13:31:07 -0700
Subject: [PATCH 70/85] feat(deploy): metrics sidecar and separate metrics port
 in Helm and Terraform (#40163)

* feat(deploy): expose SSE keepalive, pre-call checks and a metrics sidecar in Helm and Terraform

Typed reliability values on both Helm charts and the AWS/GCP Terraform
modules, a dedicated ClusterIP Service for the separate Prometheus port,
a /health route on the metrics server and dead-worker pruning so the
aggregate does not keep stale multiprocess samples.

Resolves LIT-7142

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* refactor(deploy): drop reliability config from Helm and Terraform, keep only the metrics sidecar

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(proxy): cover startup pruning of dead workers' live gauges and unsignalable pids

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 docker/component_entrypoint.sh                |   6 +
 helm/litellm-helm/templates/deployment.yaml   |  12 ++
 .../templates/service-metrics.yaml            |  17 ++
 .../templates/servicemonitor.yaml             |   2 +-
 .../tests/metrics_server_tests.yaml           | 106 +++++++++++++
 helm/litellm-helm/values.yaml                 |  10 ++
 helm/litellm/templates/_helpers.tpl           |   2 +
 .../litellm/templates/gateway/deployment.yaml |  53 ++++++-
 .../templates/gateway/service-metrics.yaml    |  18 +++
 helm/litellm/tests/metrics_server_tests.yaml  | 148 ++++++++++++++++++
 helm/litellm/values.yaml                      |  16 ++
 litellm/proxy/prometheus_cleanup.py           |  35 +++++
 litellm/proxy/prometheus_metrics_server.py    |   5 +
 litellm/proxy/proxy_server.py                 |   8 +
 terraform/litellm/aws/README.md               |  16 ++
 terraform/litellm/aws/ecs.tf                  |  54 ++++++-
 .../litellm/aws/examples/default/main.tf      |   3 +
 .../examples/default/terraform.tfvars.example |   7 +
 .../litellm/aws/examples/default/variables.tf |  12 ++
 terraform/litellm/aws/network.tf              |  11 ++
 .../aws/tests/metrics_sidecar.tftest.hcl      | 108 +++++++++++++
 terraform/litellm/aws/variables.tf            |  38 +++++
 .../proxy/proxy_server/test_lifecycle.py      |  25 +++
 .../proxy/test_prometheus_cleanup.py          |  66 +++++++-
 .../proxy/test_prometheus_metrics_server.py   |   9 +-
 .../test_litellm/test_component_entrypoint.py |  53 +++++++
 26 files changed, 830 insertions(+), 10 deletions(-)
 create mode 100644 helm/litellm-helm/templates/service-metrics.yaml
 create mode 100644 helm/litellm-helm/tests/metrics_server_tests.yaml
 create mode 100644 helm/litellm/templates/gateway/service-metrics.yaml
 create mode 100644 helm/litellm/tests/metrics_server_tests.yaml
 create mode 100644 terraform/litellm/aws/tests/metrics_sidecar.tftest.hcl

diff --git a/docker/component_entrypoint.sh b/docker/component_entrypoint.sh
index 413957b9929..173afafe1ad 100755
--- a/docker/component_entrypoint.sh
+++ b/docker/component_entrypoint.sh
@@ -1,5 +1,11 @@
 #!/bin/sh
 
+# stale samples from a previous container incarnation would be summed into the aggregate
+if [ -n "$PROMETHEUS_MULTIPROC_DIR" ]; then
+    mkdir -p "$PROMETHEUS_MULTIPROC_DIR"
+    rm -f "$PROMETHEUS_MULTIPROC_DIR"/*.db
+fi
+
 case "$USE_DDTRACE" in
     [Tt][Rr][Uu][Ee])
         export DD_TRACE_OPENAI_ENABLED="False"
diff --git a/helm/litellm-helm/templates/deployment.yaml b/helm/litellm-helm/templates/deployment.yaml
index 52ffd117535..f7c918a6827 100644
--- a/helm/litellm-helm/templates/deployment.yaml
+++ b/helm/litellm-helm/templates/deployment.yaml
@@ -152,6 +152,13 @@ spec:
             {{- if .Values.billingMetrics.enabled }}
             {{- include "litellm.billingMetricsEnv" . | nindent 12 }}
             {{- end }}
+            {{- if .Values.metricsServer.enabled }}
+            {{- if eq (int .Values.metricsServer.port) (int .Values.service.port) }}
+            {{- fail "metricsServer.port must differ from service.port" }}
+            {{- end }}
+            - name: PROMETHEUS_METRICS_PORT
+              value: {{ .Values.metricsServer.port | quote }}
+            {{- end }}
             {{- if .Values.migrationJob.enabled }}
             # Schema updates are owned by the dedicated migrations Job; skip
             # the proxy's startup `prisma db push` so N replicas don't race
@@ -189,6 +196,11 @@ spec:
             - name: http
               containerPort: {{ .Values.service.port }}
               protocol: TCP
+            {{- if .Values.metricsServer.enabled }}
+            - name: metrics
+              containerPort: {{ .Values.metricsServer.port }}
+              protocol: TCP
+            {{- end }}
           livenessProbe:
             httpGet:
               path: {{ .Values.livenessProbe.path | quote }}
diff --git a/helm/litellm-helm/templates/service-metrics.yaml b/helm/litellm-helm/templates/service-metrics.yaml
new file mode 100644
index 00000000000..1d23fe39606
--- /dev/null
+++ b/helm/litellm-helm/templates/service-metrics.yaml
@@ -0,0 +1,17 @@
+{{- if .Values.metricsServer.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "litellm.fullname" . }}-metrics
+  labels:
+    {{- include "litellm.labels" . | nindent 4 }}
+spec:
+  type: ClusterIP
+  ports:
+    - port: {{ .Values.metricsServer.port }}
+      targetPort: metrics
+      protocol: TCP
+      name: metrics
+  selector:
+    {{- include "litellm.selectorLabels" . | nindent 4 }}
+{{- end }}
diff --git a/helm/litellm-helm/templates/servicemonitor.yaml b/helm/litellm-helm/templates/servicemonitor.yaml
index 743098deb3f..68083d0da61 100644
--- a/helm/litellm-helm/templates/servicemonitor.yaml
+++ b/helm/litellm-helm/templates/servicemonitor.yaml
@@ -26,7 +26,7 @@ spec:
     {{- toYaml .namespaceSelector.matchNames | nindent 4 }}
     {{- end }}
   endpoints:
-  - port: http
+  - port: {{ ternary "metrics" "http" $.Values.metricsServer.enabled }}
     path: /metrics/
     interval: {{ .interval }}
     scrapeTimeout: {{ .scrapeTimeout }}
diff --git a/helm/litellm-helm/tests/metrics_server_tests.yaml b/helm/litellm-helm/tests/metrics_server_tests.yaml
new file mode 100644
index 00000000000..085d69ac640
--- /dev/null
+++ b/helm/litellm-helm/tests/metrics_server_tests.yaml
@@ -0,0 +1,106 @@
+suite: separate metrics server
+templates:
+  - configmap-litellm.yaml
+  - deployment.yaml
+  - service.yaml
+  - service-metrics.yaml
+  - servicemonitor.yaml
+tests:
+  - it: should not expose a metrics port or PROMETHEUS_METRICS_PORT by default
+    asserts:
+      - notContains:
+          path: spec.template.spec.containers[0].ports
+          content:
+            name: metrics
+          any: true
+        template: deployment.yaml
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: PROMETHEUS_METRICS_PORT
+          any: true
+        template: deployment.yaml
+      - lengthEqual:
+          path: spec.ports
+          count: 1
+        template: service.yaml
+      - hasDocuments:
+          count: 0
+        template: service-metrics.yaml
+
+  - it: should scrape the proxy port when the metrics server is disabled
+    template: servicemonitor.yaml
+    set:
+      serviceMonitor.enabled: true
+    asserts:
+      - equal:
+          path: spec.endpoints[0].port
+          value: http
+
+  - it: should wire the separate metrics server through container, a ClusterIP metrics service and servicemonitor
+    set:
+      metricsServer.enabled: true
+      metricsServer.port: 4101
+      serviceMonitor.enabled: true
+      service.type: LoadBalancer
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: PROMETHEUS_METRICS_PORT
+            value: "4101"
+        template: deployment.yaml
+      - contains:
+          path: spec.template.spec.containers[0].ports
+          content:
+            name: metrics
+            containerPort: 4101
+            protocol: TCP
+        template: deployment.yaml
+      - lengthEqual:
+          path: spec.ports
+          count: 1
+        template: service.yaml
+      - equal:
+          path: spec.type
+          value: LoadBalancer
+        template: service.yaml
+      - equal:
+          path: metadata.name
+          value: RELEASE-NAME-litellm-metrics
+        template: service-metrics.yaml
+      - equal:
+          path: spec.type
+          value: ClusterIP
+        template: service-metrics.yaml
+      - equal:
+          path: spec.ports
+          value:
+            - port: 4101
+              targetPort: metrics
+              protocol: TCP
+              name: metrics
+        template: service-metrics.yaml
+      - equal:
+          path: spec.selector
+          value:
+            app.kubernetes.io/name: litellm
+            app.kubernetes.io/instance: RELEASE-NAME
+        template: service-metrics.yaml
+      - equal:
+          path: spec.endpoints[0].port
+          value: metrics
+        template: servicemonitor.yaml
+      - equal:
+          path: spec.endpoints[0].path
+          value: /metrics/
+        template: servicemonitor.yaml
+
+  - it: should reject a metrics port equal to the proxy port
+    template: deployment.yaml
+    set:
+      metricsServer.enabled: true
+      metricsServer.port: 4000
+    asserts:
+      - failedTemplate:
+          errorMessage: metricsServer.port must differ from service.port
diff --git a/helm/litellm-helm/values.yaml b/helm/litellm-helm/values.yaml
index 637be2322e3..8dc7b967e11 100644
--- a/helm/litellm-helm/values.yaml
+++ b/helm/litellm-helm/values.yaml
@@ -180,6 +180,16 @@ proxy_config:
   general_settings:
     master_key: os.environ/PROXY_MASTER_KEY
 
+# Serve Prometheus /metrics from a separate process (PROMETHEUS_METRICS_PORT)
+# so a scrape never runs on an inference worker. Adds a `metrics` port to the
+# container and a dedicated ClusterIP `<release>-metrics` Service, and the
+# ServiceMonitor scrapes it instead of the proxy port. The separate port has
+# no virtual-key auth: keep it off public ingress. Needs the proxy image
+# v1.101.0 or newer.
+metricsServer:
+  enabled: false
+  port: 4001
+
 resources:
   {}
   # Unset by default so the chart installs on small clusters such as Minikube, and so an
diff --git a/helm/litellm/templates/_helpers.tpl b/helm/litellm/templates/_helpers.tpl
index be6b9093f53..c459512c7b9 100644
--- a/helm/litellm/templates/_helpers.tpl
+++ b/helm/litellm/templates/_helpers.tpl
@@ -441,3 +441,5 @@ ImplementationSpecific
 {{- .pathType -}}
 {{- end -}}
 {{- end -}}
+
+{{- define "litellm.gateway.prometheusMultiprocDir" -}}/tmp/litellm_prometheus_multiproc{{- end -}}
diff --git a/helm/litellm/templates/gateway/deployment.yaml b/helm/litellm/templates/gateway/deployment.yaml
index 5030ba2c9dc..9cb6b07e77b 100644
--- a/helm/litellm/templates/gateway/deployment.yaml
+++ b/helm/litellm/templates/gateway/deployment.yaml
@@ -64,14 +64,25 @@ spec:
             {{- if .Values.billingMetrics.enabled }}
             {{- include "litellm.billingMetricsEnv" . | nindent 12 }}
             {{- end }}
+            {{- if .Values.gateway.metricsServer.enabled }}
+            {{- if eq (int .Values.gateway.metricsServer.port) 4000 }}
+            {{- fail "gateway.metricsServer.port must differ from the gateway port 4000" }}
+            {{- end }}
+            - name: PROMETHEUS_MULTIPROC_DIR
+              value: {{ include "litellm.gateway.prometheusMultiprocDir" . }}
+            {{- end }}
           {{- include "litellm.envFrom" .Values.gateway | nindent 10 }}
-          {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.billingMetrics.enabled }}
+          {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled }}
           volumeMounts:
             {{- if .Values.gateway.config.create }}
             - name: gateway-config
               mountPath: /app/config/config.yaml
               subPath: config.yaml
             {{- end }}
+            {{- if .Values.gateway.metricsServer.enabled }}
+            - name: prometheus-multiproc
+              mountPath: {{ include "litellm.gateway.prometheusMultiprocDir" . }}
+            {{- end }}
             {{- if .Values.billingMetrics.enabled }}
             {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }}
             {{- end }}
@@ -97,16 +108,54 @@ spec:
           {{- end }}
           resources:
             {{- toYaml .Values.gateway.resources | nindent 12 }}
+        {{- if .Values.gateway.metricsServer.enabled }}
+        - name: metrics
+          image: "{{ .Values.gateway.image.repository }}:{{ .Values.gateway.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.gateway.image.pullPolicy }}
+          {{- with .Values.gateway.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          command:
+            - python
+            - -m
+            - litellm.proxy.prometheus_metrics_server
+            - --port
+            - {{ .Values.gateway.metricsServer.port | quote }}
+          env:
+            - name: PROMETHEUS_MULTIPROC_DIR
+              value: {{ include "litellm.gateway.prometheusMultiprocDir" . }}
+          ports:
+            - name: metrics
+              containerPort: {{ .Values.gateway.metricsServer.port }}
+              protocol: TCP
+          volumeMounts:
+            - name: prometheus-multiproc
+              mountPath: {{ include "litellm.gateway.prometheusMultiprocDir" . }}
+          readinessProbe:
+            tcpSocket: { port: metrics }
+            periodSeconds: 10
+          livenessProbe:
+            tcpSocket: { port: metrics }
+            periodSeconds: 15
+            failureThreshold: 6
+          resources:
+            {{- toYaml .Values.gateway.metricsServer.resources | nindent 12 }}
+        {{- end }}
       {{- with .Values.gateway.extraContainers }}
         {{- tpl (toYaml .) $ | nindent 8 }}
       {{- end }}
-      {{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.billingMetrics.enabled }}
+      {{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled }}
       volumes:
         {{- if .Values.gateway.config.create }}
         - name: gateway-config
           configMap:
             name: {{ include "litellm.gateway.fullname" . }}-config
         {{- end }}
+        {{- if .Values.gateway.metricsServer.enabled }}
+        - name: prometheus-multiproc
+          emptyDir: {}
+        {{- end }}
         {{- if .Values.billingMetrics.enabled }}
         {{- include "litellm.billingMetricsVolumes" . | nindent 8 }}
         {{- end }}
diff --git a/helm/litellm/templates/gateway/service-metrics.yaml b/helm/litellm/templates/gateway/service-metrics.yaml
new file mode 100644
index 00000000000..ad9bc05a9fd
--- /dev/null
+++ b/helm/litellm/templates/gateway/service-metrics.yaml
@@ -0,0 +1,18 @@
+{{- if and .Values.gateway.enabled .Values.gateway.metricsServer.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "litellm.gateway.fullname" . }}-metrics
+  labels:
+    {{- include "litellm.commonLabels" . | nindent 4 }}
+    app.kubernetes.io/component: gateway
+spec:
+  type: ClusterIP
+  ports:
+    - port: {{ .Values.gateway.metricsServer.port }}
+      targetPort: metrics
+      protocol: TCP
+      name: metrics
+  selector:
+    {{- include "litellm.gateway.selectorLabels" . | nindent 4 }}
+{{- end }}
diff --git a/helm/litellm/tests/metrics_server_tests.yaml b/helm/litellm/tests/metrics_server_tests.yaml
new file mode 100644
index 00000000000..0e7d9d9e9ee
--- /dev/null
+++ b/helm/litellm/tests/metrics_server_tests.yaml
@@ -0,0 +1,148 @@
+suite: test gateway metrics sidecar
+templates:
+  - gateway/configmap.yaml
+  - gateway/deployment.yaml
+  - gateway/service.yaml
+  - gateway/service-metrics.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: adds no sidecar, volume, env or service port when the metrics server is off
+    asserts:
+      - lengthEqual:
+          path: spec.template.spec.containers
+          count: 1
+        template: gateway/deployment.yaml
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: PROMETHEUS_MULTIPROC_DIR
+          any: true
+        template: gateway/deployment.yaml
+      - notContains:
+          path: spec.template.spec.volumes
+          content:
+            name: prometheus-multiproc
+          any: true
+        template: gateway/deployment.yaml
+      - lengthEqual:
+          path: spec.ports
+          count: 1
+        template: gateway/service.yaml
+      - hasDocuments:
+          count: 0
+        template: gateway/service-metrics.yaml
+
+  - it: runs the metrics server as a sidecar over a shared multiproc dir and exposes it on a ClusterIP metrics service
+    set:
+      gateway.metricsServer.enabled: true
+      gateway.metricsServer.port: 4101
+      gateway.service.type: LoadBalancer
+      gateway.image.tag: v1.101.0
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: PROMETHEUS_MULTIPROC_DIR
+            value: /tmp/litellm_prometheus_multiproc
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: prometheus-multiproc
+            mountPath: /tmp/litellm_prometheus_multiproc
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.template.spec.containers[1].name
+          value: metrics
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.template.spec.containers[1].image
+          value: ghcr.io/berriai/litellm-gateway:v1.101.0
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.template.spec.containers[1].command
+          value:
+            - python
+            - -m
+            - litellm.proxy.prometheus_metrics_server
+            - --port
+            - "4101"
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.template.spec.containers[1].env
+          value:
+            - name: PROMETHEUS_MULTIPROC_DIR
+              value: /tmp/litellm_prometheus_multiproc
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.template.spec.containers[1].ports
+          value:
+            - name: metrics
+              containerPort: 4101
+              protocol: TCP
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.template.spec.containers[1].volumeMounts
+          value:
+            - name: prometheus-multiproc
+              mountPath: /tmp/litellm_prometheus_multiproc
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.template.spec.containers[1].readinessProbe.tcpSocket.port
+          value: metrics
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.template.spec.containers[1].livenessProbe.tcpSocket.port
+          value: metrics
+        template: gateway/deployment.yaml
+      - equal:
+          path: spec.template.spec.containers[1].resources.requests.cpu
+          value: 50m
+        template: gateway/deployment.yaml
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: prometheus-multiproc
+            emptyDir: {}
+        template: gateway/deployment.yaml
+      - lengthEqual:
+          path: spec.ports
+          count: 1
+        template: gateway/service.yaml
+      - equal:
+          path: spec.type
+          value: LoadBalancer
+        template: gateway/service.yaml
+      - equal:
+          path: metadata.name
+          value: RELEASE-NAME-litellm-gateway-metrics
+        template: gateway/service-metrics.yaml
+      - equal:
+          path: spec.type
+          value: ClusterIP
+        template: gateway/service-metrics.yaml
+      - equal:
+          path: spec.ports
+          value:
+            - port: 4101
+              targetPort: metrics
+              protocol: TCP
+              name: metrics
+        template: gateway/service-metrics.yaml
+      - equal:
+          path: spec.selector
+          value:
+            app.kubernetes.io/name: litellm
+            app.kubernetes.io/instance: RELEASE-NAME
+            app.kubernetes.io/component: gateway
+        template: gateway/service-metrics.yaml
+
+  - it: rejects a metrics port equal to the gateway port
+    template: gateway/deployment.yaml
+    set:
+      gateway.metricsServer.enabled: true
+      gateway.metricsServer.port: 4000
+    asserts:
+      - failedTemplate:
+          errorMessage: gateway.metricsServer.port must differ from the gateway port 4000
diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml
index 6c9fb9440c7..b5d535c992d 100644
--- a/helm/litellm/values.yaml
+++ b/helm/litellm/values.yaml
@@ -268,6 +268,22 @@ gateway:
   config:
     create: true
     proxy_config: {}
+  # Serve Prometheus /metrics from a `metrics` sidecar container (same image,
+  # `python -m litellm.proxy.prometheus_metrics_server`) that aggregates the
+  # workers' PROMETHEUS_MULTIPROC_DIR samples over a shared emptyDir, so a
+  # scrape never runs on an inference worker. Adds a `metrics` port to the pod
+  # and a dedicated ClusterIP `<gateway>-metrics` Service; point your scrape
+  # config at it. The port has no virtual-key auth: keep it off public ingress.
+  # Needs the gateway image v1.101.0 or newer.
+  metricsServer:
+    enabled: false
+    port: 4001
+    resources:
+      requests:
+        cpu: 50m
+        memory: 128Mi
+      limits:
+        memory: 512Mi
   image:
     repository: ghcr.io/berriai/litellm-gateway
     tag: ""              # defaults to .Chart.AppVersion
diff --git a/litellm/proxy/prometheus_cleanup.py b/litellm/proxy/prometheus_cleanup.py
index d9827723887..c65aaeedfaa 100644
--- a/litellm/proxy/prometheus_cleanup.py
+++ b/litellm/proxy/prometheus_cleanup.py
@@ -8,10 +8,13 @@ from __future__ import annotations
 
 import glob
 import os
+import re
 from typing import Final
 
 from litellm._logging import verbose_proxy_logger
 
+_LIVE_GAUGE_PID: Final = re.compile(r"gauge_live[a-z]*_(\d+)\.db$")
+
 
 def wipe_directory(directory: str) -> None:
     """Delete all .db files in the directory. Called once before workers fork."""
@@ -38,3 +41,35 @@ def mark_worker_exit(worker_pid: int) -> None:
         verbose_proxy_logger.info("Prometheus cleanup: marked worker %s as dead", worker_pid)
     except Exception as e:
         verbose_proxy_logger.warning("Failed to mark prometheus worker %s as dead: %s", worker_pid, e)
+
+
+def _is_running(pid: int) -> bool:
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        return True
+    return True
+
+
+def mark_dead_workers(directory: str) -> tuple[int, ...]:
+    """Drop the live-gauge files of workers that no longer exist and return their pids.
+
+    Uvicorn's multi-worker supervisor has no exit hook, so a replacement worker calls this at startup; without it
+    a crashed worker's in-flight gauges stay in the aggregate forever.
+    """
+    owners: Final = frozenset(
+        int(match.group(1))
+        for match in map(_LIVE_GAUGE_PID.search, glob.glob(os.path.join(directory, "gauge_live*_*.db")))
+        if match is not None
+    )
+    dead: Final = tuple(sorted(pid for pid in owners if pid != os.getpid() and not _is_running(pid)))
+    if not dead:
+        return dead
+    from prometheus_client import multiprocess
+
+    for pid in dead:
+        multiprocess.mark_process_dead(pid, path=directory)
+    verbose_proxy_logger.info("Prometheus cleanup: marked dead workers %s in %s", dead, directory)
+    return dead
diff --git a/litellm/proxy/prometheus_metrics_server.py b/litellm/proxy/prometheus_metrics_server.py
index 4a9651d62e1..01479f9dd41 100644
--- a/litellm/proxy/prometheus_metrics_server.py
+++ b/litellm/proxy/prometheus_metrics_server.py
@@ -30,6 +30,7 @@ from litellm.integrations.prometheus_metrics_endpoint import make_metrics_asgi_a
 from litellm.llms.custom_httpx.http_handler import HTTPHandler
 
 METRICS_PATH: Final = "/metrics"
+HEALTH_PATH: Final = "/health"
 PID_HEADER: Final = "x-litellm-metrics-pid"
 _PARENT_POLL_INTERVAL_SECONDS: Final = 1.0
 _STARTUP_TIMEOUT_SECONDS: Final = 30.0
@@ -77,6 +78,10 @@ def build_metrics_app(multiproc_dir: str) -> FastAPI:
     app: Final = FastAPI(title="LiteLLM Prometheus metrics", docs_url=None, redoc_url=None, openapi_url=None)
     app.mount(METRICS_PATH, _add_pid_header(make_metrics_asgi_app(registry)))
 
+    @app.get(HEALTH_PATH)
+    def health() -> dict[str, str]:
+        return {"status": "healthy", "multiproc_dir": multiproc_dir}
+
     return app
 
 
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 1d6baf5fded..38c4c2e96c9 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -631,6 +631,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
 from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
     router as pass_through_router,
 )
+from litellm.proxy.prometheus_cleanup import mark_dead_workers, mark_worker_exit
 from litellm.proxy.public_endpoints import router as public_endpoints_router
 from litellm.proxy.public_endpoints.public_v1 import router as public_v1_router
 from litellm.proxy.rag_endpoints.endpoints import router as rag_router
@@ -1059,6 +1060,10 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]:
 
     init_verbose_loggers()
 
+    prometheus_multiproc_dir: Final = os.environ.get("PROMETHEUS_MULTIPROC_DIR")
+    if prometheus_multiproc_dir:
+        mark_dead_workers(prometheus_multiproc_dir)
+
     ## RUN WORKER STARTUP HOOKS (e.g., gflags initialization) ##
     _startup_hooks_env: Final = os.environ.get("LITELLM_WORKER_STARTUP_HOOKS", "")
     if _startup_hooks_env:
@@ -1365,6 +1370,9 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[None, None]:
 
     await proxy_shutdown_event(worker_heartbeat=worker_heartbeat)
 
+    if prometheus_multiproc_dir:
+        mark_worker_exit(os.getpid())
+
 
 def _generate_stable_operation_id(route: "APIRoute") -> str:
     operation_id = re.sub(r"\W", "_", f"{route.name}{route.path_format}")
diff --git a/terraform/litellm/aws/README.md b/terraform/litellm/aws/README.md
index 389027bf5ca..923a4ca2da8 100644
--- a/terraform/litellm/aws/README.md
+++ b/terraform/litellm/aws/README.md
@@ -242,6 +242,22 @@ this with `litellm_license`. To tune the export cadence, set
 `LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS` through `gateway_extra_env` /
 `backend_extra_env`
 
+### Prometheus metrics sidecar
+
+`gateway_metrics_port` adds a `metrics` sidecar
+(`python -m litellm.proxy.prometheus_metrics_server`) to the gateway task that
+aggregates the workers' samples over a shared task volume, so a scrape never
+runs on an inference worker. The ALB never routes to that port and the tasks
+security group only opens it to `gateway_metrics_scrape_cidrs`. Needs
+`gateway_image` v1.101.0 or newer. See
+[Prometheus metrics](https://docs.litellm.ai/docs/proxy/prometheus) for the
+metrics themselves.
+
+```hcl
+gateway_metrics_port         = 4001
+gateway_metrics_scrape_cidrs = ["10.0.0.0/16"]
+```
+
 ## Tenant deployment
 
 Every resource the stack creates is named `${tenant}-litellm-${env}` (or
diff --git a/terraform/litellm/aws/ecs.tf b/terraform/litellm/aws/ecs.tf
index 2fb1ca08205..aa2c3d558e2 100644
--- a/terraform/litellm/aws/ecs.tf
+++ b/terraform/litellm/aws/ecs.tf
@@ -212,6 +212,45 @@ locals {
   # pull the config from S3 first, so the command goes through `sh -c`;
   # otherwise we keep the image's ENTRYPOINT and only override `command`.
   gateway_uvicorn_args = "--host 0.0.0.0 --port 4000 --workers ${var.gateway_num_workers}"
+
+  metrics_enabled       = var.gateway_metrics_port != null
+  metrics_multiproc_dir = "/tmp/litellm_prometheus_multiproc"
+  metrics_volume        = "prometheus-multiproc"
+  metrics_env           = local.metrics_enabled ? [{ name = "PROMETHEUS_MULTIPROC_DIR", value = local.metrics_multiproc_dir }] : []
+  metrics_mount_points  = local.metrics_enabled ? [{ sourceVolume = local.metrics_volume, containerPath = local.metrics_multiproc_dir }] : []
+  metrics_health_cmd    = "import socket; socket.create_connection(('127.0.0.1', ${coalesce(var.gateway_metrics_port, 0)}), timeout=2).close()"
+
+  gateway_metrics_container = local.metrics_enabled ? [
+    {
+      name       = "metrics"
+      image      = var.gateway_image
+      essential  = false
+      entryPoint = ["python", "-m", "litellm.proxy.prometheus_metrics_server"]
+      command    = ["--port", tostring(var.gateway_metrics_port)]
+
+      portMappings = [{ containerPort = var.gateway_metrics_port, protocol = "tcp" }]
+      environment  = local.metrics_env
+      mountPoints  = local.metrics_mount_points
+
+      healthCheck = {
+        command     = ["CMD", "python", "-c", local.metrics_health_cmd]
+        interval    = 30
+        timeout     = 5
+        retries     = 3
+        startPeriod = 30
+      }
+
+      logConfiguration = {
+        logDriver = "awslogs"
+        options = {
+          awslogs-group         = aws_cloudwatch_log_group.gateway.name
+          awslogs-region        = var.region
+          awslogs-stream-prefix = "metrics"
+        }
+      }
+    }
+  ] : []
+
   backend_uvicorn_args = "--host 0.0.0.0 --port 4001"
 
   gateway_launch_cmd = "case \"$USE_DDTRACE\" in [Tt][Rr][Uu][Ee]) export DD_TRACE_OPENAI_ENABLED=\"False\"; exec ddtrace-run uvicorn gateway.main:app ${local.gateway_uvicorn_args};; *) exec uvicorn gateway.main:app ${local.gateway_uvicorn_args};; esac"
@@ -269,7 +308,7 @@ resource "aws_ecs_task_definition" "gateway" {
   execution_role_arn       = aws_iam_role.task_execution.arn
   task_role_arn            = aws_iam_role.task.arn
 
-  container_definitions = jsonencode([
+  container_definitions = jsonencode(concat([
     merge(
       {
         name      = "gateway"
@@ -283,8 +322,10 @@ resource "aws_ecs_task_definition" "gateway" {
           local.billing_metrics_env,
           local.gateway_extra_env_list,
           local.proxy_config_env,
+          local.metrics_env,
         )
-        secrets = concat(local.shared_secrets, local.gateway_extra_secrets_list)
+        secrets     = concat(local.shared_secrets, local.gateway_extra_secrets_list)
+        mountPoints = local.metrics_mount_points
 
         # Container-level healthCheck intentionally omitted — the wolfi
         # runtime image doesn't ship curl/wget. The ALB target group polls
@@ -301,7 +342,14 @@ resource "aws_ecs_task_definition" "gateway" {
       },
       local.gateway_proxy_overrides,
     )
-  ])
+  ], local.gateway_metrics_container))
+
+  dynamic "volume" {
+    for_each = local.metrics_enabled ? [1] : []
+    content {
+      name = local.metrics_volume
+    }
+  }
 
   tags = local.tags
 }
diff --git a/terraform/litellm/aws/examples/default/main.tf b/terraform/litellm/aws/examples/default/main.tf
index 2eeaf6adb50..115003f7ddd 100644
--- a/terraform/litellm/aws/examples/default/main.tf
+++ b/terraform/litellm/aws/examples/default/main.tf
@@ -48,4 +48,7 @@ module "litellm" {
   backend_extra_env     = var.backend_extra_env
   gateway_extra_secrets = var.gateway_extra_secrets
   backend_extra_secrets = var.backend_extra_secrets
+
+  gateway_metrics_port         = var.gateway_metrics_port
+  gateway_metrics_scrape_cidrs = var.gateway_metrics_scrape_cidrs
 }
diff --git a/terraform/litellm/aws/examples/default/terraform.tfvars.example b/terraform/litellm/aws/examples/default/terraform.tfvars.example
index 59301ea6aa5..880ecf56555 100644
--- a/terraform/litellm/aws/examples/default/terraform.tfvars.example
+++ b/terraform/litellm/aws/examples/default/terraform.tfvars.example
@@ -102,6 +102,13 @@ env    = "stage"
 #   }
 # }
 
+# ---------- Prometheus metrics sidecar ----------
+# Serve /metrics from a sidecar in the gateway task instead of the inference
+# workers. The port is not behind the ALB and has no auth: open it only to
+# your Prometheus subnets.
+# gateway_metrics_port         = 4001
+# gateway_metrics_scrape_cidrs = ["10.0.0.0/16"]
+
 # ---------- Extra env / secrets ----------
 # Plain-text env vars (non-sensitive). Land directly in the ECS task def.
 # gateway_extra_env = {
diff --git a/terraform/litellm/aws/examples/default/variables.tf b/terraform/litellm/aws/examples/default/variables.tf
index d8ab56b13af..f8140266fca 100644
--- a/terraform/litellm/aws/examples/default/variables.tf
+++ b/terraform/litellm/aws/examples/default/variables.tf
@@ -158,3 +158,15 @@ variable "backend_extra_secrets" {
   type        = map(string)
   default     = {}
 }
+
+variable "gateway_metrics_port" {
+  description = "Port for the Prometheus metrics sidecar in the gateway task. Null keeps /metrics on the gateway port only."
+  type        = number
+  default     = null
+}
+
+variable "gateway_metrics_scrape_cidrs" {
+  description = "CIDRs allowed to scrape gateway_metrics_port."
+  type        = list(string)
+  default     = []
+}
diff --git a/terraform/litellm/aws/network.tf b/terraform/litellm/aws/network.tf
index 4563eefbba5..c54949b4a59 100644
--- a/terraform/litellm/aws/network.tf
+++ b/terraform/litellm/aws/network.tf
@@ -156,6 +156,17 @@ resource "aws_security_group" "tasks" {
     security_groups = [aws_security_group.alb.id]
   }
 
+  dynamic "ingress" {
+    for_each = local.metrics_enabled && length(var.gateway_metrics_scrape_cidrs) > 0 ? [1] : []
+    content {
+      description = "Prometheus scrapers to the gateway metrics sidecar"
+      from_port   = var.gateway_metrics_port
+      to_port     = var.gateway_metrics_port
+      protocol    = "tcp"
+      cidr_blocks = var.gateway_metrics_scrape_cidrs
+    }
+  }
+
   egress {
     description = "All egress (LLM providers, RDS, Redis)"
     from_port   = 0
diff --git a/terraform/litellm/aws/tests/metrics_sidecar.tftest.hcl b/terraform/litellm/aws/tests/metrics_sidecar.tftest.hcl
new file mode 100644
index 00000000000..de839d7e5be
--- /dev/null
+++ b/terraform/litellm/aws/tests/metrics_sidecar.tftest.hcl
@@ -0,0 +1,108 @@
+# Plan-only coverage for the Prometheus metrics sidecar wiring. Offline via
+# mock_provider, same as byo_infrastructure.tftest.hcl.
+
+mock_provider "aws" {
+  mock_data "aws_iam_policy_document" {
+    defaults = {
+      json = "{\"Version\":\"2012-10-17\",\"Statement\":[]}"
+    }
+  }
+}
+mock_provider "random" {}
+
+variables {
+  region              = "us-east-1"
+  tenant              = "acme"
+  env                 = "test"
+  allow_plaintext_alb = true
+  azs                 = ["us-east-1a", "us-east-1b"]
+}
+
+run "defaults_change_nothing" {
+  command = plan
+
+  assert {
+    condition = alltrue([
+      length(local.gateway_metrics_container) == 0,
+      length(local.metrics_env) == 0,
+      length(local.metrics_mount_points) == 0,
+      length([for r in aws_security_group.tasks.ingress : r if r.description == "Prometheus scrapers to the gateway metrics sidecar"]) == 0,
+    ])
+    error_message = "The metrics sidecar, its env, its volume, and its security-group rule must all be absent by default."
+  }
+}
+
+run "metrics_port_adds_a_sidecar_volume_and_scrape_rule" {
+  command = plan
+
+  variables {
+    gateway_metrics_port         = 9464
+    gateway_metrics_scrape_cidrs = ["10.20.0.0/16"]
+  }
+
+  assert {
+    condition     = length(local.metrics_env) == 1 && local.metrics_env[0].name == "PROMETHEUS_MULTIPROC_DIR" && local.metrics_env[0].value == "/tmp/litellm_prometheus_multiproc"
+    error_message = "The gateway workers must write multiprocess samples to the shared dir."
+  }
+
+  assert {
+    condition     = length(local.metrics_mount_points) == 1 && local.metrics_mount_points[0].sourceVolume == "prometheus-multiproc" && local.metrics_mount_points[0].containerPath == "/tmp/litellm_prometheus_multiproc"
+    error_message = "Gateway and sidecar must mount the same task volume at the multiproc dir."
+  }
+
+  assert {
+    condition = alltrue([
+      length(local.gateway_metrics_container) == 1,
+      local.gateway_metrics_container[0].name == "metrics",
+      local.gateway_metrics_container[0].essential == false,
+      join(" ", local.gateway_metrics_container[0].entryPoint) == "python -m litellm.proxy.prometheus_metrics_server",
+      join(" ", local.gateway_metrics_container[0].command) == "--port 9464",
+      one(local.gateway_metrics_container[0].portMappings).containerPort == 9464,
+      one(local.gateway_metrics_container[0].environment).value == "/tmp/litellm_prometheus_multiproc",
+      one(local.gateway_metrics_container[0].mountPoints).sourceVolume == "prometheus-multiproc",
+      strcontains(local.gateway_metrics_container[0].healthCheck.command[3], "9464"),
+    ])
+    error_message = "The metrics sidecar must run prometheus_metrics_server on the configured port, share the multiproc volume, and health-check that port."
+  }
+
+  assert {
+    condition     = length(aws_ecs_task_definition.gateway.volume) == 1 && one(aws_ecs_task_definition.gateway.volume).name == "prometheus-multiproc"
+    error_message = "The gateway task must declare the multiproc volume."
+  }
+
+  assert {
+    condition = length([
+      for r in aws_security_group.tasks.ingress : r
+      if r.from_port == 9464 && r.to_port == 9464 && r.protocol == "tcp" && r.cidr_blocks == tolist(["10.20.0.0/16"])
+    ]) == 1
+    error_message = "The scrape CIDRs must be allowed to reach the metrics port on the tasks security group."
+  }
+
+  assert {
+    condition     = aws_lb_target_group.gateway.port == 4000 && one(aws_ecs_service.gateway.load_balancer).container_port == 4000
+    error_message = "The ALB must keep targeting the gateway port only; the metrics port is never load balanced."
+  }
+}
+
+run "metrics_port_without_scrape_cidrs_opens_nothing" {
+  command = plan
+
+  variables {
+    gateway_metrics_port = 9464
+  }
+
+  assert {
+    condition     = length(local.gateway_metrics_container) == 1 && length([for r in aws_security_group.tasks.ingress : r if r.from_port == 9464]) == 0
+    error_message = "Without scrape CIDRs the sidecar runs but the metrics port stays closed to everything but the ALB group."
+  }
+}
+
+run "metrics_port_may_not_reuse_the_gateway_port" {
+  command = plan
+
+  variables {
+    gateway_metrics_port = 4000
+  }
+
+  expect_failures = [var.gateway_metrics_port]
+}
diff --git a/terraform/litellm/aws/variables.tf b/terraform/litellm/aws/variables.tf
index 522138953d6..667f6db63c9 100644
--- a/terraform/litellm/aws/variables.tf
+++ b/terraform/litellm/aws/variables.tf
@@ -549,6 +549,44 @@ variable "proxy_config" {
   default     = {}
 }
 
+# ---------- Prometheus metrics sidecar ----------
+
+variable "gateway_metrics_port" {
+  description = <<-EOT
+    Serve Prometheus /metrics from a `metrics` sidecar container in the
+    gateway task on this port (1-65535, not 4000), so a scrape never runs on
+    an inference worker. The sidecar runs the gateway image with
+    `python -m litellm.proxy.prometheus_metrics_server` and aggregates the
+    workers' PROMETHEUS_MULTIPROC_DIR samples over a task volume. Null (the
+    default) leaves /metrics on the gateway port only. The sidecar port has
+    no virtual-key auth and is not routed through the ALB; open it to your
+    scrapers with gateway_metrics_scrape_cidrs. Needs gateway_image v1.101.0
+    or newer.
+  EOT
+  type        = number
+  default     = null
+
+  validation {
+    condition     = var.gateway_metrics_port == null || (var.gateway_metrics_port >= 1 && var.gateway_metrics_port <= 65535 && var.gateway_metrics_port != 4000)
+    error_message = "gateway_metrics_port must be between 1 and 65535 and must not be 4000 (the gateway port)."
+  }
+}
+
+variable "gateway_metrics_scrape_cidrs" {
+  description = <<-EOT
+    CIDR blocks allowed to reach gateway_metrics_port on the gateway tasks
+    (your Prometheus or collector subnets). Empty by default, so only the
+    ALB can reach the tasks. Ignored when gateway_metrics_port is null.
+  EOT
+  type        = list(string)
+  default     = []
+
+  validation {
+    condition     = alltrue([for c in var.gateway_metrics_scrape_cidrs : can(cidrnetmask(c))])
+    error_message = "gateway_metrics_scrape_cidrs must contain valid IPv4 CIDR blocks."
+  }
+}
+
 variable "log_retention_days" {
   description = "CloudWatch log retention for the three services."
   type        = number
diff --git a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py
index 7db85dc6943..dafe686c4f2 100644
--- a/tests/test_litellm/proxy/proxy_server/test_lifecycle.py
+++ b/tests/test_litellm/proxy/proxy_server/test_lifecycle.py
@@ -22,6 +22,7 @@ import inspect
 import json
 import logging
 import os
+import subprocess
 from collections.abc import Awaitable, Callable
 from typing import List, Optional, Union
 from unittest.mock import AsyncMock, MagicMock, patch
@@ -749,6 +750,30 @@ async def test_proxy_startup_event_invalid_missing_app_arg_raises():
             pass
 
 
+@pytest.mark.asyncio
+async def test_proxy_startup_event_prunes_dead_workers_live_gauges(tmp_path):
+    """With PROMETHEUS_MULTIPROC_DIR set, a booting worker drops the live-gauge files of pids that no longer
+    exist, so a crashed worker's in-flight samples leave the aggregate as soon as its replacement starts."""
+    exited = subprocess.Popen(["true"])
+    assert exited.wait(timeout=30) == 0
+    stale = tmp_path / f"gauge_livesum_{exited.pid}.db"
+    stale.touch()
+    counter = tmp_path / f"counter_{exited.pid}.db"
+    counter.touch()
+
+    clean_env = {k: v for k, v in os.environ.items() if k not in ("DATABASE_URL", "DIRECT_URL")}
+    clean_env["PROMETHEUS_MULTIPROC_DIR"] = str(tmp_path)
+    with patch.dict(os.environ, clean_env, clear=True):
+        try:
+            async with proxy_startup_event(app=None):
+                pass
+        except Exception:
+            pass
+
+    assert not stale.exists()
+    assert counter.exists()
+
+
 def test_otel_global_provider_published_after_callback_init():
     """The OTel V2 global-provider publish must run after callback
     initialization in ``proxy_startup_event``.
diff --git a/tests/test_litellm/proxy/test_prometheus_cleanup.py b/tests/test_litellm/proxy/test_prometheus_cleanup.py
index 93b9b694c2c..6a1b95c51ff 100644
--- a/tests/test_litellm/proxy/test_prometheus_cleanup.py
+++ b/tests/test_litellm/proxy/test_prometheus_cleanup.py
@@ -6,13 +6,77 @@ ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dir.
 from __future__ import annotations
 
 import os
+import subprocess
+import sys
+from pathlib import Path
+from typing import Final
 from unittest.mock import patch
 
 import pytest
+from prometheus_client import CollectorRegistry, multiprocess
 
-from litellm.proxy.prometheus_cleanup import mark_worker_exit, wipe_directory
+from litellm.proxy.prometheus_cleanup import mark_dead_workers, mark_worker_exit, wipe_directory
 from litellm.proxy.proxy_cli import ProxyInitializationHelpers
 
+_WORKER: Final = """
+import sys, time
+from prometheus_client import Gauge
+Gauge("litellm_in_flight", "", multiprocess_mode="livesum").set(float(sys.argv[1]))
+print("ready", flush=True)
+if sys.argv[2] == "stay":
+    time.sleep(120)
+"""
+
+
+def _spawn_worker(directory: Path, in_flight: str, lifetime: str) -> subprocess.Popen[str]:
+    env = {**os.environ, "PROMETHEUS_MULTIPROC_DIR": str(directory)}
+    worker = subprocess.Popen(
+        [sys.executable, "-c", _WORKER, in_flight, lifetime], env=env, stdout=subprocess.PIPE, text=True
+    )
+    assert worker.stdout is not None and worker.stdout.readline() == "ready\n"
+    return worker
+
+
+def _livesum(directory: Path) -> float:
+    registry = CollectorRegistry()
+    multiprocess.MultiProcessCollector(registry, path=str(directory))
+    value = registry.get_sample_value("litellm_in_flight")
+    return 0.0 if value is None else value
+
+
+class TestMarkDeadWorkers:
+    def test_drops_live_gauges_of_exited_workers_and_keeps_running_ones(self, tmp_path: Path) -> None:
+        """A worker that died mid-request leaves its livesum file behind; the replacement worker's startup prune
+        must remove exactly that file so the aggregate stops counting requests nobody is serving."""
+        dead = _spawn_worker(tmp_path, "3", "exit")
+        assert dead.wait(timeout=30) == 0
+        alive = _spawn_worker(tmp_path, "2", "stay")
+        try:
+            assert (tmp_path / f"gauge_livesum_{dead.pid}.db").exists()
+            assert _livesum(tmp_path) == 5.0
+
+            assert mark_dead_workers(str(tmp_path)) == (dead.pid,)
+
+            assert not (tmp_path / f"gauge_livesum_{dead.pid}.db").exists()
+            assert (tmp_path / f"gauge_livesum_{alive.pid}.db").exists()
+            assert _livesum(tmp_path) == 2.0
+            assert mark_dead_workers(str(tmp_path)) == ()
+        finally:
+            alive.kill()
+            alive.wait(timeout=30)
+
+    def test_leaves_counters_of_exited_workers_alone(self, tmp_path: Path) -> None:
+        (tmp_path / "counter_424242.db").touch()
+        (tmp_path / "histogram_424242.db").touch()
+        assert mark_dead_workers(str(tmp_path)) == ()
+        assert sorted(p.name for p in tmp_path.glob("*.db")) == ["counter_424242.db", "histogram_424242.db"]
+
+    def test_keeps_live_gauges_of_workers_it_may_not_signal(self, tmp_path: Path) -> None:
+        """Signal 0 to pid 1 raises PermissionError for an unprivileged proxy; that pid is alive, not dead."""
+        (tmp_path / "gauge_livesum_1.db").touch()
+        assert mark_dead_workers(str(tmp_path)) == ()
+        assert (tmp_path / "gauge_livesum_1.db").exists()
+
 
 class TestWipeDirectory:
     def test_deletes_all_db_files(self, tmp_path):
diff --git a/tests/test_litellm/proxy/test_prometheus_metrics_server.py b/tests/test_litellm/proxy/test_prometheus_metrics_server.py
index fc1fa381fa4..e2f461e61a6 100644
--- a/tests/test_litellm/proxy/test_prometheus_metrics_server.py
+++ b/tests/test_litellm/proxy/test_prometheus_metrics_server.py
@@ -1,5 +1,5 @@
-"""The separate metrics server must aggregate PROMETHEUS_MULTIPROC_DIR, expose only /metrics, and follow its
-parent's lifetime.
+"""The separate metrics server must aggregate PROMETHEUS_MULTIPROC_DIR, expose /metrics plus a probe-friendly
+/health, and follow its parent's lifetime.
 
 Everything here runs on loopback against a child of this test process; no LLM keys or external network.
 """
@@ -91,7 +91,10 @@ def test_metrics_app_aggregates_multiproc_dir_and_reports_pid(tmp_path: Path, mo
     assert metrics.headers[PID_HEADER] == str(os.getpid())
     assert 'litellm_requests_metric_total{model="gpt-5"} 5.0' in metrics.text
 
-    assert client.get("/health").status_code == 404
+    health: Final = client.get("/health")
+    assert health.status_code == 200
+    assert health.json() == {"status": "healthy", "multiproc_dir": str(tmp_path)}
+    assert client.get("/docs").status_code == 404
 
     empty: Final = TestClient(build_metrics_app(str(other_dir))).get("/metrics")
     assert empty.status_code == 200
diff --git a/tests/test_litellm/test_component_entrypoint.py b/tests/test_litellm/test_component_entrypoint.py
index b0969e2c694..09837d2b233 100644
--- a/tests/test_litellm/test_component_entrypoint.py
+++ b/tests/test_litellm/test_component_entrypoint.py
@@ -223,6 +223,59 @@ def test_gating_matches_the_monolithic_entrypoint_and_get_secret_bool(
     assert monolith[1] == ("args=litellm --port 4000" if traced else "args=--port 4000")
 
 
+def test_wipes_the_prometheus_multiproc_dir_before_uvicorn_forks(tmp_path: Path) -> None:
+    """A restarted container inherits the emptyDir of its predecessor, whose worker pids it may reuse, so the
+    stale .db files must be gone before any worker opens the one carrying its own pid."""
+    multiproc_dir = tmp_path / "multiproc"
+    multiproc_dir.mkdir()
+    (multiproc_dir / "gauge_livesum_7.db").write_bytes(b"stale")
+    (multiproc_dir / "counter_7.db").write_bytes(b"stale")
+    (multiproc_dir / "keep.txt").write_text("not a sample")
+
+    bin_dir = tmp_path / "bin"
+    bin_dir.mkdir()
+    _write_stubs(bin_dir, ("uvicorn",))
+    record = tmp_path / "record.txt"
+    env = {
+        **os.environ,
+        "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
+        "RECORD": str(record),
+        "PROMETHEUS_MULTIPROC_DIR": str(multiproc_dir),
+    }
+    env.pop("USE_DDTRACE", None)
+    result = subprocess.run(
+        ["sh", str(COMPONENT_ENTRYPOINT), "uvicorn", "gateway.main:app"],
+        env=env,
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+
+    assert result.returncode == 0, f"stdout={result.stdout} stderr={result.stderr}"
+    assert sorted(p.name for p in multiproc_dir.iterdir()) == ["keep.txt"]
+    assert record.read_text().splitlines()[0] == "exec=uvicorn"
+
+
+def test_creates_a_missing_prometheus_multiproc_dir(tmp_path: Path) -> None:
+    bin_dir = tmp_path / "bin"
+    bin_dir.mkdir()
+    _write_stubs(bin_dir, ("uvicorn",))
+    missing = tmp_path / "multiproc"
+    env = {
+        **os.environ,
+        "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
+        "RECORD": str(tmp_path / "record.txt"),
+        "PROMETHEUS_MULTIPROC_DIR": str(missing),
+    }
+    env.pop("USE_DDTRACE", None)
+    result = subprocess.run(
+        ["sh", str(COMPONENT_ENTRYPOINT), "uvicorn", "gateway.main:app"], env=env, capture_output=True, text=True
+    )
+
+    assert result.returncode == 0, f"stdout={result.stdout} stderr={result.stderr}"
+    assert missing.is_dir()
+
+
 def _copied_script(dockerfile: Path, image_path: str) -> Path:
     """Resolve the repo file a Dockerfile `COPY`s to `image_path`, so tests run what the image ships."""
     matches = _COPY_RE.findall(dockerfile.read_text())

From 0175c7da1c9c3d6f71c77c52436f250d43c3a8a3 Mon Sep 17 00:00:00 2001
From: tin-berri <tin@berri.ai>
Date: Tue, 8 Sep 2026 13:31:28 -0700
Subject: [PATCH 71/85] feat(router): resolve max_tokens to the tier model's
 ceiling on auto-routed requests (#40209)

A client behind an auto-router sends one max_tokens for every tier, so a value
sized for the smallest tier starves a bigger tier's thinking budget and a value
sized for the biggest is rejected by the smallest. After the complexity router
picks a tier, its per-tier litellm_params now carry max_tokens set to the
smallest max_output_tokens across that tier model's deployments (model_info,
then the cost map), applied the same way a per-tier reasoning_effort already
is, on every routing exit including plan mode, the empty-ask default and the
classifier fallback. The router seam collapses whichever ceiling alias a tier
carries onto the surface's own name, so one tier max_tokens reaches chat,
/v1/messages and /v1/responses alike, drops the caller's other carriers of the
same setting before the merge, and stamps the caller's original once so a
fallback into a group no tier owns gets it back instead of a ceiling sized for
the tier that failed.

Proxy-level reservations were sized from the caller's cap before routing, so a
raised cap left them short. Both owners now re-validate at the deployment hook:
the v3 limiter tops up its combined-TPM and project-OTPM reservations to the
final cap or writes the admitted cap back, and the budget limiter re-estimates
on the chosen deployment and grows the reservation or writes the admitted cap
back. An auto-router alias also reserves budget at its priciest tier model now
instead of pricing to zero.

An explicit per-tier max_tokens, max_completion_tokens or max_output_tokens
still wins, and max_tokens_from_tier_model: false forwards the caller's value
unchanged.
---
 litellm/constants.py                          |   2 +
 litellm/proxy/common_utils/callback_utils.py  |   2 +
 litellm/proxy/litellm_pre_call_utils.py       |   5 +-
 litellm/router.py                             | 150 ++++++-
 .../complexity_router/complexity_router.py    |  57 ++-
 .../complexity_router/config.py               |  14 +
 .../proxy/test_litellm_pre_call_utils.py      |   8 +-
 .../router_strategy/test_complexity_router.py | 392 +++++++++++++++++-
 ui/litellm-dashboard/src/lib/http/schema.d.ts |   6 +
 9 files changed, 602 insertions(+), 34 deletions(-)

diff --git a/litellm/constants.py b/litellm/constants.py
index 8ef9523a60a..8a6a044ff7b 100644
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -1461,6 +1461,8 @@ LITELLM_METADATA_FIELD: Final = "litellm_metadata"
 OLD_LITELLM_METADATA_FIELD: Final = "metadata"
 RETURN_RAW_MODEL_NAME_METADATA_KEY: Final = "_complexity_router_return_raw_model_name"
 SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: Final = "_session_deployment_affinity_ttl"
+OUTPUT_TOKEN_CEILING_PARAMS: Final = frozenset({"max_tokens", "max_completion_tokens", "max_output_tokens"})
+CLIENT_OUTPUT_CEILING_METADATA_KEY: Final = "_client_output_ceiling"
 CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags"
 INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
 SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"
diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py
index 39e74d2c8bd..d9d56778405 100644
--- a/litellm/proxy/common_utils/callback_utils.py
+++ b/litellm/proxy/common_utils/callback_utils.py
@@ -10,6 +10,7 @@ import litellm
 from litellm import get_secret
 from litellm._logging import verbose_proxy_logger
 from litellm.constants import (
+    CLIENT_OUTPUT_CEILING_METADATA_KEY,
     CONSUMED_REQUEST_TAGS_METADATA_KEY,
     PRE_CALL_EXECUTED_GUARDRAILS_KEY,
     SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
@@ -507,6 +508,7 @@ LITELLM_PROXY_INTERNAL_METADATA_KEYS: Final = frozenset(
         PRE_CALL_EXECUTED_GUARDRAILS_KEY,
         SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
         CONSUMED_REQUEST_TAGS_METADATA_KEY,
+        CLIENT_OUTPUT_CEILING_METADATA_KEY,
         "disable_global_guardrails",
         "disable_global_guardrail",
         "opted_out_global_guardrails",
diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py
index 3c186e19829..893174dab03 100644
--- a/litellm/proxy/litellm_pre_call_utils.py
+++ b/litellm/proxy/litellm_pre_call_utils.py
@@ -18,6 +18,7 @@ from litellm._logging import verbose_logger, verbose_proxy_logger
 from litellm._service_logger import ServiceLogging
 from litellm._uuid import uuid
 from litellm.constants import (
+    CLIENT_OUTPUT_CEILING_METADATA_KEY,
     CONSUMED_REQUEST_TAGS_METADATA_KEY,
     INTERNAL_CALL_ORIGIN_METADATA_KEY,
     LITELLM_PROXY_MASTER_KEY_ALIAS,
@@ -325,7 +326,9 @@ _CLIENT_PRICING_METADATA_FIELDS: Final = frozenset({"model_info", "standard_logg
 # ``attempted_fallbacks`` and ``original_model_group`` are written by the router
 # and read by spend logs as fact; a client value has no legitimate meaning and no
 # key or team setting keeps it, so the strip is never gated.
-_ROUTER_RESERVED_METADATA_FIELDS: Final = frozenset({"attempted_fallbacks", "original_model_group"})
+_ROUTER_RESERVED_METADATA_FIELDS: Final = frozenset(
+    {"attempted_fallbacks", "original_model_group", CLIENT_OUTPUT_CEILING_METADATA_KEY}
+)
 _ALLOW_CLIENT_PRICING_OVERRIDE_METADATA_KEY: Final = "allow_client_pricing_override"
 
 # Request fields whose value, when URL-valued, becomes the outbound destination
diff --git a/litellm/router.py b/litellm/router.py
index 843b916d90f..cf62ce1eb2d 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -21,7 +21,16 @@ import time
 import traceback
 import weakref
 from collections import defaultdict
-from collections.abc import AsyncGenerator, AsyncIterator, Callable, Generator, Iterator, Mapping, Sequence
+from collections.abc import (
+    AsyncGenerator,
+    AsyncIterator,
+    Callable,
+    Generator,
+    Iterator,
+    Mapping,
+    MutableMapping,
+    Sequence,
+)
 from functools import lru_cache, partial
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, Optional, TypeAlias, TypeVar, Union, cast
@@ -45,12 +54,14 @@ from litellm.caching.caching import (
     RedisClusterCache,
 )
 from litellm.constants import (
+    CLIENT_OUTPUT_CEILING_METADATA_KEY,
     CONSUMED_REQUEST_TAGS_METADATA_KEY,
     DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS,
     DEFAULT_HEALTH_CHECK_INTERVAL,
     DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER,
     DEFAULT_MAX_LRU_CACHE_SIZE,
     INTERNAL_CALL_ORIGIN_METADATA_KEY,
+    OUTPUT_TOKEN_CEILING_PARAMS,
     RUNTIME_UPDATABLE_ROUTER_SETTINGS,
     SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
 )
@@ -648,6 +659,18 @@ class FallbackAwareStreamWrapper(CustomStreamWrapper):
         self.fallback_headers_adopted = True
 
 
+def as_output_cap(value: object) -> int | None:
+    """A client-sent output cap coerced to an int: ints, floats and numeric strings, never bools
+    or negatives."""
+    if isinstance(value, bool) or not isinstance(value, (int, float, str)):
+        return None
+    try:
+        cap: Final = int(float(value))
+    except (ValueError, OverflowError):
+        return None
+    return cap if cap >= 0 else None
+
+
 class Router:
     model_names: set = set()
     cache_responses: bool | None = False
@@ -12606,7 +12629,83 @@ class Router:
             request_kwargs.pop(carrier, None)
 
     @staticmethod
-    def _drop_client_effort_carriers_a_tier_pin_supersedes(
+    def _tier_ceiling_under_the_surface_name(
+        tier_litellm_params: Mapping[str, object], responses_call: bool
+    ) -> Mapping[str, object]:
+        """``max_tokens``, ``max_completion_tokens`` and ``max_output_tokens`` are one
+        ceiling under three names, and each surface reads exactly one of them: the
+        Responses bridge builds its internal ``max_tokens`` from ``max_output_tokens``
+        and would overwrite the tier's, chat and /v1/messages never read
+        ``max_output_tokens``, and litellm already renames ``max_tokens`` to
+        ``max_completion_tokens`` for the OpenAI models that require it. Collapse
+        whatever the tier carries onto the surface's own name, preferring a value the
+        operator already wrote under that name."""
+        surface_key: Final = "max_output_tokens" if responses_call else "max_tokens"
+        carried: Final = tuple(
+            key
+            for key in (surface_key, "max_tokens", "max_completion_tokens", "max_output_tokens")
+            if key in tier_litellm_params
+        )
+        if not carried:
+            return tier_litellm_params
+        return MappingProxyType(
+            {
+                **{k: v for k, v in tier_litellm_params.items() if k not in OUTPUT_TOKEN_CEILING_PARAMS},
+                surface_key: tier_litellm_params[carried[0]],
+            }
+        )
+
+    def _pin_tier_params_onto_request(
+        self,
+        model: str,
+        tier_litellm_params: Mapping[str, object] | None,
+        request_kwargs: dict,
+        responses_call: bool,
+    ) -> bool:
+        """Apply a routing strategy's per-tier litellm_params on top of the request and report
+        whether they pinned an output ceiling, so the caller can hand the request its own ceiling
+        back on a routing pass that pins none."""
+        if not tier_litellm_params:
+            return False
+        accepted_tier_params: Final = self._tier_params_the_target_accepts(model, tier_litellm_params, request_kwargs)
+        surface_tier_params: Final = self._tier_ceiling_under_the_surface_name(
+            accepted_tier_params, responses_call=responses_call
+        )
+        self._drop_client_carriers_a_tier_pin_supersedes(request_kwargs, surface_tier_params)
+        request_kwargs.update(surface_tier_params)
+        return not OUTPUT_TOKEN_CEILING_PARAMS.isdisjoint(surface_tier_params)
+
+    @staticmethod
+    def _restore_client_ceiling_no_tier_pins(request_kwargs: MutableMapping[str, object]) -> None:
+        """A model-group fallback re-enters routing with the kwargs an earlier auto-router pass
+        already rewrote, so a ceiling sized for that pass's tier would ride onto a group no tier
+        chose. When this pass pins none, hand the request back exactly the carriers the caller
+        sent, which the first pinning pass stamped. The stamp lives in a metadata bucket a
+        caller can also write, so the proxy strips the key at ingestion and this read takes
+        nothing but the three ceiling carriers as integers: no other key ever reaches kwargs."""
+        stamped: Final = next(
+            (
+                bucket.get(CLIENT_OUTPUT_CEILING_METADATA_KEY)
+                for bucket in (request_kwargs.get("metadata"), request_kwargs.get("litellm_metadata"))
+                if isinstance(bucket, dict) and CLIENT_OUTPUT_CEILING_METADATA_KEY in bucket
+            ),
+            None,
+        )
+        if not isinstance(stamped, dict):
+            return
+        callers_ceiling: Final = MappingProxyType(
+            {
+                carrier: cap
+                for carrier, value in stamped.items()
+                if carrier in OUTPUT_TOKEN_CEILING_PARAMS and (cap := as_output_cap(value)) is not None
+            }
+        )
+        for carrier in OUTPUT_TOKEN_CEILING_PARAMS:
+            request_kwargs.pop(carrier, None)
+        request_kwargs.update(callers_ceiling)
+
+    @staticmethod
+    def _drop_client_carriers_a_tier_pin_supersedes(
         request_kwargs: dict[str, object],
         tier_litellm_params: Mapping[str, object],
     ) -> None:
@@ -12616,7 +12715,22 @@ class Router:
         the ``reasoning_effort`` alias, so a pinned effort only reaches the wire
         if the client's other encodings are removed before the merge. Non-effort
         fields a carrier also holds (``output_config.format``,
-        ``reasoning.summary``) are kept."""
+        ``reasoning.summary``) are kept. An output ceiling has the same shape:
+        ``max_tokens``, ``max_completion_tokens`` and ``max_output_tokens`` are
+        one setting under three names, and a provider handed two of them either
+        rejects the request or picks one by iteration order."""
+        if not OUTPUT_TOKEN_CEILING_PARAMS.isdisjoint(tier_litellm_params):
+            _, metadata_bucket = get_or_create_metadata_bucket(request_kwargs)
+            metadata_bucket.setdefault(
+                CLIENT_OUTPUT_CEILING_METADATA_KEY,
+                {
+                    carrier: request_kwargs[carrier]
+                    for carrier in OUTPUT_TOKEN_CEILING_PARAMS
+                    if carrier in request_kwargs
+                },
+            )
+            for carrier in OUTPUT_TOKEN_CEILING_PARAMS:
+                request_kwargs.pop(carrier, None)
         if "reasoning_effort" not in tier_litellm_params:
             return
         request_kwargs.pop("thinking", None)
@@ -12657,6 +12771,7 @@ class Router:
             # Execute Pre-Routing Hooks
             # this hook can modify the model, messages before the routing decision is made
             #########################################################
+            responses_call: Final = input is not None and messages is None
             pre_routing_hook_response: Final = await self.async_pre_routing_hook(
                 model=model,
                 request_kwargs=request_kwargs,
@@ -12668,12 +12783,14 @@ class Router:
                 model = pre_routing_hook_response.model
                 messages = pre_routing_hook_response.messages
                 record_pre_routing_selection(request_kwargs, model)
-                if pre_routing_hook_response.litellm_params:
-                    accepted_tier_params: Final = self._tier_params_the_target_accepts(
-                        model, pre_routing_hook_response.litellm_params, request_kwargs
-                    )
-                    self._drop_client_effort_carriers_a_tier_pin_supersedes(request_kwargs, accepted_tier_params)
-                    request_kwargs.update(accepted_tier_params)
+            tier_pins_ceiling: Final = self._pin_tier_params_onto_request(
+                model=model,
+                tier_litellm_params=pre_routing_hook_response.litellm_params if pre_routing_hook_response else None,
+                request_kwargs=request_kwargs,
+                responses_call=responses_call,
+            )
+            if not tier_pins_ceiling:
+                self._restore_client_ceiling_no_tier_pins(request_kwargs)
             #########################################################
 
             # Resolve the strategy and logger AFTER the pre-routing hook, since
@@ -12773,6 +12890,7 @@ class Router:
             parent_otel_span: Final = _get_parent_otel_span_from_kwargs(request_kwargs)
 
             # 1. Execute pre-routing hook
+            responses_call: Final = input is not None and messages is None
             pre_routing_hook_response: Final = await self.async_pre_routing_hook(
                 model=model,
                 request_kwargs=request_kwargs,
@@ -12784,12 +12902,14 @@ class Router:
                 model = pre_routing_hook_response.model
                 messages = pre_routing_hook_response.messages
                 record_pre_routing_selection(request_kwargs, model)
-                if pre_routing_hook_response.litellm_params:
-                    accepted_tier_params: Final = self._tier_params_the_target_accepts(
-                        model, pre_routing_hook_response.litellm_params, request_kwargs
-                    )
-                    self._drop_client_effort_carriers_a_tier_pin_supersedes(request_kwargs, accepted_tier_params)
-                    request_kwargs.update(accepted_tier_params)
+            tier_pins_ceiling: Final = self._pin_tier_params_onto_request(
+                model=model,
+                tier_litellm_params=pre_routing_hook_response.litellm_params if pre_routing_hook_response else None,
+                request_kwargs=request_kwargs,
+                responses_call=responses_call,
+            )
+            if not tier_pins_ceiling:
+                self._restore_client_ceiling_no_tier_pins(request_kwargs)
 
             # 2. Get healthy deployments
             healthy_deployments: Final = await self.async_get_healthy_deployments(
diff --git a/litellm/router_strategy/complexity_router/complexity_router.py b/litellm/router_strategy/complexity_router/complexity_router.py
index ae1f82fc40b..f9b4f50c15c 100644
--- a/litellm/router_strategy/complexity_router/complexity_router.py
+++ b/litellm/router_strategy/complexity_router/complexity_router.py
@@ -31,6 +31,7 @@ from litellm._logging import verbose_router_logger
 from litellm.constants import (
     EMPTY_MAPPING,
     INTERNAL_CALL_ORIGIN_METADATA_KEY,
+    OUTPUT_TOKEN_CEILING_PARAMS,
     RETURN_RAW_MODEL_NAME_METADATA_KEY,
     SESSION_ID_GENERATED_METADATA_KEY,
 )
@@ -2110,11 +2111,15 @@ class ComplexityRouter(CustomLogger):
         raise ValueError(f"No model configured for tier {tier_key} and no default_model set")
 
     def _litellm_params_for_model(self, tier: ComplexityTier | str | None, model: str) -> Mapping[str, object]:
-        if tier is None:
-            return MappingProxyType({})
-        entries: Final = self.config.tier_model_configs.get(_tier_name(tier), ())
+        entries: Final = self.config.tier_model_configs.get(_tier_name(tier), ()) if tier is not None else ()
         entry: Final = next((candidate for candidate in entries if candidate.model_name == model), None)
-        return entry.litellm_params if entry is not None else MappingProxyType({})
+        explicit: Final = entry.litellm_params if entry is not None else MappingProxyType({})
+        if not self.config.max_tokens_from_tier_model or not OUTPUT_TOKEN_CEILING_PARAMS.isdisjoint(explicit):
+            return explicit
+        ceiling: Final = self._group_output_ceiling(model)
+        if ceiling is None:
+            return explicit
+        return MappingProxyType({**explicit, "max_tokens": ceiling})
 
     @staticmethod
     def _pick_from_tier_value(model: str | Sequence[str], tier_key: str) -> str:
@@ -2449,12 +2454,15 @@ class ComplexityRouter(CustomLogger):
         return name if self.config.has_custom_tiers else ComplexityTier(name)
 
     def _deployment_window(self, group: str, deployment: Mapping[str, object]) -> int | None:
+        return self._deployment_limit(group, deployment, "max_input_tokens")
+
+    def _deployment_limit(
+        self, group: str, deployment: Mapping[str, object], key: Literal["max_input_tokens", "max_output_tokens"]
+    ) -> int | None:
         from litellm.litellm_core_utils.get_llm_provider_logic import declared_authenticating_provider
 
         deployment_model_info: Final = deployment.get("model_info")
-        declared: Final = (
-            deployment_model_info.get("max_input_tokens") if isinstance(deployment_model_info, Mapping) else None
-        )
+        declared: Final = deployment_model_info.get(key) if isinstance(deployment_model_info, Mapping) else None
         if isinstance(declared, int):
             return declared
         litellm_params: Final = deployment.get("litellm_params")
@@ -2471,18 +2479,34 @@ class ComplexityRouter(CustomLogger):
                 deployment=cast(dict, deployment),  # cast-ok: router deployments are plain dicts
                 received_model_name=group,
             )
-            window: Final = model_info.get("max_input_tokens")
+            limit: Final = model_info.get(key)
         except Exception:  # noqa: BLE001  # best-effort: an unmappable deployment must not hide the others
             return None
-        return window if isinstance(window, int) else None
+        return limit if isinstance(limit, int) else None
+
+    def _group_deployments(self, group: str) -> Sequence[Mapping[str, object]]:
+        list_models: Final = getattr(self.litellm_router_instance, "get_model_list", None)
+        deployments: Final = list_models(model_name=group) if callable(list_models) else None
+        return tuple(deployments) if isinstance(deployments, list) else ()
+
+    def _group_output_ceiling(self, group: str) -> int | None:
+        """Smallest max_output_tokens across the group's deployments, or None when any deployment
+        declares none: the core router picks within the group without a fit check, and a ceiling
+        above an unmapped member's real limit is a provider 400 on that member."""
+        deployments: Final = self._group_deployments(group)
+        ceilings: Final = tuple(
+            ceiling
+            for deployment in deployments
+            if (ceiling := self._deployment_limit(group, deployment, "max_output_tokens")) is not None
+        )
+        return min(ceilings) if ceilings and len(ceilings) == len(deployments) else None
 
     def _group_window_facts(self, group: str) -> tuple[int | None, bool]:
         """(smallest declared context window across the group's deployments, whether any deployment
         declares none). The core router picks a deployment within the group without a fit check, so
         the group is only as safe as its smallest member."""
-        list_models: Final = getattr(self.litellm_router_instance, "get_model_list", None)
-        deployments: Final = list_models(model_name=group) if callable(list_models) else None
-        if not isinstance(deployments, list) or not deployments:
+        deployments: Final = self._group_deployments(group)
+        if not deployments:
             return (None, True)
         windows: Final = tuple(
             window for deployment in deployments if (window := self._deployment_window(group, deployment)) is not None
@@ -3531,6 +3555,7 @@ class ComplexityRouter(CustomLogger):
                     ComplexityTier.MEDIUM, messages, resolved_messages, request_kwargs
                 )
             fallback_tier: Final = None if default_model_first else ComplexityTier.MEDIUM
+            default_tier_params: Final = self._litellm_params_for_model(fallback_tier, routed_model)
             return PreRoutingHookResponse(
                 model=routed_model,
                 messages=messages if has_original_messages else None,
@@ -3539,7 +3564,9 @@ class ComplexityRouter(CustomLogger):
                     cause="default_fallback",
                     tier=fallback_tier,
                     conversation_continuing=conversation_continuing,
+                    tier_litellm_params=default_tier_params,
                 ),
+                litellm_params=default_tier_params,
             )
 
         ask: Final = user_message or ""
@@ -3566,6 +3593,7 @@ class ComplexityRouter(CustomLogger):
                 _tier_name(plan_floor),
                 routed_model,
             )
+            plan_tier_params: Final = self._litellm_params_for_model(plan_floor, routed_model)
             return PreRoutingHookResponse(
                 model=routed_model,
                 messages=messages if has_original_messages else None,
@@ -3577,7 +3605,9 @@ class ComplexityRouter(CustomLogger):
                     matched_keyword=plan_mode_sentinel,
                     escalation_keyword=escalation_keyword,
                     escalated=False,
+                    tier_litellm_params=plan_tier_params,
                 ),
+                litellm_params=plan_tier_params,
             )
 
         override: Final = await self._resolve_keyword_tier_override(ask, request_kwargs)
@@ -3670,6 +3700,7 @@ class ComplexityRouter(CustomLogger):
                 outcome.signals,
                 fallback_model,
             )
+            fallback_tier_params: Final = self._litellm_params_for_model(None, fallback_model)
             return PreRoutingHookResponse(
                 model=fallback_model,
                 messages=messages if has_original_messages else None,
@@ -3680,7 +3711,9 @@ class ComplexityRouter(CustomLogger):
                     signals=outcome.signals,
                     escalation_keyword=escalation_keyword,
                     escalated=False,
+                    tier_litellm_params=fallback_tier_params,
                 ),
+                litellm_params=fallback_tier_params,
             )
         if self.config.adaptive:
             # hard_floor rather than a hard pick, and passed whenever the sentinel is present
diff --git a/litellm/router_strategy/complexity_router/config.py b/litellm/router_strategy/complexity_router/config.py
index 37375d9727d..58f56d746c5 100644
--- a/litellm/router_strategy/complexity_router/config.py
+++ b/litellm/router_strategy/complexity_router/config.py
@@ -1089,6 +1089,20 @@ class ComplexityRouterConfig(BaseModel):
             "wording the built-ins don't cover, or after a client release changes its strings."
         ),
     )
+    max_tokens_from_tier_model: bool = Field(
+        default=True,
+        description=(
+            "Set max_tokens on every routed request to the output ceiling of the tier model it "
+            "lands on, replacing whatever the caller sent. A caller behind an auto-router cannot "
+            "pick one value that fits every tier: the smallest tier's ceiling starves a bigger "
+            "tier's thinking budget, and a bigger tier's ceiling is rejected by the smallest. The "
+            "ceiling is the smallest max_output_tokens across the tier model's deployments, read "
+            "from each deployment's model_info and then the model cost map; a tier model with a "
+            "deployment whose ceiling is unknown keeps the caller's value. A max_tokens, "
+            "max_completion_tokens or max_output_tokens in the tier's own litellm_params still "
+            "wins. Set false to forward the caller's value unchanged."
+        ),
+    )
     route_housekeeping_to_cheapest_tier: bool = Field(
         default=True,
         description=(
diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py
index 7070617ce3e..74799072984 100644
--- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py
+++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py
@@ -7272,7 +7272,12 @@ def _reserved_stamp_key(key_metadata: dict | None = None) -> UserAPIKeyAuth:
     )
 
 
-_PLANTED_STAMPS = {"attempted_fallbacks": 99, "original_model_group": "spoofed-group", "client_key": "client_value"}
+_PLANTED_STAMPS = {
+    "attempted_fallbacks": 99,
+    "original_model_group": "spoofed-group",
+    "_client_output_ceiling": {"api_base": "https://attacker.example"},
+    "client_key": "client_value",
+}
 
 
 @pytest.mark.asyncio
@@ -7301,6 +7306,7 @@ async def test_add_litellm_data_to_request_strips_router_reserved_stamps_from_bo
     assert "litellm_metadata" not in updated
     assert "attempted_fallbacks" not in updated["metadata"]
     assert "original_model_group" not in updated["metadata"]
+    assert "_client_output_ceiling" not in updated["metadata"]
     assert updated["metadata"]["client_key"] == "client_value"
 
 
diff --git a/tests/test_litellm/router_strategy/test_complexity_router.py b/tests/test_litellm/router_strategy/test_complexity_router.py
index b0f2a65bb9b..565e77b0ae0 100644
--- a/tests/test_litellm/router_strategy/test_complexity_router.py
+++ b/tests/test_litellm/router_strategy/test_complexity_router.py
@@ -16,6 +16,7 @@ from pydantic import ValidationError
 
 import litellm
 from litellm import Router
+from litellm.integrations.custom_logger import CustomLogger
 from litellm.router_utils.auto_router_model_naming import (
     CUSTOMIZATION_CAPABILITY,
     GATED_AUTO_ROUTER_CAPABILITIES,
@@ -24,7 +25,12 @@ from litellm.router_utils.auto_router_model_naming import (
 )
 from litellm._logging import verbose_router_logger
 from litellm.caching.dual_cache import DualCache
-from litellm.constants import RETURN_RAW_MODEL_NAME_METADATA_KEY, SESSION_ID_GENERATED_METADATA_KEY
+from litellm.constants import (
+    OUTPUT_TOKEN_CEILING_PARAMS,
+    RETURN_RAW_MODEL_NAME_METADATA_KEY,
+    SESSION_ID_GENERATED_METADATA_KEY,
+)
+from litellm.router import as_output_cap
 from litellm.router_strategy.complexity_router.complexity_router import (
     _CLASSIFICATION_CURRENT_MESSAGE_ONLY,
     _CLASSIFICATION_WITH_CONVERSATION,
@@ -3415,11 +3421,11 @@ class TestRouterPreRoutingAliasOverrides:
 
     def test_drop_client_effort_carriers_helper_edge_shapes(self):
         no_pin: Dict = {"thinking": {"type": "adaptive"}}
-        Router._drop_client_effort_carriers_a_tier_pin_supersedes(no_pin, {"temperature": 0.1})
+        Router._drop_client_carriers_a_tier_pin_supersedes(no_pin, {"temperature": 0.1})
         assert no_pin == {"thinking": {"type": "adaptive"}}
 
         non_dict_carriers: Dict = {"output_config": "max", "reasoning": 3}
-        Router._drop_client_effort_carriers_a_tier_pin_supersedes(non_dict_carriers, {"reasoning_effort": "low"})
+        Router._drop_client_carriers_a_tier_pin_supersedes(non_dict_carriers, {"reasoning_effort": "low"})
         assert non_dict_carriers == {"output_config": "max", "reasoning": 3}
 
         effort_only: Dict = {"output_config": {"effort": "max"}, "reasoning": {"effort": "high"}}
@@ -3873,11 +3879,11 @@ class TestRouterPreRoutingSharedAliasName:
         }
 
     @staticmethod
-    async def _routed_call_kwargs(router: Router, **request_params) -> dict:
+    async def _routed_call_kwargs(router: Router, prompt: str = "hi", **request_params) -> dict:
         mock_acompletion = AsyncMock(return_value=litellm.ModelResponse(choices=[{"message": {"content": "hi"}}]))
         with patch.object(litellm, "acompletion", mock_acompletion):
             await router.acompletion(
-                model="smart-router", messages=[{"role": "user", "content": "hi"}], **request_params
+                model="smart-router", messages=[{"role": "user", "content": prompt}], **request_params
             )
         return mock_acompletion.call_args.kwargs
 
@@ -13278,3 +13284,379 @@ class TestClassifierVision:
     def test_max_images_must_be_positive(self):
         with pytest.raises(ValidationError):
             ClassifierLLMConfig(model="clf", vision={"enabled": True, "max_images": 0})
+
+
+class TestMaxTokensFromTierModel:
+    """The auto-router replaces the caller's output ceiling with the tier model's own, so one
+    client-side value no longer starves a bigger tier or gets rejected by a smaller one."""
+
+    COMPLEX_PROMPT: Final = (
+        "Design a distributed rate limiter with Redis, sharding and failover. Analyze the consistency "
+        "tradeoffs and implement the algorithm step by step with tests."
+    )
+    SMALL: Final = {
+        "model_name": "small",
+        "litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "k"},
+        "model_info": {"max_output_tokens": 8192},
+    }
+
+    @staticmethod
+    def _router(
+        tier_litellm_params: dict | None = None,
+        max_tokens_from_tier_model: bool | None = None,
+        simple_deployments: list[dict] | None = None,
+        extra_config: dict | None = None,
+    ) -> Router:
+        simple_tier: dict = {"model_name": "small"}
+        if tier_litellm_params:
+            simple_tier["litellm_params"] = tier_litellm_params
+        config: dict = {
+            "tiers": {"SIMPLE": simple_tier, "MEDIUM": "big", "COMPLEX": "big", "REASONING": "big"},
+            **(extra_config or {}),
+        }
+        if max_tokens_from_tier_model is not None:
+            config["max_tokens_from_tier_model"] = max_tokens_from_tier_model
+        return Router(
+            model_list=[
+                {
+                    "model_name": "smart-router",
+                    "litellm_params": {"model": "auto_router/complexity_router", "complexity_router_config": config},
+                },
+                *(simple_deployments or [TestMaxTokensFromTierModel.SMALL]),
+                {
+                    "model_name": "big",
+                    "litellm_params": {"model": "anthropic/claude-sonnet-5", "api_key": "k"},
+                    "model_info": {"max_output_tokens": 64000},
+                },
+            ]
+        )
+
+    @staticmethod
+    async def _routed(router: Router, prompt: str = "hi", **request_kwargs) -> dict:
+        """Drive the real routing entry point and return the request kwargs it leaves behind."""
+        deployment = await router.async_get_available_deployment(
+            model="smart-router", request_kwargs=request_kwargs, messages=[{"role": "user", "content": prompt}]
+        )
+        return {"model": deployment["litellm_params"]["model"], **request_kwargs}
+
+    @staticmethod
+    async def _routed_responses(router: Router, prompt: str = "hi", **request_kwargs) -> dict:
+        """The Responses surface hands the router `input` both as the prompt argument and inside the
+        request kwargs, so the hook sees the same shape the real call carries."""
+        routed: dict = {"input": prompt, **request_kwargs}
+        deployment = await router.async_get_available_deployment(
+            model="smart-router", request_kwargs=routed, input=prompt
+        )
+        return {"model": deployment["litellm_params"]["model"], **routed}
+
+    @pytest.mark.asyncio
+    async def test_client_ceiling_is_replaced_by_the_routed_tier_models_ceiling(self):
+        router = self._router()
+
+        simple = await self._routed(router, max_tokens=8192)
+        complex_ = await self._routed(router, self.COMPLEX_PROMPT, max_tokens=8192)
+
+        assert (simple["model"], simple["max_tokens"]) == ("anthropic/claude-haiku-4-5", 8192)
+        assert (complex_["model"], complex_["max_tokens"]) == ("anthropic/claude-sonnet-5", 64000)
+        assert "max_output_tokens" not in complex_
+
+    @pytest.mark.asyncio
+    async def test_every_client_carrier_of_the_ceiling_is_replaced(self):
+        sent = await self._routed(self._router(), self.COMPLEX_PROMPT, max_completion_tokens=8192)
+
+        assert sent["max_tokens"] == 64000
+        assert "max_completion_tokens" not in sent
+
+    @pytest.mark.asyncio
+    async def test_responses_surface_gets_the_ceiling_under_its_own_name(self):
+        sent = await self._routed_responses(self._router(), self.COMPLEX_PROMPT, max_output_tokens=8192)
+
+        assert (sent["model"], sent["max_output_tokens"]) == ("anthropic/claude-sonnet-5", 64000)
+        assert "max_tokens" not in sent
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize(
+        "tier_params, responses_call",
+        [
+            ({"max_tokens": 4321}, False),
+            ({"max_tokens": 4321}, True),
+            ({"max_completion_tokens": 4321}, False),
+            ({"max_completion_tokens": 4321}, True),
+            ({"max_output_tokens": 4321}, False),
+        ],
+    )
+    async def test_operators_own_tier_ceiling_wins_under_the_surface_name(self, tier_params, responses_call):
+        router = self._router(tier_litellm_params=tier_params)
+        if responses_call:
+            sent = await self._routed_responses(router, max_output_tokens=8192)
+        else:
+            sent = await self._routed(router, max_tokens=8192)
+
+        surface_key = "max_output_tokens" if responses_call else "max_tokens"
+        assert sent[surface_key] == 4321
+        assert not (OUTPUT_TOKEN_CEILING_PARAMS - {surface_key}) & sent.keys()
+
+    @pytest.mark.asyncio
+    async def test_opting_out_forwards_the_client_value_unchanged(self):
+        sent = await self._routed(self._router(max_tokens_from_tier_model=False), self.COMPLEX_PROMPT, max_tokens=8192)
+
+        assert sent["max_tokens"] == 8192
+
+    @pytest.mark.asyncio
+    async def test_a_tier_model_with_an_unknown_ceiling_keeps_the_client_value(self):
+        unmapped: dict = {"model_name": "small", "litellm_params": {"model": "openai/not-in-any-map", "api_key": "k"}}
+
+        sent = await self._routed(self._router(simple_deployments=[self.SMALL, unmapped]), max_tokens=4000)
+
+        assert sent["max_tokens"] == 4000
+
+    @pytest.mark.asyncio
+    async def test_a_multi_deployment_tier_model_uses_its_smallest_ceiling(self):
+        smaller: dict = {
+            **self.SMALL,
+            "litellm_params": {**self.SMALL["litellm_params"], "api_key": "k2"},
+            "model_info": {"max_output_tokens": 4096},
+        }
+
+        sent = await self._routed(self._router(simple_deployments=[self.SMALL, smaller]), max_tokens=100000)
+
+        assert sent["max_tokens"] == 4096
+
+    @pytest.mark.asyncio
+    async def test_ceiling_falls_back_to_the_cost_map(self, monkeypatch):
+        monkeypatch.setitem(
+            litellm.model_cost,
+            "auto-cap-probe-model",
+            {"litellm_provider": "openai", "mode": "chat", "max_output_tokens": 4242, "max_input_tokens": 100000},
+        )
+        mapped_only: dict = {
+            "model_name": "small",
+            "litellm_params": {"model": "openai/auto-cap-probe-model", "api_key": "k"},
+        }
+
+        sent = await self._routed(self._router(simple_deployments=[mapped_only]), max_tokens=8192)
+
+        assert sent["max_tokens"] == 4242
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("client_kwargs", [{}, {"max_tokens": 0}], ids=["omitted", "zero"])
+    async def test_omitted_and_zero_are_replaced_like_any_other_value(self, client_kwargs):
+        sent = await self._routed(self._router(), self.COMPLEX_PROMPT, **client_kwargs)
+
+        assert sent["max_tokens"] == 64000
+
+    @pytest.mark.parametrize(
+        "tier_params, responses_call, expected",
+        [
+            ({"max_tokens": 1, "temperature": 0.2}, False, {"max_tokens": 1, "temperature": 0.2}),
+            ({"max_tokens": 1}, True, {"max_output_tokens": 1}),
+            ({"max_completion_tokens": 2}, False, {"max_tokens": 2}),
+            ({"max_completion_tokens": 2}, True, {"max_output_tokens": 2}),
+            ({"max_output_tokens": 3}, False, {"max_tokens": 3}),
+            ({"max_output_tokens": 3}, True, {"max_output_tokens": 3}),
+            ({"max_tokens": 1, "max_completion_tokens": 2, "max_output_tokens": 3}, False, {"max_tokens": 1}),
+            ({"max_tokens": 1, "max_completion_tokens": 2, "max_output_tokens": 3}, True, {"max_output_tokens": 3}),
+            ({"max_completion_tokens": 2, "max_output_tokens": 3}, False, {"max_tokens": 2}),
+            ({"reasoning_effort": "low"}, True, {"reasoning_effort": "low"}),
+        ],
+    )
+    def test_every_tier_alias_collapses_onto_the_surface_key(self, tier_params, responses_call, expected):
+        assert dict(Router._tier_ceiling_under_the_surface_name(tier_params, responses_call=responses_call)) == expected
+
+    @pytest.mark.asyncio
+    async def test_the_default_fallback_exit_carries_the_ceiling(self):
+        routed: dict = {"max_tokens": 8192}
+        deployment = await self._router().async_get_available_deployment(
+            model="smart-router", request_kwargs=routed, messages=[{"role": "system", "content": "be nice"}]
+        )
+
+        assert routed["metadata"]["routing_decision"]["cause"] == "default_fallback"
+        assert (deployment["litellm_params"]["model"], routed["max_tokens"]) == ("anthropic/claude-sonnet-5", 64000)
+
+    @pytest.mark.asyncio
+    async def test_the_plan_mode_exit_carries_the_ceiling(self):
+        routed: dict = {"max_tokens": 8192}
+        deployment = await self._router(
+            extra_config={"plan_mode_min_tier": "REASONING"}
+        ).async_get_available_deployment(
+            model="smart-router",
+            request_kwargs=routed,
+            messages=[
+                {"role": "user", "content": "plan the refactor"},
+                {"role": "system", "content": "Plan mode is active"},
+            ],
+        )
+
+        assert routed["metadata"]["routing_decision"]["cause"] == "plan_mode"
+        assert (deployment["litellm_params"]["model"], routed["max_tokens"]) == ("anthropic/claude-sonnet-5", 64000)
+
+    @pytest.mark.asyncio
+    async def test_a_default_model_landing_with_no_tier_still_gets_its_ceiling(self):
+        strategy = ComplexityRouter(
+            model_name="smart-router",
+            litellm_router_instance=self._router(),
+            complexity_router_config={"tiers": {"SIMPLE": "small"}, "default_model": "big"},
+        )
+
+        assert dict(strategy._litellm_params_for_model(None, "big")) == {"max_tokens": 64000}
+
+    @pytest.mark.asyncio
+    async def test_a_fallback_into_a_plain_group_gets_the_callers_ceiling_back(self):
+        """A model-group fallback re-enters routing with the same kwargs; a Sonnet-sized ceiling
+        must not ride onto the plain group the caller configured as the fallback."""
+        big: dict = {
+            "model_name": "big",
+            "litellm_params": {
+                "model": "anthropic/claude-sonnet-5",
+                "api_key": "k",
+                "mock_response": "litellm.InternalServerError",
+            },
+            "model_info": {"max_output_tokens": 64000},
+        }
+        plain: dict = {
+            "model_name": "plain",
+            "litellm_params": {"model": "anthropic/claude-haiku-4-5", "api_key": "k", "mock_response": "ok"},
+        }
+        router = Router(
+            model_list=[
+                {
+                    "model_name": "smart-router",
+                    "litellm_params": {
+                        "model": "auto_router/complexity_router",
+                        "complexity_router_config": {
+                            "tiers": {"SIMPLE": "big", "MEDIUM": "big", "COMPLEX": "big", "REASONING": "big"}
+                        },
+                    },
+                },
+                big,
+                plain,
+            ],
+            fallbacks=[{"smart-router": ["plain"]}],
+            num_retries=0,
+        )
+        recorder = _OutputCeilingRecorder()
+        litellm.callbacks.append(recorder)
+        try:
+            await router.acompletion(
+                model="smart-router", messages=[{"role": "user", "content": self.COMPLEX_PROMPT}], max_tokens=8192
+            )
+        finally:
+            litellm.callbacks.remove(recorder)
+
+        assert recorder.seen == [("claude-sonnet-5", 64000), ("claude-haiku-4-5", 8192)]
+
+    @pytest.mark.asyncio
+    async def test_a_caller_seeded_stamp_cannot_inject_kwargs_on_a_plain_group(self):
+        """The stamp sits in a metadata bucket a caller can write; a planted one must yield
+        nothing but integer ceiling carriers, never a redirected api_base or credential."""
+        planted: dict = {
+            "api_base": "https://attacker.example",
+            "api_key": "stolen",
+            "max_tokens": "not-an-int",
+            "max_completion_tokens": True,
+            "max_output_tokens": 321,
+        }
+        routed: dict = {"max_tokens": 8192, "metadata": {"_client_output_ceiling": planted}}
+
+        await self._router().async_get_available_deployment(
+            model="big", request_kwargs=routed, messages=[{"role": "user", "content": "hi"}]
+        )
+
+        assert {k: v for k, v in routed.items() if k not in ("metadata", "model_info")} == {"max_output_tokens": 321}
+
+    @pytest.mark.asyncio
+    async def test_the_pass_through_routing_entry_point_pins_and_restores_the_same_way(self):
+        pass_through: dict = {**self.SMALL["litellm_params"], "use_in_pass_through": True}
+        small: dict = {**self.SMALL, "litellm_params": pass_through}
+        plain: dict = {**small, "model_name": "plain"}
+        router = self._router(simple_deployments=[small, plain])
+        for deployment in router.model_list:
+            deployment["litellm_params"]["use_in_pass_through"] = True
+        routed: dict = {"max_tokens": 8192}
+
+        deployment = await router.async_get_available_deployment_for_pass_through(
+            model="smart-router", request_kwargs=routed, messages=[{"role": "user", "content": self.COMPLEX_PROMPT}]
+        )
+        pinned = routed["max_tokens"]
+        await router.async_get_available_deployment_for_pass_through(
+            model="plain", request_kwargs=routed, messages=[{"role": "user", "content": "hi"}]
+        )
+
+        assert (deployment["litellm_params"]["model"], pinned, routed["max_tokens"]) == (
+            "anthropic/claude-sonnet-5",
+            64000,
+            8192,
+        )
+
+    @pytest.mark.asyncio
+    async def test_the_classifier_fallback_exit_carries_the_ceiling(self):
+        router = self._router(
+            extra_config={
+                "classifier_type": "llm",
+                "classifier_llm_config": {"model": "no-such-classifier", "timeout_ms": 400},
+                "classifier_fallback": "default_model",
+                "default_model": "big",
+            }
+        )
+        routed: dict = {"max_tokens": 8192}
+
+        deployment = await router.async_get_available_deployment(
+            model="smart-router", request_kwargs=routed, messages=[{"role": "user", "content": "hi"}]
+        )
+
+        assert routed["metadata"]["routing_decision"]["cause"] == "default_model_fallback"
+        assert (deployment["litellm_params"]["model"], routed["max_tokens"]) == ("anthropic/claude-sonnet-5", 64000)
+
+    @pytest.mark.parametrize(
+        "value, expected",
+        [(8192, 8192), ("8192", 8192), (100.9, 100), (0, 0), (-1, None), (True, None), ("x", None), (None, None)],
+    )
+    def test_a_client_cap_is_read_as_an_integer_or_ignored(self, value, expected):
+        assert as_output_cap(value) == expected
+
+    def test_restoring_the_callers_ceiling_reads_the_stamp_and_replaces_every_carrier(self):
+        stamped: dict = {"max_output_tokens": 500, "metadata": {"_client_output_ceiling": {"max_tokens": 8192}}}
+        Router._restore_client_ceiling_no_tier_pins(stamped)
+        assert {k: v for k, v in stamped.items() if k != "metadata"} == {"max_tokens": 8192}
+
+        coerced: dict = {
+            "max_tokens": 64000,
+            "metadata": {"_client_output_ceiling": {"max_tokens": "8192", "max_completion_tokens": 100.0}},
+        }
+        Router._restore_client_ceiling_no_tier_pins(coerced)
+        assert {k: v for k, v in coerced.items() if k != "metadata"} == {
+            "max_tokens": 8192,
+            "max_completion_tokens": 100,
+        }
+
+        unstamped: dict = {"max_tokens": 64000, "metadata": {}}
+        Router._restore_client_ceiling_no_tier_pins(unstamped)
+        assert unstamped["max_tokens"] == 64000
+
+    @pytest.mark.asyncio
+    async def test_pinning_stamps_the_callers_carriers_once(self):
+        router = self._router()
+        request_kwargs: dict = {"max_completion_tokens": 8192}
+
+        first = router._pin_tier_params_onto_request(
+            model="big", tier_litellm_params={"max_tokens": 64000}, request_kwargs=request_kwargs, responses_call=False
+        )
+        second = router._pin_tier_params_onto_request(
+            model="big", tier_litellm_params={"max_tokens": 32000}, request_kwargs=request_kwargs, responses_call=False
+        )
+        none = router._pin_tier_params_onto_request(
+            model="big", tier_litellm_params=None, request_kwargs=request_kwargs, responses_call=False
+        )
+
+        assert (first, second, none) == (True, True, False)
+        assert request_kwargs["max_tokens"] == 32000
+        assert request_kwargs["metadata"]["_client_output_ceiling"] == {"max_completion_tokens": 8192}
+
+
+class _OutputCeilingRecorder(CustomLogger):
+    def __init__(self) -> None:
+        super().__init__()
+        self.seen: list[tuple[str, int | None]] = []
+
+    def log_pre_api_call(self, model, messages, kwargs):
+        self.seen.append((model, kwargs.get("optional_params", {}).get("max_tokens")))
diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts
index d1431d8956f..b52f0922390 100644
--- a/ui/litellm-dashboard/src/lib/http/schema.d.ts
+++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts
@@ -34996,6 +34996,12 @@ export interface components {
              * @default 0.5
              */
             match_threshold: number;
+            /**
+             * Max Tokens From Tier Model
+             * @description Set max_tokens on every routed request to the output ceiling of the tier model it lands on, replacing whatever the caller sent. A caller behind an auto-router cannot pick one value that fits every tier: the smallest tier's ceiling starves a bigger tier's thinking budget, and a bigger tier's ceiling is rejected by the smallest. The ceiling is the smallest max_output_tokens across the tier model's deployments, read from each deployment's model_info and then the model cost map; a tier model with a deployment whose ceiling is unknown keeps the caller's value. A max_tokens, max_completion_tokens or max_output_tokens in the tier's own litellm_params still wins. Set false to forward the caller's value unchanged.
+             * @default true
+             */
+            max_tokens_from_tier_model: boolean;
             /**
              * Modality Pin Override
              * @description Let modality_routing replace a kept session-affinity pin on the turns that carry an image. Without this, a session pinned to a text-only model fails every image turn with a provider 400, since the pin is exempt from the modality gate. When enabled, such a turn routes to a capable model for that request only and the stored pin is left untouched, so the next text turn replays the session's own model; the override is reported as cause modality_pin_override and is never itself pinned. Inert unless modality_routing is also enabled.

From 29fe1e895f8392712d11a296bd51ebbcce826e8d Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 8 Sep 2026 13:53:25 -0700
Subject: [PATCH 72/85] feat(otel): make the OTel v2 trace export URL
 configurable (#40286)

Add traces_endpoint (env OTEL_TRACES_ENDPOINT / OTEL_EXPORTER_OTLP_TRACES_ENDPOINT, per-exporter key traces_endpoint, Admin UI field) as a complete OTLP/HTTP trace URL used verbatim, so collectors that do not serve /v1/traces can receive traces. endpoint keeps its existing base-URL + signal-path normalization.

Resolves LIT-7218

Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/integrations/callback_configs.json    |  6 ++
 litellm/integrations/otel/model/config.py     | 19 ++++-
 .../integrations/otel/plumbing/providers.py   | 11 ++-
 litellm/proxy/_types.py                       |  1 +
 .../otel/test_otel_v2_components.py           | 74 +++++++++++++++++++
 5 files changed, 108 insertions(+), 3 deletions(-)

diff --git a/litellm/integrations/callback_configs.json b/litellm/integrations/callback_configs.json
index 7a2295a35ae..b98eef8329b 100644
--- a/litellm/integrations/callback_configs.json
+++ b/litellm/integrations/callback_configs.json
@@ -356,6 +356,12 @@
         "description": "OpenTelemetry collector endpoint URL",
         "required": true
       },
+      "otel_traces_endpoint": {
+        "type": "text",
+        "ui_name": "Traces Endpoint URL",
+        "description": "Complete trace export URL used verbatim when the collector does not serve /v1/traces (OTel v2 only)",
+        "required": false
+      },
       "otel_headers": {
         "type": "text",
         "ui_name": "Headers",
diff --git a/litellm/integrations/otel/model/config.py b/litellm/integrations/otel/model/config.py
index 9e3064c2bff..e3501d0ee94 100644
--- a/litellm/integrations/otel/model/config.py
+++ b/litellm/integrations/otel/model/config.py
@@ -72,6 +72,14 @@ class ExporterSpec(BaseModel):
         description="console | in_memory | otlp_http | otlp_grpc | <factory kind>",
     )
     endpoint: str | None = None
+    traces_endpoint: str | None = Field(
+        default=None,
+        description=(
+            "Complete OTLP/HTTP trace URL, used verbatim. Set this when the "
+            "collector serves traces on a path other than ``/v1/traces``; "
+            "``endpoint`` is a base URL the signal path is appended to."
+        ),
+    )
     headers: str | None = None
     owner: ExporterOwner | None = Field(
         default=None,
@@ -127,6 +135,14 @@ class OpenTelemetryV2Config(BaseSettings):
         default=None,
         validation_alias=AliasChoices("OTEL_ENDPOINT", "OTEL_EXPORTER_OTLP_ENDPOINT"),
     )
+    traces_endpoint: str | None = Field(
+        default=None,
+        validation_alias=AliasChoices("OTEL_TRACES_ENDPOINT", "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"),
+        description=(
+            "Complete OTLP/HTTP trace URL for the single-destination shorthand, "
+            "used verbatim instead of ``endpoint`` + ``/v1/traces``."
+        ),
+    )
     headers: str | None = Field(
         default=None,
         validation_alias=AliasChoices("OTEL_HEADERS", "OTEL_EXPORTER_OTLP_HEADERS"),
@@ -250,7 +266,7 @@ class OpenTelemetryV2Config(BaseSettings):
     @model_validator(mode="after")
     def _normalize(self) -> "OpenTelemetryV2Config":
         # An endpoint with the default exporter kind implies OTLP/HTTP.
-        if self.endpoint and self.exporter == "console":
+        if (self.endpoint or self.traces_endpoint) and self.exporter == "console":
             self.exporter = "otlp_http"
         # When no explicit destinations are given, fold the single-destination
         # shorthand into one spec so the provider always has a destination.
@@ -259,6 +275,7 @@ class OpenTelemetryV2Config(BaseSettings):
                 ExporterSpec(
                     kind=self.exporter,
                     endpoint=self.endpoint,
+                    traces_endpoint=self.traces_endpoint,
                     headers=self.headers,
                 )
             ]
diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py
index fb74ff85e5b..d8da53017ab 100644
--- a/litellm/integrations/otel/plumbing/providers.py
+++ b/litellm/integrations/otel/plumbing/providers.py
@@ -170,7 +170,7 @@ def _exporter_from_spec(spec: ExporterSpec) -> SpanExporter:
         )
 
         return HTTPExporter(
-            endpoint=_otlp_traces_endpoint(spec.endpoint),
+            endpoint=spec.traces_endpoint or _otlp_traces_endpoint(spec.endpoint),
             headers=parse_headers(spec.headers),
         )
     if kind in _OTLP_GRPC_KINDS:
@@ -201,7 +201,14 @@ def build_span_exporter(config: OpenTelemetryV2Config) -> SpanExporter:
     ``exporter`` / ``endpoint`` / ``headers`` fields. To configure multiple
     exporters, populate ``config.exporters`` directly.
     """
-    return _exporter_from_spec(ExporterSpec(kind=config.exporter, endpoint=config.endpoint, headers=config.headers))
+    return _exporter_from_spec(
+        ExporterSpec(
+            kind=config.exporter,
+            endpoint=config.endpoint,
+            traces_endpoint=config.traces_endpoint,
+            headers=config.headers,
+        )
+    )
 
 
 def _otlp_metrics_endpoint(endpoint: str | None) -> str | None:
diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index 4dbae6394f6..d1b391d57a3 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -3585,6 +3585,7 @@ class AllCallbacks(LiteLLMPydanticObjectBase):
         litellm_callback_params=[
             "OTEL_EXPORTER",
             "OTEL_ENDPOINT",
+            "OTEL_TRACES_ENDPOINT",
             "OTEL_HEADERS",
         ],
     )
diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_components.py b/tests/test_litellm/integrations/otel/test_otel_v2_components.py
index 4aa28b5abfd..6676714d7f5 100644
--- a/tests/test_litellm/integrations/otel/test_otel_v2_components.py
+++ b/tests/test_litellm/integrations/otel/test_otel_v2_components.py
@@ -3,7 +3,10 @@ baggage helpers, metrics, the typed coercion helpers, mapper branches, span-name
 builders, and the registry validator's failure paths. Needs the OTel SDK."""
 
 import json
+import threading
+from collections.abc import Iterator
 from dataclasses import replace
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 
 import pytest
 
@@ -538,6 +541,77 @@ def test_build_span_exporter_variants():
     assert "OTLPSpanExporter" in type(http_exporter).__name__
 
 
+@pytest.fixture
+def otlp_collector() -> Iterator[tuple[str, list[str]]]:
+    received_paths: list[str] = []
+
+    class RecordingHandler(BaseHTTPRequestHandler):
+        def do_POST(self) -> None:
+            self.rfile.read(int(self.headers.get("Content-Length", "0")))
+            received_paths.append(self.path)
+            self.send_response(200)
+            self.end_headers()
+
+        def log_message(self, format: str, *args: object) -> None:
+            pass
+
+    server = ThreadingHTTPServer(("127.0.0.1", 0), RecordingHandler)
+    threading.Thread(target=server.serve_forever, daemon=True).start()
+    try:
+        yield f"http://127.0.0.1:{server.server_port}", received_paths
+    finally:
+        server.shutdown()
+        server.server_close()
+
+
+def _export_one_span(cfg: OpenTelemetryV2Config) -> None:
+    provider = providers.build_tracer_provider(cfg)
+    provider.get_tracer("probe").start_span("probe").end()
+    assert provider.force_flush()
+    provider.shutdown()
+
+
+def test_traces_endpoint_env_posts_to_the_configured_url_verbatim(monkeypatch, otlp_collector):
+    base_url, received_paths = otlp_collector
+    for var in ("OTEL_EXPORTER", "OTEL_EXPORTER_OTLP_PROTOCOL", "OTEL_EXPORTER_OTLP_ENDPOINT"):
+        monkeypatch.delenv(var, raising=False)
+    monkeypatch.setenv("OTEL_ENDPOINT", f"{base_url}/services/collector")
+    monkeypatch.setenv("OTEL_TRACES_ENDPOINT", f"{base_url}/services/collector/traces")
+
+    cfg = OpenTelemetryV2Config.from_env()
+    assert cfg.exporter == "otlp_http"
+    _export_one_span(cfg)
+    assert received_paths == ["/services/collector/traces"]
+
+
+def test_traces_endpoint_alias_alone_implies_otlp_http(monkeypatch, otlp_collector):
+    base_url, received_paths = otlp_collector
+    for var in ("OTEL_EXPORTER", "OTEL_EXPORTER_OTLP_PROTOCOL", "OTEL_ENDPOINT", "OTEL_EXPORTER_OTLP_ENDPOINT"):
+        monkeypatch.delenv(var, raising=False)
+    monkeypatch.setenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", f"{base_url}/custom/traces")
+
+    cfg = OpenTelemetryV2Config.from_env()
+    assert cfg.exporter == "otlp_http"
+    _export_one_span(cfg)
+    assert received_paths == ["/custom/traces"]
+
+
+def test_traces_endpoint_per_exporter_coexists_with_default_normalization(otlp_collector):
+    base_url, received_paths = otlp_collector
+    cfg = OpenTelemetryV2Config(
+        exporters=[
+            {"kind": "otlp_http", "endpoint": base_url},
+            {
+                "kind": "otlp_http",
+                "endpoint": f"{base_url}/services/collector",
+                "traces_endpoint": f"{base_url}/services/collector/traces",
+            },
+        ]
+    )
+    _export_one_span(cfg)
+    assert sorted(received_paths) == ["/services/collector/traces", "/v1/traces"]
+
+
 def test_otlp_metric_exporter_uses_cumulative_histogram_temporality():
     """Histograms must export as cumulative, not delta.
 

From b0071f363f8dd55a3a252e61f51f088ac5bab73c Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 8 Sep 2026 13:54:05 -0700
Subject: [PATCH 73/85] fix(router): keep per-request routing_strategy override
 selectors out of global callbacks (#40229)

* fix(router): keep per-request routing_strategy override selectors out of global callbacks

Override selectors built by _get_override_strategy_selector were registered in litellm.callbacks / litellm.input_callback and cached for the router's lifetime, so one request with router_settings_override made every later request on the worker run that selector's pre-call check (rpm enforcement for usage-based-routing-v2). Build them with register_callbacks=False and run the selector's own pre_call_check / async_pre_call_check only for the request that carried the override.

Resolves LIT-7064

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(router): run override selector pre-call check on specific-deployment early returns

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(router): cover pass-through and affinity-pinned override pre-call paths

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/router.py                             |  51 +++++-
 .../test_router_routing_groups.py             | 157 +++++++++++++++++-
 2 files changed, 203 insertions(+), 5 deletions(-)

diff --git a/litellm/router.py b/litellm/router.py
index cf62ce1eb2d..a4fa4a16d12 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -1546,9 +1546,33 @@ class Router:
                 self._override_selectors[strategy] = self._build_strategy_selector(
                     strategy=strategy,
                     routing_strategy_args={},
+                    register_callbacks=False,
                 )
             return self._override_selectors[strategy]
 
+    def _override_selector_pre_call_check(
+        self, strategy: str | None, selector: RouterStrategySelector | None, deployment: dict
+    ) -> None:
+        """
+        Override selectors are not in `litellm.callbacks`, so the pre-call check that
+        `routing_strategy_pre_call_checks` runs for the router's own selectors (rpm
+        accounting for `usage-based-routing-v2`) runs here, for the overriding request only.
+        """
+        if selector is None or strategy is None or selector is not self._override_selectors.get(strategy):
+            return
+        selector.pre_call_check(deployment)
+
+    async def _async_override_selector_pre_call_check(
+        self,
+        strategy: str | None,
+        selector: RouterStrategySelector | None,
+        deployment: dict,
+        parent_otel_span: Span | None,
+    ) -> None:
+        if selector is None or strategy is None or selector is not self._override_selectors.get(strategy):
+            return
+        await selector.async_pre_call_check(deployment, parent_otel_span)
+
     def _get_routing_context(
         self, model: str, request_kwargs: dict | None = None
     ) -> tuple[str | None, RouterStrategySelector | None]:
@@ -12807,10 +12831,16 @@ class Router:
                 parent_otel_span=parent_otel_span,
             )
             if isinstance(healthy_deployments, dict):
+                await self._async_override_selector_pre_call_check(
+                    strategy, strategy_selector, healthy_deployments, parent_otel_span
+                )
                 return healthy_deployments
 
             # When encrypted content affinity pins to a specific deployment,
             if request_kwargs.get("_encrypted_content_affinity_pinned") and len(healthy_deployments) == 1:
+                await self._async_override_selector_pre_call_check(
+                    strategy, strategy_selector, healthy_deployments[0], parent_otel_span
+                )
                 return healthy_deployments[0]
 
             start_time: Final = time.time()
@@ -12836,6 +12866,9 @@ class Router:
                     parent_otel_span=parent_otel_span,
                 )
                 raise exception
+            await self._async_override_selector_pre_call_check(
+                strategy, strategy_selector, deployment, parent_otel_span
+            )
             verbose_router_logger.info(
                 "get_available_deployment for model: %s, Selected deployment: %s for model: %s",
                 model,
@@ -12921,6 +12954,8 @@ class Router:
                 parent_otel_span=parent_otel_span,
             )
 
+            strategy, strategy_selector = self._get_routing_context(model, request_kwargs)
+
             # 3. If specific deployment returned, verify if it supports pass-through
             if isinstance(healthy_deployments, dict):
                 if (healthy_deployments.get("model_info") or {}).get("blocked") is True:
@@ -12931,6 +12966,9 @@ class Router:
                     )
                 litellm_params: Final = healthy_deployments.get("litellm_params", {})
                 if litellm_params.get("use_in_pass_through"):
+                    await self._async_override_selector_pre_call_check(
+                        strategy, strategy_selector, healthy_deployments, parent_otel_span
+                    )
                     return healthy_deployments
                 else:
                     raise litellm.BadRequestError(
@@ -12951,7 +12989,6 @@ class Router:
 
             # 5. Apply load balancing strategy
             start_time: Final = time.perf_counter()
-            strategy, strategy_selector = self._get_routing_context(model, request_kwargs)
             if strategy == "simple-shuffle":
                 return simple_shuffle(
                     llm_router_instance=self,
@@ -12975,6 +13012,9 @@ class Router:
                     parent_otel_span=parent_otel_span,
                 )
                 raise exception
+            await self._async_override_selector_pre_call_check(
+                strategy, strategy_selector, deployment, parent_otel_span
+            )
 
             verbose_router_logger.info(
                 "async_get_available_deployment_for_pass_through model: %s, selected deployment: %s",
@@ -13550,6 +13590,7 @@ class Router:
             specific_deployment=specific_deployment,
             request_kwargs=request_kwargs,
         )
+        strategy, strategy_selector = self._get_routing_context(model, request_kwargs)
 
         if isinstance(healthy_deployments, dict):
             if (healthy_deployments.get("model_info") or {}).get("blocked") is True:
@@ -13558,6 +13599,7 @@ class Router:
                     model=model,
                     llm_provider="",
                 )
+            self._override_selector_pre_call_check(strategy, strategy_selector, healthy_deployments)
             return healthy_deployments
 
         parent_otel_span: Final[Span | None] = _get_parent_otel_span_from_kwargs(request_kwargs)
@@ -13633,7 +13675,6 @@ class Router:
                 cooldown_list=_cooldown_list,
             )
 
-        strategy, strategy_selector = self._get_routing_context(model, request_kwargs)
         if strategy == "simple-shuffle":
             # if users pass rpm or tpm, we do a random weighted pick - based on rpm/tpm
             ############## Check 'weight' param set for weighted pick #################
@@ -13665,6 +13706,7 @@ class Router:
                 enable_pre_call_checks=self.enable_pre_call_checks,
                 cooldown_list=_cooldown_list,
             )
+        self._override_selector_pre_call_check(strategy, strategy_selector, deployment)
         verbose_router_logger.info(
             "get_available_deployment for model: %s, Selected deployment: %s for model: %s",
             model,
@@ -13708,6 +13750,8 @@ class Router:
             specific_deployment=specific_deployment,
         )
 
+        strategy, strategy_selector = self._get_routing_context(model, request_kwargs)
+
         # 2. If the returned is a specific deployment (Dict), verify and return directly
         if isinstance(healthy_deployments, dict):
             if (healthy_deployments.get("model_info") or {}).get("blocked") is True:
@@ -13718,6 +13762,7 @@ class Router:
                 )
             litellm_params: Final = healthy_deployments.get("litellm_params", {})
             if litellm_params.get("use_in_pass_through"):
+                self._override_selector_pre_call_check(strategy, strategy_selector, healthy_deployments)
                 return healthy_deployments
             else:
                 # Specific deployment does not support pass-through
@@ -13777,7 +13822,6 @@ class Router:
             )
 
         # 6. Apply load balancing strategy
-        strategy, strategy_selector = self._get_routing_context(model, request_kwargs)
         if strategy == "simple-shuffle":
             return simple_shuffle(
                 llm_router_instance=self,
@@ -13809,6 +13853,7 @@ class Router:
                 enable_pre_call_checks=self.enable_pre_call_checks,
                 cooldown_list=_cooldown_list,
             )
+        self._override_selector_pre_call_check(strategy, strategy_selector, deployment)
 
         verbose_router_logger.info(
             "get_available_deployment_for_pass_through model: %s, selected deployment: %s",
diff --git a/tests/test_litellm/router_strategy/test_router_routing_groups.py b/tests/test_litellm/router_strategy/test_router_routing_groups.py
index af390c3292b..5f37842305d 100644
--- a/tests/test_litellm/router_strategy/test_router_routing_groups.py
+++ b/tests/test_litellm/router_strategy/test_router_routing_groups.py
@@ -792,15 +792,168 @@ def test_strategy_reinit_unregisters_override_selectors():
     router = _build_router(routing_strategy="least-busy")
     override_selector = router._get_override_strategy_selector("latency-based-routing")
     assert override_selector is not None
-    assert any(id(cb) == id(override_selector) for cb in litellm.callbacks)
+    assert not any(cb is override_selector for cb in litellm.callbacks)
 
     router.update_settings(routing_strategy="latency-based-routing")
 
     assert router._override_selectors == {}
-    assert not any(id(cb) == id(override_selector) for cb in litellm.callbacks)
+    assert not any(cb is override_selector for cb in litellm.callbacks)
     assert router._get_override_strategy_selector("latency-based-routing") is router.lowestlatency_logger
 
 
+def test_override_selectors_are_not_registered_process_wide(monkeypatch):
+    monkeypatch.setattr(litellm, "callbacks", [])
+    monkeypatch.setattr(litellm, "input_callback", [])
+    router = _build_router(routing_strategy="simple-shuffle")
+    v2_selector = router._get_override_strategy_selector("usage-based-routing-v2")
+    least_busy_selector = router._get_override_strategy_selector("least-busy")
+    assert v2_selector is not None and least_busy_selector is not None
+
+    assert litellm.callbacks == []
+    assert litellm.input_callback == []
+
+
+def _rpm_limited_model_list():
+    return [
+        {
+            "model_name": "other-model",
+            "litellm_params": {
+                "model": "openai/gpt-4o",
+                "api_key": "sk-test-3",
+                "api_base": "https://example.invalid",
+                "rpm": 1,
+            },
+            "model_info": {"id": "deploy-3"},
+        },
+    ]
+
+
+async def _mock_completion(router, **override):
+    return await router.acompletion(
+        model="other-model", messages=[{"role": "user", "content": "hi"}], mock_response="ok", **override
+    )
+
+
+@pytest.mark.asyncio
+async def test_usage_based_v2_override_stays_scoped_to_the_request_that_asked_for_it():
+    router = Router(model_list=_rpm_limited_model_list(), routing_strategy="simple-shuffle", num_retries=0)
+
+    await _mock_completion(router, routing_strategy="usage-based-routing-v2")
+    override_selector = router._override_selectors["usage-based-routing-v2"]
+    assert not any(cb is override_selector for cb in litellm.callbacks)
+
+    with patch.object(
+        override_selector, "async_pre_call_check", wraps=override_selector.async_pre_call_check
+    ) as pre_call_spy:
+        for _ in range(2):
+            plain = await _mock_completion(router)
+            assert plain.choices[0].message.content == "ok"
+        assert not pre_call_spy.called
+
+        with pytest.raises(litellm.RateLimitError):
+            await _mock_completion(router, routing_strategy="usage-based-routing-v2")
+
+
+def test_sync_usage_based_v2_override_stays_scoped_to_the_request_that_asked_for_it():
+    router = Router(model_list=_rpm_limited_model_list(), routing_strategy="simple-shuffle", num_retries=0)
+    messages = [{"role": "user", "content": "hi"}]
+
+    router.completion(
+        model="other-model", messages=messages, mock_response="ok", routing_strategy="usage-based-routing-v2"
+    )
+    override_selector = router._override_selectors["usage-based-routing-v2"]
+    assert not any(cb is override_selector for cb in litellm.callbacks)
+
+    for _ in range(2):
+        plain = router.completion(model="other-model", messages=messages, mock_response="ok")
+        assert plain.choices[0].message.content == "ok"
+
+    with pytest.raises(ValueError, match="No deployments available"):
+        router.completion(
+            model="other-model", messages=messages, mock_response="ok", routing_strategy="usage-based-routing-v2"
+        )
+
+
+@pytest.mark.asyncio
+async def test_override_selector_pre_call_check_only_runs_for_override_selectors(monkeypatch):
+    monkeypatch.setattr(litellm, "callbacks", [])
+    deployment = _rpm_limited_model_list()[0]
+
+    override_router = Router(model_list=_rpm_limited_model_list(), routing_strategy="simple-shuffle")
+    override_selector = override_router._get_override_strategy_selector("usage-based-routing-v2")
+    await override_router._async_override_selector_pre_call_check(
+        "usage-based-routing-v2", override_selector, deployment, None
+    )
+    with pytest.raises(litellm.RateLimitError):
+        override_router._override_selector_pre_call_check("usage-based-routing-v2", override_selector, deployment)
+
+    default_router = Router(model_list=_rpm_limited_model_list(), routing_strategy="usage-based-routing-v2")
+    for _ in range(2):
+        await default_router._async_override_selector_pre_call_check(
+            "usage-based-routing-v2", default_router.lowesttpm_logger_v2, deployment, None
+        )
+        default_router._override_selector_pre_call_check(
+            "usage-based-routing-v2", default_router.lowesttpm_logger_v2, deployment
+        )
+    await default_router._async_override_selector_pre_call_check(None, None, deployment, None)
+    default_router._override_selector_pre_call_check(None, None, deployment)
+
+
+@pytest.mark.asyncio
+async def test_usage_based_v2_override_enforces_rpm_when_a_specific_deployment_is_requested():
+    router = Router(model_list=_rpm_limited_model_list(), routing_strategy="simple-shuffle", num_retries=0)
+    kwargs = {"model": "deploy-3", "messages": [{"role": "user", "content": "hi"}], "mock_response": "ok"}
+
+    first = await router.acompletion(**kwargs, routing_strategy="usage-based-routing-v2")
+    assert first.choices[0].message.content == "ok"
+    with pytest.raises(litellm.RateLimitError):
+        await router.acompletion(**kwargs, routing_strategy="usage-based-routing-v2")
+    assert (await router.acompletion(**kwargs)).choices[0].message.content == "ok"
+
+
+def test_sync_usage_based_v2_override_enforces_rpm_when_a_specific_deployment_is_requested():
+    router = Router(model_list=_rpm_limited_model_list(), routing_strategy="simple-shuffle", num_retries=0)
+    kwargs = {"model": "deploy-3", "messages": [{"role": "user", "content": "hi"}], "mock_response": "ok"}
+
+    first = router.completion(**kwargs, routing_strategy="usage-based-routing-v2")
+    assert first.choices[0].message.content == "ok"
+    with pytest.raises(litellm.RateLimitError):
+        router.completion(**kwargs, routing_strategy="usage-based-routing-v2")
+    assert router.completion(**kwargs).choices[0].message.content == "ok"
+
+
+def _pass_through_rpm_limited_model_list():
+    deployment = _rpm_limited_model_list()[0]
+    return [{**deployment, "litellm_params": {**deployment["litellm_params"], "use_in_pass_through": True}}]
+
+
+@pytest.mark.asyncio
+async def test_async_early_return_paths_run_the_override_pre_call_check():
+    router = Router(model_list=_pass_through_rpm_limited_model_list(), routing_strategy="simple-shuffle")
+    override = {"routing_strategy": "usage-based-routing-v2"}
+
+    pinned = await router.async_get_available_deployment(
+        model="other-model", request_kwargs={**override, "_encrypted_content_affinity_pinned": True}
+    )
+    assert pinned["model_info"]["id"] == "deploy-3"
+    with pytest.raises(litellm.RateLimitError):
+        await router.async_get_available_deployment_for_pass_through(model="deploy-3", request_kwargs=override)
+    plain = await router.async_get_available_deployment_for_pass_through(model="deploy-3", request_kwargs={})
+    assert plain["model_info"]["id"] == "deploy-3"
+
+
+def test_sync_pass_through_specific_deployment_runs_the_override_pre_call_check():
+    router = Router(model_list=_pass_through_rpm_limited_model_list(), routing_strategy="simple-shuffle")
+    override = {"routing_strategy": "usage-based-routing-v2"}
+
+    first = router.get_available_deployment_for_pass_through(model="deploy-3", request_kwargs=override)
+    assert first["model_info"]["id"] == "deploy-3"
+    with pytest.raises(litellm.RateLimitError):
+        router.get_available_deployment_for_pass_through(model="deploy-3", request_kwargs=override)
+    plain = router.get_available_deployment_for_pass_through(model="deploy-3", request_kwargs={})
+    assert plain["model_info"]["id"] == "deploy-3"
+
+
 def _quality_group(strategy="latency-based-routing"):
     return [{"group_name": "quality", "models": ["filtered-model", "other-model"], "routing_strategy": strategy}]
 

From 935c7190ebe9f344e16079c8cd0dd9849db312aa Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 8 Sep 2026 13:54:48 -0700
Subject: [PATCH 74/85] fix(router): count allowed_fails in the shared router
 cache so multi-worker proxies bench a deployment fleet-wide (#40224)

Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/router.py                             |  1 -
 litellm/router_utils/cooldown_handlers.py     | 33 ++++++---
 .../test_router_cooldown_per_deployment.py    |  8 +--
 .../router_utils/test_cooldown_handlers.py    | 70 +++++++++++++++++--
 ..._health_check_allowed_fails_integration.py |  4 +-
 5 files changed, 94 insertions(+), 22 deletions(-)

diff --git a/litellm/router.py b/litellm/router.py
index a4fa4a16d12..9b2badeaa69 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -978,7 +978,6 @@ class Router:
             DEFAULT_HEALTH_CHECK_INTERVAL * DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER
         )
         self.health_state_cache = DeploymentHealthCache(cache=self.cache, staleness_threshold=float(_staleness))
-        self.failed_calls = InMemoryCache()  # cache to track failed call per deployment, if num failed calls within 1 minute > allowed fails, then add it to cooldown
 
         if num_retries is not None:
             self.num_retries = num_retries
diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py
index 4534fa114b3..f722b6fd20c 100644
--- a/litellm/router_utils/cooldown_handlers.py
+++ b/litellm/router_utils/cooldown_handlers.py
@@ -14,6 +14,7 @@ from typing import TYPE_CHECKING, Any, Final
 
 import litellm
 from litellm._logging import verbose_router_logger
+from litellm.caching.dual_cache import DualCache
 from litellm.constants import (
     DEFAULT_COOLDOWN_TIME_SECONDS,
     DEFAULT_FAILURE_THRESHOLD_MINIMUM_REQUESTS,
@@ -558,9 +559,12 @@ def should_cooldown_based_on_allowed_fails_policy(
     When *allowed_fails_override* / *cooldown_time_override* are supplied they
     take precedence over the router-level values (used by deployment-level overrides).
 
+    The counter lives in the router's shared ``DualCache`` (Redis when configured), so
+    every worker process increments the same key and the threshold applies fleet-wide.
+
     When *cache_key_suffix* is supplied the fail counter is keyed as
-    ``{deployment}:{cache_key_suffix}`` so that different exception types are
-    tracked independently per deployment.
+    ``deployment:{deployment}:allowed_fails:{cache_key_suffix}`` so that different
+    exception types are tracked independently per deployment.
 
     Returns:
     - True if fails exceed the allowed limit (should cooldown)
@@ -584,16 +588,25 @@ def should_cooldown_based_on_allowed_fails_policy(
         else (litellm_router_instance.cooldown_time or DEFAULT_COOLDOWN_TIME_SECONDS)
     )
 
-    cache_key: Final = f"{deployment}:{cache_key_suffix}" if cache_key_suffix else deployment
-    current_fails: Final = litellm_router_instance.failed_calls.get_cache(key=cache_key) or 0
-    updated_fails: Final = current_fails + 1
+    base_key: Final = f"deployment:{deployment}:allowed_fails"
+    cache_key: Final = f"{base_key}:{cache_key_suffix}" if cache_key_suffix else base_key
+    updated_fails: Final = _increment_allowed_fails(
+        cache=litellm_router_instance.cache, cache_key=cache_key, ttl=cooldown_time
+    )
+    return updated_fails > allowed_fails
 
-    if updated_fails > allowed_fails:
-        return True
-    else:
-        litellm_router_instance.failed_calls.set_cache(key=cache_key, value=updated_fails, ttl=cooldown_time)
 
-    return False
+def _increment_allowed_fails(cache: DualCache, cache_key: str, ttl: float) -> int:
+    """
+    Return the fleet-wide fail count. ``DualCache.increment_cache`` bumps the in-memory tier
+    before Redis and re-raises a Redis error, so a Redis outage degrades to this worker's own count.
+    """
+    try:
+        return cache.increment_cache(key=cache_key, value=1, ttl=ttl)
+    except Exception as e:  # noqa: BLE001  # a Redis outage must not stop failing deployments from cooling down
+        verbose_router_logger.warning("allowed_fails counter fell back to this worker's in-memory count: %s", e)
+        local_fails: Final = cache.get_cache(key=cache_key, local_only=True)
+        return local_fails if isinstance(local_fails, int) else 0
 
 
 def _is_allowed_fails_set_on_router(
diff --git a/tests/router_unit_tests/test_router_cooldown_per_deployment.py b/tests/router_unit_tests/test_router_cooldown_per_deployment.py
index 228dfed38b9..964782c348b 100644
--- a/tests/router_unit_tests/test_router_cooldown_per_deployment.py
+++ b/tests/router_unit_tests/test_router_cooldown_per_deployment.py
@@ -204,8 +204,8 @@ class TestExceptionTypeCountersTrackedIndependently:
                 cache_key_suffix="RateLimitError",
             )
 
-        rl_counter = router.failed_calls.get_cache(key="primary:RateLimitError") or 0
-        generic_counter = router.failed_calls.get_cache(key="primary:generic") or 0
+        rl_counter = router.cache.get_cache(key="deployment:primary:allowed_fails:RateLimitError") or 0
+        generic_counter = router.cache.get_cache(key="deployment:primary:allowed_fails:generic") or 0
 
         assert rl_counter == 3, "RateLimitError counter should be 3"
         assert generic_counter == 0, "generic counter must be untouched by RateLimitError increments"
@@ -218,8 +218,8 @@ class TestExceptionTypeCountersTrackedIndependently:
             cache_key_suffix="generic",
         )
 
-        generic_counter_after = router.failed_calls.get_cache(key="primary:generic") or 0
-        rl_counter_after = router.failed_calls.get_cache(key="primary:RateLimitError") or 0
+        generic_counter_after = router.cache.get_cache(key="deployment:primary:allowed_fails:generic") or 0
+        rl_counter_after = router.cache.get_cache(key="deployment:primary:allowed_fails:RateLimitError") or 0
 
         assert generic_counter_after == 1, "generic counter should now be 1"
         assert rl_counter_after == 3, "RateLimitError counter must remain unchanged after InternalServerError"
diff --git a/tests/test_litellm/router_utils/test_cooldown_handlers.py b/tests/test_litellm/router_utils/test_cooldown_handlers.py
index 4768988fc87..7ee0ed3701b 100644
--- a/tests/test_litellm/router_utils/test_cooldown_handlers.py
+++ b/tests/test_litellm/router_utils/test_cooldown_handlers.py
@@ -1,6 +1,8 @@
 from unittest.mock import MagicMock, patch
 
 import litellm
+from litellm.caching.dual_cache import DualCache
+from litellm.caching.in_memory_cache import InMemoryCache
 from litellm.router_utils.cooldown_handlers import (
     _get_deployment_cooldown_policy,
     _resolve_allowed_fails_from_policy,
@@ -269,18 +271,20 @@ class TestShouldCooldownBasedOnDeploymentPolicy:
 
 
 class TestShouldCooldownBasedOnAllowedFailsPolicy:
-    def _make_router(self, cooldown_time: float = 60.0) -> MagicMock:
+    def _make_router(self, cooldown_time: float = 60.0, cache: DualCache | None = None) -> MagicMock:
         router = MagicMock()
         router.cooldown_time = cooldown_time
         router.allowed_fails = 0
         router.allowed_fails_policy = None
         router.get_allowed_fails_from_policy.return_value = None
-        router.failed_calls.get_cache.return_value = None
+        router.cache = cache if cache is not None else DualCache(in_memory_cache=InMemoryCache())
         return router
 
     def test_cooldown_time_override_zero_is_not_falsy(self):
         """cooldown_time_override=0 must be honored; it must not fall through to the router-level value."""
         router = self._make_router(cooldown_time=60.0)
+        router.cache = MagicMock()
+        router.cache.increment_cache.return_value = 1
         exc = litellm.RateLimitError("429", "openai", "gpt-4")
 
         should_cooldown_based_on_allowed_fails_policy(
@@ -291,12 +295,68 @@ class TestShouldCooldownBasedOnAllowedFailsPolicy:
             cooldown_time_override=0.0,
         )
 
-        set_cache_call = router.failed_calls.set_cache.call_args
-        assert set_cache_call is not None
-        assert set_cache_call[1]["ttl"] == 0.0, (
+        increment_call = router.cache.increment_cache.call_args
+        assert increment_call is not None
+        assert increment_call[1]["ttl"] == 0.0, (
             "cooldown_time_override=0 should be used as TTL, not the router-level 60.0"
         )
 
+    def test_fail_counter_is_shared_across_router_instances(self):
+        """Two workers (two Router objects over one shared cache) must pool their failures toward allowed_fails."""
+        shared_cache = DualCache(in_memory_cache=InMemoryCache())
+        workers = (self._make_router(cache=shared_cache), self._make_router(cache=shared_cache))
+        exc = litellm.AuthenticationError("401", "openai", "gpt-4")
+
+        results = [
+            should_cooldown_based_on_allowed_fails_policy(
+                litellm_router_instance=workers[i % 2],
+                deployment="dep-1",
+                original_exception=exc,
+                allowed_fails_override=5,
+            )
+            for i in range(6)
+        ]
+
+        assert results == [False, False, False, False, False, True]
+        assert shared_cache.get_cache(key="deployment:dep-1:allowed_fails") == 6
+
+    def test_fleet_wide_count_from_redis_decides_cooldown(self):
+        """The Redis (fleet-wide) count decides, even when this process has only seen one failure."""
+        redis_cache = MagicMock()
+        redis_cache.increment_cache.return_value = 6
+        router = self._make_router(cache=DualCache(in_memory_cache=InMemoryCache(), redis_cache=redis_cache))
+        exc = litellm.AuthenticationError("401", "openai", "gpt-4")
+
+        result = should_cooldown_based_on_allowed_fails_policy(
+            litellm_router_instance=router,
+            deployment="dep-1",
+            original_exception=exc,
+            allowed_fails_override=5,
+        )
+
+        assert result is True
+        redis_cache.increment_cache.assert_called_once_with("deployment:dep-1:allowed_fails", 1, ttl=60.0)
+
+    def test_redis_outage_falls_back_to_this_workers_count(self):
+        """When every Redis increment fails, the worker's own in-memory count must still cool the deployment down."""
+        redis_cache = MagicMock()
+        redis_cache.increment_cache.side_effect = ConnectionError("redis down")
+        router = self._make_router(cache=DualCache(in_memory_cache=InMemoryCache(), redis_cache=redis_cache))
+        exc = litellm.AuthenticationError("401", "openai", "gpt-4")
+
+        results = [
+            should_cooldown_based_on_allowed_fails_policy(
+                litellm_router_instance=router,
+                deployment="dep-1",
+                original_exception=exc,
+                allowed_fails_override=5,
+            )
+            for _ in range(6)
+        ]
+
+        assert results == [False, False, False, False, False, True]
+        assert redis_cache.increment_cache.call_count == 6
+
 
 class TestRoutingGroupCooldownAlternatives:
     def _router(self, routing_groups=None):
diff --git a/tests/test_litellm/router_utils/test_health_check_allowed_fails_integration.py b/tests/test_litellm/router_utils/test_health_check_allowed_fails_integration.py
index 6effbc5fa7f..9021d842daa 100644
--- a/tests/test_litellm/router_utils/test_health_check_allowed_fails_integration.py
+++ b/tests/test_litellm/router_utils/test_health_check_allowed_fails_integration.py
@@ -179,7 +179,7 @@ class TestHealthCheckCooldownIntegration:
         assert result is False
 
         # Check counter was incremented
-        current_fails = router.failed_calls.get_cache(key="deploy-1")
+        current_fails = router.cache.get_cache(key="deployment:deploy-1:allowed_fails")
         assert current_fails == 1
 
     def test_health_check_failure_triggers_cooldown_at_threshold(self):
@@ -263,7 +263,7 @@ class TestHealthCheckCooldownIntegration:
         assert "exception" not in healthy_endpoint
 
         # Verify failed_calls counter is untouched
-        current_fails = router.failed_calls.get_cache(key="deploy-1")
+        current_fails = router.cache.get_cache(key="deployment:deploy-1:allowed_fails")
         assert current_fails is None
 
     def test_disable_cooldowns_prevents_health_check_cooldown(self):

From 3496ab95182c96e596dd971808ff0d0e511ee34c Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 8 Sep 2026 13:55:36 -0700
Subject: [PATCH 75/85] fix(router): let simple-shuffle weight by any
 deployment's weight/rpm/tpm (#40222)

simple_shuffle only looked at healthy_deployments[0] to decide whether a
metric was configured, so a weight, rpm, or tpm on a later deployment was
ignored and the pick fell back to uniform random. Decide the metric from
all healthy deployments and keep the total_weight <= 0 fall-through.

Resolves LIT-7112

Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/router_strategy/simple_shuffle.py     |  3 +-
 .../router_strategy/test_simple_shuffle.py    | 54 +++++++++++++++++++
 2 files changed, 55 insertions(+), 2 deletions(-)
 create mode 100644 tests/test_litellm/router_strategy/test_simple_shuffle.py

diff --git a/litellm/router_strategy/simple_shuffle.py b/litellm/router_strategy/simple_shuffle.py
index 65bcce0e532..860e89cea22 100644
--- a/litellm/router_strategy/simple_shuffle.py
+++ b/litellm/router_strategy/simple_shuffle.py
@@ -41,8 +41,7 @@ def simple_shuffle(
 
     ############## Check if 'weight' or 'rpm' or 'tpm' param set for a weighted pick #################
     for weight_by in ["weight", "rpm", "tpm"]:
-        weight = healthy_deployments[0].get("litellm_params").get(weight_by, None)
-        if weight is not None:
+        if any(m["litellm_params"].get(weight_by) is not None for m in healthy_deployments):
             weights = [m["litellm_params"].get(weight_by, 0) for m in healthy_deployments]
             verbose_router_logger.debug("\nweight %s", weights)
             total_weight = sum(weights)
diff --git a/tests/test_litellm/router_strategy/test_simple_shuffle.py b/tests/test_litellm/router_strategy/test_simple_shuffle.py
new file mode 100644
index 00000000000..165c1751f63
--- /dev/null
+++ b/tests/test_litellm/router_strategy/test_simple_shuffle.py
@@ -0,0 +1,54 @@
+from collections import Counter
+
+import pytest
+
+from litellm import Router
+from litellm.types.router import DeploymentTypedDict, LiteLLMParamsTypedDict
+
+DRAWS = 200
+
+
+def _deployment(dep_id: str, metric: LiteLLMParamsTypedDict | None = None) -> DeploymentTypedDict:
+    params: LiteLLMParamsTypedDict = {"model": "gpt-4o", "api_key": "key", "mock_response": f"from {dep_id}"}
+    return {
+        "model_name": "test-model",
+        "litellm_params": {**params, **(metric or {})},
+        "model_info": {"id": dep_id},
+    }
+
+
+async def _draw_model_ids(router: Router) -> Counter[str]:
+    counts: Counter[str] = Counter()
+    for _ in range(DRAWS):
+        response = await router.acompletion(model="test-model", messages=[{"role": "user", "content": "hi"}])
+        counts[response._hidden_params["model_id"]] += 1
+    return counts
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("metric", [{"weight": 5}, {"rpm": 5}, {"tpm": 5}], ids=["weight", "rpm", "tpm"])
+async def test_weighted_pick_when_only_a_later_deployment_carries_the_metric(metric: LiteLLMParamsTypedDict):
+    router = Router(
+        model_list=[_deployment("unweighted"), _deployment("weighted", metric)],
+        routing_strategy="simple-shuffle",
+        num_retries=0,
+    )
+
+    counts = await _draw_model_ids(router)
+
+    assert counts["weighted"] == DRAWS
+    assert counts["unweighted"] == 0
+
+
+@pytest.mark.asyncio
+async def test_uniform_pick_when_every_configured_weight_is_zero():
+    router = Router(
+        model_list=[_deployment("unweighted"), _deployment("standby", {"weight": 0})],
+        routing_strategy="simple-shuffle",
+        num_retries=0,
+    )
+
+    counts = await _draw_model_ids(router)
+
+    assert counts["unweighted"] > 0
+    assert counts["standby"] > 0

From 9fffda4117536ff7996718ea8e9a1e1d4611063a Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 8 Sep 2026 13:56:06 -0700
Subject: [PATCH 76/85] fix(router): give cost-based routing its own cache key
 so it stops overwriting latency samples (#40225)

* fix(router): give cost-based routing its own cache key so it stops overwriting latency samples

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(router): prefix the cost routing cache key so it cannot alias another group's latency key

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/router_strategy/lowest_cost.py        | 11 ++---
 .../router_strategy/test_lowest_cost.py       | 46 ++++++++++++++++++-
 2 files changed, 50 insertions(+), 7 deletions(-)

diff --git a/litellm/router_strategy/lowest_cost.py b/litellm/router_strategy/lowest_cost.py
index 6eb4d86d280..d271349914e 100644
--- a/litellm/router_strategy/lowest_cost.py
+++ b/litellm/router_strategy/lowest_cost.py
@@ -39,7 +39,7 @@ class LowestCostLoggingHandler(CustomLogger):
                 # ------------
                 """
                 {
-                    {model_group}_map: {
+                    cost_map:{model_group}: {
                         id: {
                             f"{date:hour:minute}" : {"tpm": 34, "rpm": 3}
                         }
@@ -50,7 +50,7 @@ class LowestCostLoggingHandler(CustomLogger):
                 current_hour: Final = datetime.now().strftime("%H")
                 current_minute: Final = datetime.now().strftime("%M")
                 precise_minute: Final = f"{current_date}-{current_hour}-{current_minute}"
-                cost_key: Final = f"{model_group}_map"
+                cost_key: Final = f"cost_map:{model_group}"
 
                 total_tokens = 0
 
@@ -112,15 +112,14 @@ class LowestCostLoggingHandler(CustomLogger):
                 # ------------
                 """
                 {
-                    {model_group}_map: {
+                    cost_map:{model_group}: {
                         id: {
-                            "cost": [..]
                             f"{date:hour:minute}" : {"tpm": 34, "rpm": 3}
                         }
                     }
                 }
                 """
-                cost_key: Final = f"{model_group}_map"
+                cost_key: Final = f"cost_map:{model_group}"
 
                 current_date: Final = datetime.now().strftime("%Y-%m-%d")
                 current_hour: Final = datetime.now().strftime("%H")
@@ -176,7 +175,7 @@ class LowestCostLoggingHandler(CustomLogger):
         """
         Returns a deployment with the lowest cost
         """
-        cost_key: Final = f"{model_group}_map"
+        cost_key: Final = f"cost_map:{model_group}"
 
         request_count_dict: Final = await self.router_cache.async_get_cache(key=cost_key) or {}
 
diff --git a/tests/test_litellm/router_strategy/test_lowest_cost.py b/tests/test_litellm/router_strategy/test_lowest_cost.py
index 108053dddd9..ab3ef099410 100644
--- a/tests/test_litellm/router_strategy/test_lowest_cost.py
+++ b/tests/test_litellm/router_strategy/test_lowest_cost.py
@@ -1,3 +1,4 @@
+import copy
 from datetime import datetime
 
 import pytest
@@ -7,6 +8,8 @@ from litellm.caching.caching import DualCache
 from litellm.router_strategy.lowest_cost import LowestCostLoggingHandler
 
 DEPLOYMENT_ID = "9876"
+COST_KEY = "cost_map:gpt-5.5-pool"
+LATENCY_KEYS = ("gpt-5.5-pool_map", "gpt-5.5-pool_cost_map")
 KWARGS = {
     "litellm_params": {
         "metadata": {"model_group": "gpt-5.5-pool"},
@@ -24,7 +27,7 @@ def _chat_response_with_no_completion_tokens() -> litellm.ModelResponse:
 
 
 def _recorded_minute_counters(cache: DualCache) -> dict[str, int]:
-    cached = cache.get_cache(key="gpt-5.5-pool_map") or {}
+    cached = cache.get_cache(key=COST_KEY) or {}
     minute_buckets = cached.get(DEPLOYMENT_ID, {})
     assert len(minute_buckets) == 1, f"expected one minute bucket, got {minute_buckets}"
     return next(iter(minute_buckets.values()))
@@ -44,6 +47,47 @@ def test_log_success_event_counts_a_response_with_no_completion_tokens():
     assert _recorded_minute_counters(cache) == {"tpm": 12, "rpm": 1}
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"])
+async def test_log_success_event_keeps_cost_bookkeeping_out_of_the_latency_routing_entry(use_async: bool):
+    cache = DualCache()
+    latency_entry = {DEPLOYMENT_ID: {"latency": [0.5], "time_to_first_token": [0.1]}}
+    for latency_key in LATENCY_KEYS:
+        cache.set_cache(key=latency_key, value=copy.deepcopy(latency_entry))
+    handler = LowestCostLoggingHandler(router_cache=cache)
+    call_args = {
+        "kwargs": KWARGS,
+        "response_obj": _chat_response_with_no_completion_tokens(),
+        "start_time": datetime(2026, 1, 1, 12, 0, 0),
+        "end_time": datetime(2026, 1, 1, 12, 0, 2),
+    }
+
+    if use_async:
+        await handler.async_log_success_event(**call_args)
+    else:
+        handler.log_success_event(**call_args)
+
+    assert [cache.get_cache(key=latency_key) for latency_key in LATENCY_KEYS] == [latency_entry, latency_entry]
+    assert _recorded_minute_counters(cache) == {"tpm": 12, "rpm": 1}
+
+
+@pytest.mark.asyncio
+async def test_async_get_available_deployments_applies_rpm_limit_from_the_cost_entry():
+    cache = DualCache()
+    handler = LowestCostLoggingHandler(router_cache=cache)
+    precise_minute = datetime.now().strftime("%Y-%m-%d-%H-%M")
+    cache.set_cache(key=COST_KEY, value={DEPLOYMENT_ID: {precise_minute: {"tpm": 12, "rpm": 1}}})
+    healthy_deployments = [{"model_info": {"id": DEPLOYMENT_ID}, "litellm_params": {"model": "gpt-5.5", "rpm": 1}}]
+
+    picked = await handler.async_get_available_deployments(
+        model_group="gpt-5.5-pool",
+        healthy_deployments=healthy_deployments,
+        messages=[{"role": "user", "content": "hi"}],
+    )
+
+    assert picked is None
+
+
 @pytest.mark.asyncio
 async def test_async_log_success_event_counts_a_response_with_no_completion_tokens():
     cache = DualCache()

From 35451ecc7b3a2a9a2309091d397c3d21cd71ca29 Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 8 Sep 2026 21:15:36 +0000
Subject: [PATCH 77/85] fix(router): keep deployment tags out of retry and
 fallback tag routing (#40226)

* fix(router): keep deployment tags out of retry and fallback tag routing

Deployment-level tags merged into request metadata for spend attribution were also read as caller tag constraints on later attempts, so a tag-filtered group re-narrowed to the deployment that just failed. Snapshot the caller's routing tags before the merge and have tag routing read that snapshot.

Resolves LIT-7113

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* refactor(router): drop redundant comment in tag routing

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(router): make tag retry regression deterministic and cover routing snapshot helper

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/constants.py                          |  1 +
 litellm/proxy/common_utils/callback_utils.py  |  2 +
 litellm/proxy/litellm_pre_call_utils.py       |  2 +
 litellm/router.py                             |  6 +++
 litellm/router_strategy/tag_based_routing.py  | 11 +++--
 .../test_router_tag_routing.py                | 15 ++++++
 tests/test_litellm/test_router.py             | 46 ++++++++++++++++++-
 7 files changed, 78 insertions(+), 5 deletions(-)

diff --git a/litellm/constants.py b/litellm/constants.py
index 8a6a044ff7b..82ca92475ec 100644
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -1464,6 +1464,7 @@ SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: Final = "_session_deployment_affin
 OUTPUT_TOKEN_CEILING_PARAMS: Final = frozenset({"max_tokens", "max_completion_tokens", "max_output_tokens"})
 CLIENT_OUTPUT_CEILING_METADATA_KEY: Final = "_client_output_ceiling"
 CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags"
+ROUTING_REQUEST_TAGS_METADATA_KEY: Final = "_routing_request_tags"
 INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
 SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"
 SESSION_ID_OMITTED_METADATA_KEY: Final = "litellm_session_id_omitted"
diff --git a/litellm/proxy/common_utils/callback_utils.py b/litellm/proxy/common_utils/callback_utils.py
index d9d56778405..770963a1f24 100644
--- a/litellm/proxy/common_utils/callback_utils.py
+++ b/litellm/proxy/common_utils/callback_utils.py
@@ -13,6 +13,7 @@ from litellm.constants import (
     CLIENT_OUTPUT_CEILING_METADATA_KEY,
     CONSUMED_REQUEST_TAGS_METADATA_KEY,
     PRE_CALL_EXECUTED_GUARDRAILS_KEY,
+    ROUTING_REQUEST_TAGS_METADATA_KEY,
     SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
 )
 from litellm.integrations.custom_logger import CustomLogger
@@ -509,6 +510,7 @@ LITELLM_PROXY_INTERNAL_METADATA_KEYS: Final = frozenset(
         SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
         CONSUMED_REQUEST_TAGS_METADATA_KEY,
         CLIENT_OUTPUT_CEILING_METADATA_KEY,
+        ROUTING_REQUEST_TAGS_METADATA_KEY,
         "disable_global_guardrails",
         "disable_global_guardrail",
         "opted_out_global_guardrails",
diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py
index 893174dab03..f81fe312645 100644
--- a/litellm/proxy/litellm_pre_call_utils.py
+++ b/litellm/proxy/litellm_pre_call_utils.py
@@ -24,6 +24,7 @@ from litellm.constants import (
     LITELLM_PROXY_MASTER_KEY_ALIAS,
     OTEL_SERVICE_NAME_METADATA_KEYS,
     PRE_CALL_EXECUTED_GUARDRAILS_KEY,
+    ROUTING_REQUEST_TAGS_METADATA_KEY,
     SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
     SESSION_ID_GENERATED_METADATA_KEY,
     SESSION_ID_OMITTED_METADATA_KEY,
@@ -290,6 +291,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS: Final = (
     GATEWAY_INJECTED_CACHE_METADATA_KEY,
     SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
     CONSUMED_REQUEST_TAGS_METADATA_KEY,
+    ROUTING_REQUEST_TAGS_METADATA_KEY,
     INTERNAL_CALL_ORIGIN_METADATA_KEY,
     "standard_logging_object",
     "proxy_server_request",
diff --git a/litellm/router.py b/litellm/router.py
index 9b2badeaa69..b0a1ddd164e 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -62,6 +62,7 @@ from litellm.constants import (
     DEFAULT_MAX_LRU_CACHE_SIZE,
     INTERNAL_CALL_ORIGIN_METADATA_KEY,
     OUTPUT_TOKEN_CEILING_PARAMS,
+    ROUTING_REQUEST_TAGS_METADATA_KEY,
     RUNTIME_UPDATABLE_ROUTER_SETTINGS,
     SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
 )
@@ -3772,6 +3773,11 @@ class Router:
         refund_stale_reservation_before_retry(self.cache, kwargs)
         set_io_token_rate_limit_request_kwargs(kwargs, store_in_context=deployment_has_io_token_limits(deployment))
 
+        kwargs[metadata_variable_name].setdefault(
+            ROUTING_REQUEST_TAGS_METADATA_KEY,
+            tuple(_get_tags_from_request_kwargs(kwargs, metadata_variable_name=metadata_variable_name)),
+        )
+
         ## DEPLOYMENT-LEVEL TAGS
         deployment_tags: Final = deployment.get("litellm_params", {}).get("tags")
         if deployment_tags:
diff --git a/litellm/router_strategy/tag_based_routing.py b/litellm/router_strategy/tag_based_routing.py
index eabd9278cf6..d4f46e94579 100644
--- a/litellm/router_strategy/tag_based_routing.py
+++ b/litellm/router_strategy/tag_based_routing.py
@@ -13,7 +13,7 @@ from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, overload
 
 from litellm._logging import verbose_logger
-from litellm.constants import CONSUMED_REQUEST_TAGS_METADATA_KEY
+from litellm.constants import CONSUMED_REQUEST_TAGS_METADATA_KEY, ROUTING_REQUEST_TAGS_METADATA_KEY
 from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs
 from litellm.types.router import ConsumedRequestTagsStamp, DeploymentTypedDict, RouterErrors
 
@@ -461,7 +461,10 @@ def _request_tags_after_router_consumption(metadata: object, model: str) -> Sequ
     if not isinstance(metadata, Mapping):
         return None
     typed_metadata: Final[Mapping[str, object]] = metadata
-    request_tags: Final = _tags_in_metadata(typed_metadata)
+    request_tags: Final = _tags_in_metadata(
+        typed_metadata,
+        key=ROUTING_REQUEST_TAGS_METADATA_KEY if ROUTING_REQUEST_TAGS_METADATA_KEY in typed_metadata else "tags",
+    )
     stamp: Final = typed_metadata.get(CONSUMED_REQUEST_TAGS_METADATA_KEY)
     if not isinstance(stamp, ConsumedRequestTagsStamp) or stamp.model_group != model:
         return request_tags
@@ -646,7 +649,7 @@ async def get_deployments_for_tag(
     return healthy_deployments
 
 
-def _tags_in_metadata(metadata: object) -> list[str]:
+def _tags_in_metadata(metadata: object, key: str = "tags") -> list[str]:
     """
     Tags out of a metadata bucket the caller controls the shape of.
 
@@ -657,7 +660,7 @@ def _tags_in_metadata(metadata: object) -> list[str]:
     if not isinstance(metadata, Mapping):
         return []
     typed_metadata: Final[Mapping[str, object]] = metadata
-    tags: Final = typed_metadata.get("tags")
+    tags: Final = typed_metadata.get(key)
     if isinstance(tags, str) or not isinstance(tags, Sequence):
         return []
     typed_tags: Final[Sequence[object]] = tags
diff --git a/tests/test_litellm/router_strategy/test_router_tag_routing.py b/tests/test_litellm/router_strategy/test_router_tag_routing.py
index 59a59c7e16d..16c641b8d29 100644
--- a/tests/test_litellm/router_strategy/test_router_tag_routing.py
+++ b/tests/test_litellm/router_strategy/test_router_tag_routing.py
@@ -3031,6 +3031,21 @@ def test_request_tags_after_router_consumption_drops_only_the_consumed_tags():
     assert _request_tags_after_router_consumption(partially_consumed, "gemini-flash") == ("deploy:us",)
 
 
+def test_request_tags_after_router_consumption_ignores_tags_merged_from_prior_deployments():
+    from litellm.constants import CONSUMED_REQUEST_TAGS_METADATA_KEY, ROUTING_REQUEST_TAGS_METADATA_KEY
+    from litellm.router_strategy.tag_based_routing import _request_tags_after_router_consumption
+    from litellm.types.router import ConsumedRequestTagsStamp
+
+    metadata = {
+        "tags": ["route", "&region:eu", "free"],
+        ROUTING_REQUEST_TAGS_METADATA_KEY: ("route", "&region:eu"),
+        "inherited_tags": ["&region:eu"],
+        CONSUMED_REQUEST_TAGS_METADATA_KEY: ConsumedRequestTagsStamp(model_group="gemini-flash", tags=("route",)),
+    }
+    assert _request_tags_after_router_consumption(metadata, "gemini-flash") == ("&region:eu",)
+    assert _request_tags_after_router_consumption(metadata, "other-group") == ["route", "&region:eu"]
+
+
 @pytest.mark.asyncio()
 async def test_non_router_tags_still_pick_the_matching_tier_deployment():
     # tags=["route", "deploy:us"]: "route" picks the router and is spent there,
diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py
index eed34c79a06..29df6df0477 100644
--- a/tests/test_litellm/test_router.py
+++ b/tests/test_litellm/test_router.py
@@ -40,7 +40,7 @@ from litellm.router import (
     _is_retriable_anthropic_status,
 )
 from litellm.router_strategy import simple_shuffle
-from litellm.types.router import DeploymentTypedDict
+from litellm.types.router import DeploymentTypedDict, RetryPolicy
 
 
 def test_update_kwargs_does_not_mutate_defaults_and_merges_metadata():
@@ -6132,6 +6132,50 @@ def test_update_kwargs_with_deployment_no_tags():
     assert "tags" not in kwargs["metadata"]
 
 
+@pytest.mark.asyncio
+async def test_retry_does_not_narrow_tag_filtered_group_to_failed_deployments_tags():
+    router = Router(
+        model_list=[
+            {
+                "model_name": "tagged-group",
+                "litellm_params": {
+                    "model": "openai/gpt-5.5",
+                    "api_key": "fake-key",
+                    "tags": ["free"],
+                    "input_cost_per_token": 0.000001,
+                    "output_cost_per_token": 0.000001,
+                    "mock_response": "litellm.ContextWindowExceededError",
+                },
+                "model_info": {"id": "tagged-failing"},
+            },
+            {
+                "model_name": "tagged-group",
+                "litellm_params": {
+                    "model": "openai/gpt-5.5",
+                    "api_key": "fake-key",
+                    "input_cost_per_token": 0.001,
+                    "output_cost_per_token": 0.001,
+                    "mock_response": "ok",
+                },
+                "model_info": {"id": "untagged-healthy"},
+            },
+        ],
+        routing_strategy="cost-based-routing",
+        enable_tag_filtering=True,
+        num_retries=2,
+        retry_after=0,
+        retry_policy=RetryPolicy(BadRequestErrorRetries=2),
+    )
+    metadata: Final[dict[str, object]] = {}
+
+    response = await router.acompletion(
+        model="tagged-group", messages=[{"role": "user", "content": "hi"}], metadata=metadata
+    )
+
+    assert response._hidden_params["model_id"] == "untagged-healthy"
+    assert metadata["tags"] == ["free"]
+
+
 def test_update_kwargs_with_deployment_merges_tools():
     """
     Test that when both deployment litellm_params and request have tools,

From 6475443efbfecb52ef693bfd4e7149908b4283a2 Mon Sep 17 00:00:00 2001
From: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
Date: Tue, 8 Sep 2026 15:00:25 -0700
Subject: [PATCH 78/85] fix(policy_engine): run per-chunk hook guardrails whose
 post_call pipeline cannot stream

The per-chunk streaming hook skipped every guardrail stepped by a post_call pipeline, even when the pipeline is left out of the stream for lacking the unified apply_guardrail interface, so a default_on guardrail that only implements async_post_call_streaming_hook stopped governing streams it governed on the merge base. The skip set now comes from the pipelines that gate the stream, the same way the iterator hook already computes it
---
 litellm/proxy/utils.py                        | 29 ++++++++++++---
 .../proxy_logging/test_guardrail_pipeline.py  | 37 ++++++++++++++++++-
 2 files changed, 59 insertions(+), 7 deletions(-)

diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index 8b36061c6be..d40cc34476a 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -555,6 +555,24 @@ def _pipeline_is_streamable(policy_name: str, pipeline: "GuardrailPipeline") ->
     return False
 
 
+def _route_supports_streaming_pipelines(user_api_key_dict: UserAPIKeyAuth) -> bool:
+    return not user_api_key_dict.request_route or resolve_endpoint_translation(user_api_key_dict, None) is not None
+
+
+def _stream_gated_guardrail_names(
+    request_data: Mapping[str, object], user_api_key_dict: UserAPIKeyAuth
+) -> frozenset[str]:
+    if not _route_supports_streaming_pipelines(user_api_key_dict):
+        return frozenset()
+    return _pipeline_step_guardrail_names(
+        tuple(
+            (policy_name, pipeline)
+            for policy_name, pipeline in _post_call_pipelines(request_data)
+            if all(_pipeline_step_supports_unified_streaming(step.guardrail) for step in pipeline.steps)
+        )
+    )
+
+
 def _streamable_post_call_pipelines(
     request_data: Mapping[str, object], user_api_key_dict: UserAPIKeyAuth
 ) -> tuple[tuple[str, "GuardrailPipeline"], ...]:
@@ -571,13 +589,12 @@ def _streamable_post_call_pipelines(
     post_call_pipelines: Final = _post_call_pipelines(request_data)
     if not post_call_pipelines:
         return ()
-    route: Final = user_api_key_dict.request_route
-    if route and resolve_endpoint_translation(user_api_key_dict, None) is None:
+    if not _route_supports_streaming_pipelines(user_api_key_dict):
         verbose_proxy_logger.warning(
             "Policies with post_call guardrail pipelines cannot scan streaming responses on route %s yet "
             "(no endpoint guardrail translation); the stream skips the pipelines and their guardrails run "
             "on their own: %s",
-            route,
+            user_api_key_dict.request_route,
             ", ".join(policy_name for policy_name, _pipeline in post_call_pipelines),
         )
         return ()
@@ -3271,15 +3288,15 @@ class ProxyLogging:
             # dict lookups + llm_router.get_deployment() per callback per chunk.
             _cached_guardrail_data: dict | None = None
             _guardrail_data_computed = False
-            pipeline_managed: Final = (
-                _pipeline_managed_guardrail_names(data, "post_call") if caps.has_guardrail else frozenset()
+            pipeline_gated: Final = (
+                _stream_gated_guardrail_names(data, user_api_key_dict) if caps.has_guardrail else frozenset()
             )
 
             for callback in litellm.callbacks:
                 try:
                     _callback: CustomLogger | None = None
                     if isinstance(callback, CustomGuardrail):
-                        if callback.guardrail_name in pipeline_managed:
+                        if callback.guardrail_name in pipeline_gated:
                             continue
                         # Main - V2 Guardrails implementation
                         from litellm.types.guardrails import GuardrailEventHooks
diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
index 73dd6746b29..5cb595840fc 100644
--- a/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
+++ b/tests/test_litellm/proxy/utils/proxy_logging/test_guardrail_pipeline.py
@@ -2128,7 +2128,11 @@ async def test_per_chunk_streaming_hook_skips_pipeline_managed_guardrail(
             seen[self.guardrail_name] = seen.get(self.guardrail_name, 0) + 1
             return None
 
-    managed = RecordingGuardrail(
+    class UnifiedRecordingGuardrail(RecordingGuardrail):
+        async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None):
+            return inputs
+
+    managed = UnifiedRecordingGuardrail(
         guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=True
     )
     free = RecordingGuardrail(
@@ -2147,3 +2151,34 @@ async def test_per_chunk_streaming_hook_skips_pipeline_managed_guardrail(
     assert result is not None
     assert seen.get("gr-post") is None
     assert seen["gr-free"] == 1
+
+
+@pytest.mark.asyncio
+async def test_per_chunk_streaming_hook_runs_guardrail_whose_pipeline_cannot_stream(
+    proxy_logging, make_user_api_key_auth, monkeypatch
+):
+    seen: Dict[str, Any] = {}
+
+    class ChunkHookGuardrail(CustomGuardrail):
+        async def async_post_call_streaming_hook(self, user_api_key_dict, response):
+            seen["count"] = seen.get("count", 0) + 1
+            seen["response"] = response
+            return None
+
+    monkeypatch.setattr(
+        litellm,
+        "callbacks",
+        [ChunkHookGuardrail(guardrail_name="gr-post", event_hook=GuardrailEventHooks.post_call, default_on=True)],
+    )
+    monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", None, raising=False)
+    data = _post_call_pipeline_data(stream=True)
+
+    result = await proxy_logging.async_post_call_streaming_hook(
+        data=data,
+        response=_stream_chunks()[0],
+        user_api_key_dict=make_user_api_key_auth(request_route="/v1/chat/completions"),
+    )
+
+    assert result is not None
+    assert seen["count"] == 1
+    assert seen["response"] == "hello "

From d963e9fa6e770e359bdf17bbf2c47c729667e1b6 Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 8 Sep 2026 15:06:20 -0700
Subject: [PATCH 79/85] fix(proxy): keep team member budget enforced at the cap
 and across Redis counter expiry (#40304)

* fix(proxy): keep team member budget enforced at the cap and across Redis counter expiry

The cached-key auth path admitted a request when the team member counter sat exactly at max_budget, and a Redis counter that expired during a long stream was reconciled against this pod's stale in-memory copy, driving the shared counter negative and reopening the budget. Reject at >= like every other budget check, read Redis before the per-pod copy when judging the reconcile delta, and add the settled request cost after a DB reseed since reserved keys skip the normal increment

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(proxy): reseeded reservation counter also carries the settled request cost

The reseed test asserted counter == DB floor. The floor is read before the async spend flush writes this request, so the counter now lands at floor plus settled cost, matching the after leg in the PR proof

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/proxy/auth/user_api_key_auth.py       |   2 +-
 litellm/proxy/proxy_server.py                 |  46 ++++----
 .../spend_tracking/budget_reservation.py      |  28 +++--
 .../proxy/auth/test_user_api_key_auth.py      | 103 ++++++++++++++++++
 .../proxy/test_budget_reservation.py          |  68 ++++++++++++
 tests/test_litellm/proxy/test_proxy_server.py |   8 +-
 6 files changed, 212 insertions(+), 43 deletions(-)

diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py
index b39b1f330b3..d8679627bc3 100644
--- a/litellm/proxy/auth/user_api_key_auth.py
+++ b/litellm/proxy/auth/user_api_key_auth.py
@@ -2038,7 +2038,7 @@ async def _user_api_key_auth_builder(
                                     fallback_spend=team_member_spend,
                                     max_budget=team_member_budget,
                                 )
-                            if team_member_spend > team_member_budget:
+                            if team_member_spend >= team_member_budget:
                                 _entity_id: Final = f"{valid_token.user_id}:{valid_token.team_id}"
                                 raise litellm.BudgetExceededError(
                                     current_cost=team_member_spend,
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 38c4c2e96c9..238fd841ce1 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -2582,10 +2582,11 @@ async def _repair_stale_spend_counter(counter_key: str, db_spend: float) -> None
             )
 
 
-async def reseed_spend_counter_from_db(counter_key: str) -> None:
+async def reseed_spend_counter_from_db(counter_key: str) -> bool:
     """Recover a counter that the reservation reconcile found in an inconsistent
     state (missing, or where applying the reconcile delta would drive it
-    negative) by reseeding it from the DB instead of deleting it.
+    negative) by reseeding it from the DB instead of deleting it. Returns
+    whether a DB row was found and the counter was reseeded.
 
     The DB row is a LAGGING authoritative floor, not post-request truth: the
     entity .spend column is flushed in batches (every PROXY_BATCH_WRITE_AT), so
@@ -2600,8 +2601,9 @@ async def reseed_spend_counter_from_db(counter_key: str) -> None:
     """
     db_spend: Final = await SpendCounterReseed.from_db(prisma_client=prisma_client, counter_key=counter_key)
     if db_spend is None:
-        return
+        return False
     await _repair_stale_spend_counter(counter_key=counter_key, db_spend=db_spend)
+    return True
 
 
 async def _floor_spend_from_db(
@@ -2663,21 +2665,14 @@ async def _authoritative_floor_spend(
     return db_spend
 
 
-async def _read_spend_counter_estimate(counter_key: str, fallback_spend: float) -> tuple[float, bool]:
-    """Return (spend, authoritative). ``authoritative`` is True when the value
-    came from Redis or a fresh DB read (cross-pod truth), False when it came
-    from the per-pod in-memory copy or the caller's fallback. Only the
-    fail-closed path reads the flag; normal callers ignore it."""
-    # 1. Redis first (cross-pod authoritative). On clean miss, skip
-    # in-memory: per-pod in-memory only has this pod's writes, so it
-    # would mask cross-pod increments.
-    redis_clean_miss = False
+async def read_spend_counter_cache_value(counter_key: str) -> tuple[float | None, bool]:
+    """Return (value, authoritative) for the live counter, None when absent. A clean
+    Redis miss is final: the per-pod in-memory copy outlives the Redis TTL and only
+    holds this pod's writes, so it is consulted only when Redis is unreachable."""
     if spend_counter_cache.redis_cache is not None:
         try:
-            val = await spend_counter_cache.redis_cache.async_get_cache(key=counter_key)
-            if val is not None:
-                return float(val), True
-            redis_clean_miss = True
+            redis_val: Final = await spend_counter_cache.redis_cache.async_get_cache(key=counter_key)
+            return (float(redis_val) if redis_val is not None else None), True
         except Exception as e:
             verbose_proxy_logger.debug(
                 "get_current_spend: Redis read failed for %s, falling back to in-memory: %s",
@@ -2685,13 +2680,20 @@ async def _read_spend_counter_estimate(counter_key: str, fallback_spend: float)
                 e,
             )
 
-    # 2. In-memory only when Redis is unreachable.
-    if not redis_clean_miss:
-        val = spend_counter_cache.in_memory_cache.get_cache(key=counter_key)
-        if val is not None:
-            return float(val), False
+    in_memory_val: Final = spend_counter_cache.in_memory_cache.get_cache(key=counter_key)
+    return (float(in_memory_val) if in_memory_val is not None else None), False
 
-    # 3. Reseed from DB - fallback_spend lags cross-pod, would allow bypass.
+
+async def _read_spend_counter_estimate(counter_key: str, fallback_spend: float) -> tuple[float, bool]:
+    """Return (spend, authoritative). ``authoritative`` is True when the value
+    came from Redis or a fresh DB read (cross-pod truth), False when it came
+    from the per-pod in-memory copy or the caller's fallback. Only the
+    fail-closed path reads the flag; normal callers ignore it."""
+    cached_val, cached_authoritative = await read_spend_counter_cache_value(counter_key=counter_key)
+    if cached_val is not None:
+        return cached_val, cached_authoritative
+
+    # Reseed from DB - fallback_spend lags cross-pod, would allow bypass.
     db_spend: Final = await SpendCounterReseed.coalesced(
         prisma_client=prisma_client,
         spend_counter_cache=spend_counter_cache,
diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py
index 152f3befa15..d9f6e33c43c 100644
--- a/litellm/proxy/spend_tracking/budget_reservation.py
+++ b/litellm/proxy/spend_tracking/budget_reservation.py
@@ -905,13 +905,13 @@ async def _set_reserved_entry_actual_cost(
             increment=adjustment,
         )
     elif reseed_on_inconsistent:
-        # Post-call reconcile / release: the counter was flushed or reseeded
-        # between reservation and reconcile (Redis restart / cross-pod reset),
-        # so the optimistic delta no longer applies. Recover by reseeding from
-        # the DB's lagging authoritative floor rather than deleting the counter
-        # and failing open — deleting it is what left budgets unenforced after a
-        # Redis reload.
-        await reseed_spend_counter_from_db(counter_key=counter_key)
+        # Post-call reconcile / release: the counter was flushed, expired or reseeded
+        # between reservation and reconcile, so the optimistic delta no longer applies.
+        # Reseed from the DB floor (which cannot include this request's cost yet) and
+        # add the settled cost, since increment_spend_counters skips reserved keys.
+        reseeded: Final = await reseed_spend_counter_from_db(counter_key=counter_key)
+        if reseeded and actual_cost > 0:
+            await _increment_spend_counter_cache(counter_key=counter_key, increment=actual_cost)
     else:
         # Pre-call admission resize: the in-flight reservation cost is not yet
         # persisted, so the DB floor would discard it. Keep the original
@@ -925,18 +925,16 @@ async def _counter_can_apply_adjustment(
     counter_key: str,
     adjustment: float,
 ) -> bool:
-    from litellm.proxy.proxy_server import spend_counter_cache
+    from litellm.proxy.proxy_server import read_spend_counter_cache_value
 
-    current_value: Final = await spend_counter_cache.async_get_cache(key=counter_key)
+    try:
+        current_value, _ = await read_spend_counter_cache_value(counter_key=counter_key)
+    except (TypeError, ValueError):
+        return False
     if current_value is None:
         return False
 
-    try:
-        current_float: Final = float(current_value)
-    except (TypeError, ValueError):
-        return False
-
-    return not (adjustment < 0 and current_float + adjustment < -1e-12)
+    return not (adjustment < 0 and current_value + adjustment < -1e-12)
 
 
 async def _release_applied_entries_best_effort(
diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py
index 541aeabcbcd..f1a269cc00a 100644
--- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py
+++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py
@@ -6767,6 +6767,109 @@ async def test_temp_budget_increase_applied_for_cached_key():
     assert cached_after.max_budget == 2.0
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "team_member_spend, expect_blocked",
+    [
+        (2.4, True),
+        (2.4000000000000004, True),
+        (2.39, False),
+    ],
+)
+async def test_cached_key_team_member_budget_blocks_at_exact_cap(team_member_spend, expect_blocked):
+    """A team member counter sitting exactly at the cap (where a resized reservation
+    lands it) must be rejected by the cached-key auth path like every other budget check."""
+    from litellm.proxy._types import LiteLLM_TeamMembership, LiteLLM_TeamTableCachedObj
+    from litellm.proxy.common_utils.user_api_key_cache import team_membership_auth_cache_key
+    from litellm.proxy.utils import hash_token
+
+    api_key = "sk-team-member-exact-cap"
+    hashed_token = hash_token(api_key)
+    team_id = "team-exact-cap"
+    user_id = "user-exact-cap"
+    max_budget = 2.4
+
+    user_api_key_cache = DualCache()
+    await _cache_key_object(
+        hashed_token=hashed_token,
+        user_api_key_obj=UserAPIKeyAuth(
+            token=hashed_token,
+            team_id=team_id,
+            user_id=user_id,
+            team_member_spend=team_member_spend,
+        ),
+        user_api_key_cache=user_api_key_cache,
+        proxy_logging_obj=None,
+    )
+    await user_api_key_cache.async_set_cache(
+        key=f"team_id:{team_id}",
+        value=LiteLLM_TeamTableCachedObj(team_id=team_id),
+    )
+    await user_api_key_cache.async_set_cache(
+        key=user_id,
+        value=LiteLLM_UserTable(user_id=user_id, user_role=LitellmUserRoles.INTERNAL_USER),
+    )
+    await user_api_key_cache.async_set_cache(
+        key=team_membership_auth_cache_key(team_id=team_id, user_id=user_id),
+        value=LiteLLM_TeamMembership(
+            user_id=user_id,
+            team_id=team_id,
+            spend=team_member_spend,
+            budget_id="budget-exact-cap",
+            litellm_budget_table=LiteLLM_BudgetTable(max_budget=max_budget),
+        ),
+    )
+
+    mock_request = MagicMock()
+    mock_request.url.path = "/v1/messages"
+    mock_request.method = "POST"
+    mock_request.headers = {"authorization": f"Bearer {api_key}"}
+    mock_request.query_params = {}
+    mock_request.state = SimpleNamespace()
+
+    proxy_logging_obj = MagicMock()
+    proxy_logging_obj.budget_alerts = AsyncMock()
+    proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None)
+
+    async def _auth():
+        return await _user_api_key_auth_builder(
+            request=mock_request,
+            api_key=f"Bearer {api_key}",
+            azure_api_key_header="",
+            anthropic_api_key_header=None,
+            google_ai_studio_api_key_header=None,
+            azure_apim_header=None,
+            request_data={"model": "claude-sonnet-5", "messages": [{"role": "user", "content": "hi"}]},
+        )
+
+    with (
+        patch(  # test-quality-ok: the builder reads proxy settings from module globals, no injection seam
+            "litellm.proxy.proxy_server.general_settings", {"disable_budget_reservation": True}
+        ),
+        patch("litellm.proxy.proxy_server.master_key", "sk-master"),  # test-quality-ok: module-global proxy state
+        patch("litellm.proxy.proxy_server.prisma_client", MagicMock()),  # test-quality-ok: module-global proxy state
+        patch(  # test-quality-ok: seed the cached key, team and membership without a DB
+            "litellm.proxy.proxy_server.user_api_key_cache", user_api_key_cache
+        ),
+        patch(  # test-quality-ok: module-global proxy state
+            "litellm.proxy.proxy_server.proxy_logging_obj", proxy_logging_obj
+        ),
+        patch(  # test-quality-ok: the live counter needs Redis or a DB; pin the spend the check compares
+            "litellm.proxy.proxy_server.get_current_spend",
+            new=AsyncMock(return_value=team_member_spend),
+        ),
+    ):
+        if not expect_blocked:
+            result = await _auth()
+            assert result.team_member_spend == team_member_spend
+            return
+        with pytest.raises(ProxyException) as exc_info:
+            await _auth()
+
+    assert exc_info.value.type == ProxyErrorTypes.budget_exceeded
+    assert f"TeamMember={user_id}:{team_id}" in exc_info.value.message
+
+
 async def _proxy_exception_for_key(
     api_key: str,
     general_settings: dict[str, bool],
diff --git a/tests/test_litellm/proxy/test_budget_reservation.py b/tests/test_litellm/proxy/test_budget_reservation.py
index b8fb6170d34..6dab054d8ea 100644
--- a/tests/test_litellm/proxy/test_budget_reservation.py
+++ b/tests/test_litellm/proxy/test_budget_reservation.py
@@ -2198,6 +2198,74 @@ async def test_release_non_numeric_counter_reseeds_from_db(spend_counter_state):
     assert reservation["finalized"] is True
 
 
+class _ExpiringRedisCache:
+    def __init__(self) -> None:
+        self.store: dict[str, float] = {}
+
+    async def async_get_cache(self, key: str, *args: object, **kwargs: object) -> float | None:
+        return self.store.get(key)
+
+    async def async_increment(self, key: str, value: float, **kwargs: object) -> float:
+        self.store[key] = self.store.get(key, 0.0) + float(value)
+        return self.store[key]
+
+    async def async_set_max(self, key: str, value: float, **kwargs: object) -> float:
+        self.store[key] = max(self.store.get(key, float("-inf")), float(value))
+        return self.store[key]
+
+    async def async_set_cache(self, key: str, value: float, *args: object, **kwargs: object) -> bool:
+        self.store[key] = float(value)
+        return True
+
+    async def async_delete_cache(self, key: str, *args: object, **kwargs: object) -> None:
+        self.store.pop(key, None)
+
+
+@pytest.mark.asyncio
+async def test_reconcile_after_redis_counter_expiry_keeps_request_cost_enforced(
+    spend_counter_state,
+):
+    """Redis key expired mid-stream while the pod's in-memory copy still holds the
+    reserved value: reconcile must reseed from the DB floor plus the settled cost
+    instead of applying ``actual - reserved`` to the empty key."""
+    import litellm.proxy.proxy_server as ps
+
+    counter_cache, _ = spend_counter_state
+    counter_key = "spend:team_member:user-expiry:team-expiry"
+    redis_cache = _ExpiringRedisCache()
+    counter_cache.redis_cache = redis_cache
+    counter_cache.in_memory_cache.set_cache(key=counter_key, value=0.6)
+
+    reservation = {
+        "reserved_cost": 0.6,
+        "entries": [
+            {
+                "counter_key": counter_key,
+                "entity_type": "TeamMember",
+                "entity_id": "user-expiry:team-expiry",
+                "reserved_cost": 0.6,
+                "applied_adjustment": 0.0,
+            }
+        ],
+        "finalized": False,
+    }
+
+    with patch.object(  # test-quality-ok: the reseed reads the DB floor through a Prisma client the test has no seam for
+        ps.SpendCounterReseed, "from_db", AsyncMock(return_value=0.3)
+    ):
+        await ps.increment_spend_counters(
+            token="key-expiry",
+            team_id="team-expiry",
+            user_id="user-expiry",
+            response_cost=0.05,
+            budget_reservation=reservation,
+        )
+
+    assert redis_cache.store[counter_key] == pytest.approx(0.35)
+    assert counter_cache.in_memory_cache.get_cache(key=counter_key) == pytest.approx(0.35)
+    assert reservation["finalized"] is True
+
+
 @pytest.mark.asyncio
 async def test_should_invalidate_reserved_counters_after_persisted_spend_failure(
     spend_counter_state,
diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py
index 6a901c5e7de..54579e6cb7c 100644
--- a/tests/test_litellm/proxy/test_proxy_server.py
+++ b/tests/test_litellm/proxy/test_proxy_server.py
@@ -8356,8 +8356,8 @@ async def test_increment_spend_counters_reseeds_from_db_on_bad_reserved_counter(
     """When the reservation reconcile finds the counter in an inconsistent state
     (here: missing), it must NOT delete the counter and fail open (the old
     behavior, which left the counter unenforced after a Redis reload). It reseeds
-    from the authoritative DB so the counter reflects the recorded total and
-    budget gating continues."""
+    from the authoritative DB and adds this request's settled cost, which the
+    async spend flush has not written yet, so budget gating continues."""
     from litellm.caching.dual_cache import DualCache
     from litellm.proxy.proxy_server import increment_spend_counters
     from litellm.proxy.db.spend_counter_reseed import SpendCounterReseed
@@ -8394,9 +8394,7 @@ async def test_increment_spend_counters_reseeds_from_db_on_bad_reserved_counter(
             )
 
         assert budget_reservation["finalized"] is True
-        # counter reseeded to the authoritative DB value, not deleted/left None
-        # and not double-counted via a direct increment
-        assert counter_cache.in_memory_cache.get_cache(key="spend:key:key-bad-reserved-counter") == pytest.approx(0.6)
+        assert counter_cache.in_memory_cache.get_cache(key="spend:key:key-bad-reserved-counter") == pytest.approx(0.85)
     finally:
         ps.spend_counter_cache = orig_counter
         ps.prisma_client = orig_prisma

From 183d05ae052799c5d19ad597f2747ee333827182 Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 8 Sep 2026 15:45:32 -0700
Subject: [PATCH 80/85] feat(otel): add http/json export protocol for OTel v2
 traces (#40290)

* feat(otel): add http/json export protocol for OTel v2 traces

OTEL_EXPORTER_OTLP_PROTOCOL=http/json was accepted but routed to the protobuf
OTLP/HTTP exporter, so collectors that only decode JSON rejected every batch.
Route it to an OTLP/JSON span exporter that reuses the SDK HTTP transport and
expose the protocol as a select field on the OpenTelemetry callback in the
admin UI.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* refactor(otel): walk the fixed OTLP shape instead of recursing when hex-encoding ids

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(ui): map stored callback variables onto their form fields when editing a callback

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/integrations/callback_configs.json    |   7 ++
 litellm/integrations/otel/model/config.py     |   2 +-
 .../integrations/otel/plumbing/otlp_json.py   |  70 ++++++++++++
 .../integrations/otel/plumbing/providers.py   |  10 +-
 litellm/proxy/_types.py                       |   1 +
 .../otel/test_otel_v2_components.py           | 104 +++++++++++++++++-
 .../src/components/callback_info_helpers.tsx  |   1 +
 .../src/components/settings.test.tsx          |  83 ++++++++++++++
 .../src/components/settings.tsx               |  84 +++++++++-----
 9 files changed, 334 insertions(+), 28 deletions(-)
 create mode 100644 litellm/integrations/otel/plumbing/otlp_json.py

diff --git a/litellm/integrations/callback_configs.json b/litellm/integrations/callback_configs.json
index b98eef8329b..c40b90cee25 100644
--- a/litellm/integrations/callback_configs.json
+++ b/litellm/integrations/callback_configs.json
@@ -367,6 +367,13 @@
         "ui_name": "Headers",
         "description": "Headers for OTEL exporter (e.g., x-honeycomb-team=YOUR_API_KEY)",
         "required": false
+      },
+      "otel_exporter_otlp_protocol": {
+        "type": "select",
+        "ui_name": "Export Protocol",
+        "description": "OTLP wire format for trace exports. Use http/json for collectors that cannot decode protobuf",
+        "options": ["http/protobuf", "http/json"],
+        "required": false
       }
     },
     "description": "OpenTelemetry Logging Integration"
diff --git a/litellm/integrations/otel/model/config.py b/litellm/integrations/otel/model/config.py
index e3501d0ee94..422c8409411 100644
--- a/litellm/integrations/otel/model/config.py
+++ b/litellm/integrations/otel/model/config.py
@@ -69,7 +69,7 @@ class ExporterSpec(BaseModel):
 
     kind: str = Field(
         default="console",
-        description="console | in_memory | otlp_http | otlp_grpc | <factory kind>",
+        description="console | in_memory | otlp_http | http/json | otlp_grpc | <factory kind>",
     )
     endpoint: str | None = None
     traces_endpoint: str | None = Field(
diff --git a/litellm/integrations/otel/plumbing/otlp_json.py b/litellm/integrations/otel/plumbing/otlp_json.py
new file mode 100644
index 00000000000..b4b659f1e01
--- /dev/null
+++ b/litellm/integrations/otel/plumbing/otlp_json.py
@@ -0,0 +1,70 @@
+"""OTLP/HTTP span exporter that sends the OTLP/JSON encoding instead of protobuf.
+
+The SDK only ships a protobuf OTLP/HTTP exporter; this reuses its transport and
+retry loop and swaps the payload for OTLP/JSON (enums as integers, ids as hex).
+"""
+
+import base64
+import json
+from collections.abc import Mapping, Sequence
+from types import MappingProxyType
+from typing import Final, TypeAlias
+
+from google.protobuf.json_format import MessageToDict
+from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk.trace import ReadableSpan
+
+JSON_CONTENT_TYPE: Final = "application/json"
+_HEX_ID_KEYS: Final = frozenset({"traceId", "spanId", "parentSpanId"})
+
+_JsonValue: TypeAlias = "Mapping[str, _JsonValue] | Sequence[_JsonValue] | str | int | float | bool | None"
+_JsonObject: TypeAlias = Mapping[str, "_JsonValue"]
+
+
+def _objects(node: _JsonObject, key: str) -> tuple[_JsonObject, ...]:
+    items: Final = node.get(key)
+    if isinstance(items, str) or not isinstance(items, Sequence):
+        return ()
+    return tuple(item for item in items if isinstance(item, Mapping))
+
+
+def _hex_ids(node: _JsonObject) -> _JsonObject:
+    return MappingProxyType(
+        {
+            key: base64.b64decode(item).hex() if key in _HEX_ID_KEYS and isinstance(item, str) else item
+            for key, item in node.items()
+        }
+    )
+
+
+def _hex_span(span: _JsonObject) -> _JsonObject:
+    links: Final = _objects(span, "links")
+    if not links:
+        return _hex_ids(span)
+    return MappingProxyType({**_hex_ids(span), "links": tuple(_hex_ids(link) for link in links)})
+
+
+def _hex_scope_spans(scope: _JsonObject) -> _JsonObject:
+    return MappingProxyType({**scope, "spans": tuple(_hex_span(span) for span in _objects(scope, "spans"))})
+
+
+def _hex_resource_spans(resource: _JsonObject) -> _JsonObject:
+    scope_spans: Final = tuple(_hex_scope_spans(scope) for scope in _objects(resource, "scopeSpans"))
+    return MappingProxyType({**resource, "scopeSpans": scope_spans})
+
+
+def encode_spans_json(spans: Sequence[ReadableSpan]) -> bytes:
+    payload: Final[_JsonObject] = MessageToDict(encode_spans(spans), use_integers_for_enums=True)
+    resource_spans: Final = tuple(_hex_resource_spans(resource) for resource in _objects(payload, "resourceSpans"))
+    hexed: Final[_JsonObject] = MappingProxyType({**payload, "resourceSpans": resource_spans})
+    return json.dumps(hexed, default=dict, separators=(",", ":")).encode()
+
+
+class OTLPJsonSpanExporter(OTLPSpanExporter):
+    def __init__(self, endpoint: str | None, headers: dict[str, str]) -> None:  # mutable-ok: SDK __init__ takes Dict
+        super().__init__(endpoint=endpoint, headers=headers)
+        self._session.headers["Content-Type"] = JSON_CONTENT_TYPE
+
+    def _serialize_spans(self, spans: Sequence[ReadableSpan]) -> bytes:
+        return encode_spans_json(spans)
diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py
index d8da53017ab..90e68b7c2ee 100644
--- a/litellm/integrations/otel/plumbing/providers.py
+++ b/litellm/integrations/otel/plumbing/providers.py
@@ -136,7 +136,8 @@ def parse_headers(raw: str | None) -> dict[str, str]:
 
 
 _IN_MEMORY_KINDS: Final = ("in_memory", "inmemory", "memory")
-_OTLP_HTTP_KINDS: Final = ("otlp_http", "http", "http/protobuf", "http/json")
+_OTLP_HTTP_JSON_KINDS: Final = ("http/json",)
+_OTLP_HTTP_KINDS: Final = ("otlp_http", "http", "http/protobuf", *_OTLP_HTTP_JSON_KINDS)
 _OTLP_GRPC_KINDS: Final = ("otlp_grpc", "grpc")
 
 
@@ -164,6 +165,13 @@ def _exporter_from_spec(spec: ExporterSpec) -> SpanExporter:
         return factory(spec)
     if kind in _IN_MEMORY_KINDS:
         return InMemorySpanExporter()
+    if kind in _OTLP_HTTP_JSON_KINDS:
+        from litellm.integrations.otel.plumbing.otlp_json import OTLPJsonSpanExporter
+
+        return OTLPJsonSpanExporter(
+            endpoint=spec.traces_endpoint or _otlp_traces_endpoint(spec.endpoint),
+            headers=parse_headers(spec.headers),
+        )
     if kind in _OTLP_HTTP_KINDS:
         from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
             OTLPSpanExporter as HTTPExporter,
diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index d1b391d57a3..50b87264f3f 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -3584,6 +3584,7 @@ class AllCallbacks(LiteLLMPydanticObjectBase):
         ui_callback_name="OpenTelemetry",
         litellm_callback_params=[
             "OTEL_EXPORTER",
+            "OTEL_EXPORTER_OTLP_PROTOCOL",
             "OTEL_ENDPOINT",
             "OTEL_TRACES_ENDPOINT",
             "OTEL_HEADERS",
diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_components.py b/tests/test_litellm/integrations/otel/test_otel_v2_components.py
index 6676714d7f5..ae41c74944d 100644
--- a/tests/test_litellm/integrations/otel/test_otel_v2_components.py
+++ b/tests/test_litellm/integrations/otel/test_otel_v2_components.py
@@ -6,14 +6,18 @@ import json
 import threading
 from collections.abc import Iterator
 from dataclasses import replace
-from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from http.server import BaseHTTPRequestHandler, HTTPServer, ThreadingHTTPServer
 
 import pytest
 
 pytest.importorskip("opentelemetry")
 
+from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import (  # noqa: E402
+    ExportTraceServiceRequest,
+)
 from opentelemetry.sdk.metrics import MeterProvider  # noqa: E402
 from opentelemetry.sdk.metrics.export import InMemoryMetricReader  # noqa: E402
+from opentelemetry.sdk.trace import TracerProvider  # noqa: E402
 from opentelemetry.sdk.trace.export import (  # noqa: E402
     BatchSpanProcessor,
     ConsoleSpanExporter,
@@ -541,6 +545,89 @@ def test_build_span_exporter_variants():
     assert "OTLPSpanExporter" in type(http_exporter).__name__
 
 
+def _export_one_trace_to_local_collector(exporter_kind: str) -> tuple[list[dict], tuple[int, int, int]]:
+    """Run a parent/child trace through the configured exporter against a
+    throwaway HTTP collector. Returns the requests as the collector saw them
+    (child first, since it ends first) and (trace_id, parent span_id, child span_id)."""
+    received: list[dict] = []
+
+    class Collector(BaseHTTPRequestHandler):
+        def do_POST(self):
+            body = self.rfile.read(int(self.headers["Content-Length"]))
+            received.append({"path": self.path, "headers": dict(self.headers), "body": body})
+            self.send_response(200)
+            self.end_headers()
+
+        def log_message(self, *_args):
+            pass
+
+    server = HTTPServer(("127.0.0.1", 0), Collector)
+    threading.Thread(target=server.serve_forever, daemon=True).start()
+    try:
+        config = OpenTelemetryV2Config(
+            exporter=exporter_kind,
+            endpoint=f"http://127.0.0.1:{server.server_port}",
+            headers="x-collector-token=secret",
+        )
+        provider = TracerProvider()
+        provider.add_span_processor(SimpleSpanProcessor(providers.build_span_exporter(config)))
+        tracer = provider.get_tracer("test")
+        with tracer.start_as_current_span("parent", kind=SpanKind.SERVER) as parent:
+            with tracer.start_as_current_span("child") as child:
+                ids = (
+                    parent.get_span_context().trace_id,
+                    parent.get_span_context().span_id,
+                    child.get_span_context().span_id,
+                )
+        provider.shutdown()
+    finally:
+        server.shutdown()
+        server.server_close()
+    assert len(received) == 2
+    return received, ids
+
+
+def _only_span(request: dict) -> dict:
+    scope_spans = json.loads(request["body"])["resourceSpans"][0]["scopeSpans"][0]["spans"]
+    assert len(scope_spans) == 1
+    return scope_spans[0]
+
+
+def test_http_json_exporter_posts_otlp_json_to_traces_endpoint():
+    """``http/json`` must put the OTLP/JSON mapping on the wire (camelCase
+    fields, integer enums, hex ids) with a JSON content type, so collectors that
+    cannot decode protobuf can ingest the trace. Headers still travel."""
+    (child_request, parent_request), (trace_id, parent_id, child_id) = _export_one_trace_to_local_collector("http/json")
+
+    assert parent_request["path"] == "/v1/traces"
+    assert parent_request["headers"]["Content-Type"] == "application/json"
+    assert parent_request["headers"]["x-collector-token"] == "secret"
+    parent = _only_span(parent_request)
+    assert parent["name"] == "parent"
+    assert parent["kind"] == 2
+    assert parent["traceId"] == format(trace_id, "032x")
+    assert parent["spanId"] == format(parent_id, "016x")
+    assert "parentSpanId" not in parent
+    child = _only_span(child_request)
+    assert child["traceId"] == format(trace_id, "032x")
+    assert child["spanId"] == format(child_id, "016x")
+    assert child["parentSpanId"] == format(parent_id, "016x")
+
+
+def test_http_protobuf_exporter_still_posts_protobuf():
+    (_child_request, parent_request), (trace_id, _parent_id, _child_id) = _export_one_trace_to_local_collector(
+        "http/protobuf"
+    )
+
+    assert parent_request["path"] == "/v1/traces"
+    assert parent_request["headers"]["Content-Type"] == "application/x-protobuf"
+    assert format(trace_id, "032x").encode() not in parent_request["body"]
+    decoded = ExportTraceServiceRequest.FromString(parent_request["body"])
+    span = decoded.resource_spans[0].scope_spans[0].spans[0]
+    assert span.name == "parent"
+    assert span.trace_id == trace_id.to_bytes(16, "big")
+
+
 @pytest.fixture
 def otlp_collector() -> Iterator[tuple[str, list[str]]]:
     received_paths: list[str] = []
@@ -612,6 +699,21 @@ def test_traces_endpoint_per_exporter_coexists_with_default_normalization(otlp_c
     assert sorted(received_paths) == ["/services/collector/traces", "/v1/traces"]
 
 
+def test_http_json_exporter_honors_traces_endpoint(otlp_collector):
+    base_url, received_paths = otlp_collector
+    cfg = OpenTelemetryV2Config(
+        exporters=[
+            {
+                "kind": "http/json",
+                "endpoint": base_url,
+                "traces_endpoint": f"{base_url}/services/collector/traces",
+            }
+        ]
+    )
+    _export_one_span(cfg)
+    assert received_paths == ["/services/collector/traces"]
+
+
 def test_otlp_metric_exporter_uses_cumulative_histogram_temporality():
     """Histograms must export as cumulative, not delta.
 
diff --git a/ui/litellm-dashboard/src/components/callback_info_helpers.tsx b/ui/litellm-dashboard/src/components/callback_info_helpers.tsx
index 4b6f6233fe8..dfa35de1b34 100644
--- a/ui/litellm-dashboard/src/components/callback_info_helpers.tsx
+++ b/ui/litellm-dashboard/src/components/callback_info_helpers.tsx
@@ -158,6 +158,7 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [
     dynamic_params: {
       otel_endpoint: "text",
       otel_headers: "text",
+      otel_exporter_otlp_protocol: "select",
     },
     description: "OpenTelemetry Logging Integration",
   },
diff --git a/ui/litellm-dashboard/src/components/settings.test.tsx b/ui/litellm-dashboard/src/components/settings.test.tsx
index 762b23f413e..08cb9550646 100644
--- a/ui/litellm-dashboard/src/components/settings.test.tsx
+++ b/ui/litellm-dashboard/src/components/settings.test.tsx
@@ -219,6 +219,89 @@ describe("Settings", () => {
     expect(vi.mocked(setCallbacksCall)).not.toHaveBeenCalled();
   });
 
+  const mockOtelCallback = (variables: Record<string, string | null>) => {
+    mockGetCallbacksCall.mockResolvedValue({
+      callbacks: [{ name: "otel", variables }],
+      available_callbacks: {
+        otel: {
+          litellm_callback_name: "otel",
+          litellm_callback_params: ["OTEL_EXPORTER", "OTEL_EXPORTER_OTLP_PROTOCOL", "OTEL_ENDPOINT", "OTEL_HEADERS"],
+          ui_callback_name: "OpenTelemetry",
+        },
+      },
+      alerts: [],
+    });
+    mockGetCallbackConfigsCall.mockResolvedValue([
+      {
+        id: "otel",
+        displayName: "Open Telemetry",
+        dynamic_params: {
+          otel_endpoint: { type: "text", ui_name: "Endpoint URL", required: true },
+          otel_exporter_otlp_protocol: {
+            type: "select",
+            ui_name: "Export Protocol",
+            options: ["http/protobuf", "http/json"],
+            required: false,
+          },
+        },
+      },
+    ]);
+  };
+
+  const openOtelEditModal = async () => {
+    const user = userEvent.setup();
+    render(<Settings {...defaultProps} />);
+    await user.click(await screen.findByTestId("callback-actions-otel-success"));
+    await user.click(await screen.findByTestId("callback-action-edit"));
+    return user;
+  };
+
+  it("should post the chosen export protocol when a select dynamic param is saved", async () => {
+    mockOtelCallback({ OTEL_ENDPOINT: "http://collector:4318" });
+    const user = await openOtelEditModal();
+
+    expect(await screen.findByLabelText("Endpoint URL")).toHaveValue("http://collector:4318");
+    await user.click(screen.getByLabelText("Export Protocol"));
+    await user.click(await screen.findByRole("option", { name: "http/json" }));
+    expect(screen.getByLabelText("Export Protocol")).toHaveTextContent("http/json");
+
+    await user.click(within(screen.getByRole("dialog")).getByRole("button", { name: "Save Changes" }));
+
+    await waitFor(() => {
+      expect(vi.mocked(setCallbacksCall)).toHaveBeenCalledWith(
+        "token",
+        expect.objectContaining({
+          environment_variables: expect.objectContaining({
+            callback: "otel",
+            otel_endpoint: "http://collector:4318",
+            otel_exporter_otlp_protocol: "http/json",
+          }),
+        }),
+      );
+    });
+  });
+
+  it("should show the saved export protocol in the edit modal and keep it on an unchanged save", async () => {
+    mockOtelCallback({ OTEL_ENDPOINT: "http://collector:4318", OTEL_EXPORTER_OTLP_PROTOCOL: "http/json" });
+    const user = await openOtelEditModal();
+
+    expect(await screen.findByLabelText("Endpoint URL")).toHaveValue("http://collector:4318");
+    expect(screen.getByLabelText("Export Protocol")).toHaveTextContent("http/json");
+
+    await user.click(within(screen.getByRole("dialog")).getByRole("button", { name: "Save Changes" }));
+
+    await waitFor(() => {
+      expect(vi.mocked(setCallbacksCall)).toHaveBeenCalledWith("token", {
+        environment_variables: {
+          callback: "otel",
+          otel_endpoint: "http://collector:4318",
+          otel_exporter_otlp_protocol: "http/json",
+        },
+        litellm_settings: { success_callback: ["otel"] },
+      });
+    });
+  });
+
   it("should send the typed webhook url for an alert type when the alerting tab is saved", async () => {
     const user = userEvent.setup();
     render(<Settings {...defaultProps} />);
diff --git a/ui/litellm-dashboard/src/components/settings.tsx b/ui/litellm-dashboard/src/components/settings.tsx
index 9cb55b6ed5c..e549770af6e 100644
--- a/ui/litellm-dashboard/src/components/settings.tsx
+++ b/ui/litellm-dashboard/src/components/settings.tsx
@@ -14,6 +14,7 @@ import {
 } from "@/components/ui/combobox";
 import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/ui/dialog";
 import { Input } from "@/components/ui/input";
+import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select";
 import { Switch } from "@/components/ui/switch";
 import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table";
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
@@ -59,7 +60,7 @@ interface DynamicParamsFieldsProps {
 }
 
 const DynamicParamsFields: React.FC<DynamicParamsFieldsProps> = ({ params, callbackConfigs, selectedCallback }) => {
-  const { register, formState } = useFormContext<CallbackFormValues>();
+  const { register, control, formState } = useFormContext<CallbackFormValues>();
   const fieldIdPrefix = React.useId();
 
   if (!params || params.length === 0) {
@@ -74,37 +75,63 @@ const DynamicParamsFields: React.FC<DynamicParamsFieldsProps> = ({ params, callb
         const paramType = paramConfig.type || "text";
         const fieldLabel = paramConfig.ui_name || param.replace(/_/g, " ").replace(/\b\w/g, (l) => l.toUpperCase());
         const isRequired = paramConfig.required || false;
+        const selectOptions: string[] = Array.isArray(paramConfig.options) ? paramConfig.options : [];
+        const isSelect = paramType === "select" && selectOptions.length > 0;
         const fieldId = `${fieldIdPrefix}-${param}`;
-        const registration = register(
-          param,
-          isRequired ? { required: `Please enter the ${fieldLabel.toLowerCase()}` } : undefined,
-        );
+        const validationRules = isRequired ? { required: `Please enter the ${fieldLabel.toLowerCase()}` } : undefined;
+        const registration = isSelect ? undefined : register(param, validationRules);
 
         return (
           <Field key={param} className="mb-4">
             <FieldLabel htmlFor={fieldId}>
               <span className="text-sm font-medium text-foreground">{fieldLabel} </span>
             </FieldLabel>
-            {paramType === "password" ? (
-              <Input
-                id={fieldId}
-                type="password"
-                placeholder={`Enter your ${fieldLabel.toLowerCase()}`}
-                {...registration}
+            {isSelect && (
+              <Controller
+                control={control}
+                name={param}
+                rules={validationRules}
+                render={({ field }) => (
+                  <Select
+                    items={selectOptions.map((option) => ({ label: option, value: option }))}
+                    value={field.value || null}
+                    onValueChange={(selected: string | null) => field.onChange(selected ?? "")}
+                  >
+                    <SelectTrigger id={fieldId} className="w-full" onBlur={field.onBlur}>
+                      <SelectValue placeholder={`Select ${fieldLabel.toLowerCase()}`} />
+                    </SelectTrigger>
+                    <SelectContent>
+                      {selectOptions.map((option) => (
+                        <SelectItem key={option} value={option}>
+                          {option}
+                        </SelectItem>
+                      ))}
+                    </SelectContent>
+                  </Select>
+                )}
               />
-            ) : paramType === "number" ? (
-              <Input
-                id={fieldId}
-                type="number"
-                placeholder={`Enter ${fieldLabel.toLowerCase()}`}
-                min={0}
-                max={1}
-                step={0.1}
-                {...registration}
-              />
-            ) : (
-              <Input id={fieldId} placeholder={`Enter your ${fieldLabel.toLowerCase()}`} {...registration} />
             )}
+            {!isSelect &&
+              (paramType === "password" ? (
+                <Input
+                  id={fieldId}
+                  type="password"
+                  placeholder={`Enter your ${fieldLabel.toLowerCase()}`}
+                  {...registration}
+                />
+              ) : paramType === "number" ? (
+                <Input
+                  id={fieldId}
+                  type="number"
+                  placeholder={`Enter ${fieldLabel.toLowerCase()}`}
+                  min={0}
+                  max={1}
+                  step={0.1}
+                  {...registration}
+                />
+              ) : (
+                <Input id={fieldId} placeholder={`Enter your ${fieldLabel.toLowerCase()}`} {...registration} />
+              ))}
             <FieldError errors={[formState.errors[param]]} />
           </Field>
         );
@@ -271,15 +298,22 @@ const Settings: React.FC<SettingsPageProps> = ({ accessToken, userRole, userID,
 
   useEffect(() => {
     if (showEditCallback && selectedEditCallback) {
+      const params = getDynamicParamsForCallback(
+        selectedEditCallback.name,
+        callbackConfigs,
+        selectedEditCallback.variables,
+      );
+      const fieldNameFor = (variable: string) =>
+        params.find((param) => param.toUpperCase() === variable.toUpperCase()) ?? variable;
       const normalized = Object.fromEntries(
-        Object.entries(selectedEditCallback.variables || {}).map(([k, v]) => [k, v ?? ""]),
+        Object.entries(selectedEditCallback.variables || {}).map(([k, v]) => [fieldNameFor(k), v ?? ""]),
       );
       editForm.reset({
         ...normalized,
         callback: selectedEditCallback.name,
       });
     }
-  }, [showEditCallback, selectedEditCallback, editForm]);
+  }, [showEditCallback, selectedEditCallback, editForm, callbackConfigs]);
 
   const handleSwitchChange = (alertName: string) => {
     if (activeAlerts.includes(alertName)) {

From 0f886d9006f6a29101cffd658c05e7f755a4923c Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 8 Sep 2026 15:53:43 -0700
Subject: [PATCH 81/85] perf: move Anthropic, Vertex Anthropic, Ollama and HF
 template fetches off the event loop (#40311)

Direct Anthropic http image inlining, Vertex AI Anthropic forced base64 conversion, Ollama completion image download and the watsonx GPT-OSS Hugging Face chat template lookup all ran synchronous HTTP inside the async request path. Each provider config now transforms through async_inline_remote_media on the async path, the Anthropic handler awaits the config's async_transform_request before dispatch and in the Rust fallback, and watsonx text exposes async_transform_request and always awaits ahf_chat_template

Resolves LIT-7028

Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../prompt_templates/image_handling.py        |  13 +-
 litellm/llms/anthropic/chat/handler.py        | 200 ++++++++----------
 litellm/llms/anthropic/chat/transformation.py |  24 +++
 .../llms/ollama/completion/transformation.py  |  24 +++
 .../anthropic/transformation.py               |   4 +
 litellm/llms/watsonx/chat/transformation.py   |   8 +-
 .../llms/watsonx/completion/transformation.py |  12 +-
 .../chat/test_anthropic_chat_handler.py       |  51 ++++-
 .../test_ollama_completion_transformation.py  |  44 +++-
 ..._vertex_ai_anthropic_image_url_handling.py |  62 +++++-
 .../test_litellm/llms/watsonx/test_watsonx.py |  68 ++++++
 11 files changed, 380 insertions(+), 130 deletions(-)

diff --git a/litellm/litellm_core_utils/prompt_templates/image_handling.py b/litellm/litellm_core_utils/prompt_templates/image_handling.py
index 24f3b8bca7f..c9933422cc3 100644
--- a/litellm/litellm_core_utils/prompt_templates/image_handling.py
+++ b/litellm/litellm_core_utils/prompt_templates/image_handling.py
@@ -183,6 +183,7 @@ class _RemoteSource:
 class RemoteMedia:
     url: str
     fields: Mapping[str, object]
+    part_type: str
 
 
 _NO_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
@@ -192,6 +193,10 @@ def inline_every_remote_url(_media: RemoteMedia) -> bool:
     return True
 
 
+def inline_remote_image_urls(media: RemoteMedia) -> bool:
+    return media.part_type == "image_url"
+
+
 def _parse_remote_image(fields: Mapping[str, object]) -> _RemoteImage | None:
     if fields.get("type") != "image_url":
         return None
@@ -223,11 +228,11 @@ def _parse_remote_part(part: object) -> _RemoteImage | _RemoteFile | _RemoteSour
 def _remote_media(remote: _RemoteImage | _RemoteFile | _RemoteSource) -> RemoteMedia:
     match remote:
         case _RemoteImage(_, image_url, url):
-            return RemoteMedia(url, image_url if image_url is not None else _NO_FIELDS)
+            return RemoteMedia(url, image_url if image_url is not None else _NO_FIELDS, "image_url")
         case _RemoteFile(_, file, url):
-            return RemoteMedia(url, file)
-        case _RemoteSource(_, source, url):
-            return RemoteMedia(url, source)
+            return RemoteMedia(url, file, "file")
+        case _RemoteSource(part, source, url):
+            return RemoteMedia(url, source, str(part.get("type")))
 
 
 _PDF_FORMAT: Final = MappingProxyType({"format": "application/pdf"})
diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py
index c82be07a5c5..d1fe4cadf40 100644
--- a/litellm/llms/anthropic/chat/handler.py
+++ b/litellm/llms/anthropic/chat/handler.py
@@ -368,27 +368,92 @@ class AnthropicChatCompletion(BaseLLM):
         if config is None:
             raise ValueError(f"Provider config not found for model: {model} and provider: {custom_llm_provider}")
 
-        def build_request() -> tuple[dict, dict]:  # mutable-ok: rewritten in place downstream
-            """Translate the request the Python way, returning `(headers, data)`.
+        transform_params: Final = {**optional_params, "is_vertex_request": is_vertex_request}
+
+        def finish_request(request_data: dict) -> tuple[dict, dict]:  # mutable-ok: rewritten in place downstream
+            """Filter beta headers and emit pre_call, returning `(headers, data)`.
 
             The pair stays mutable because the streaming path rewrites it in
-            place (`data["stream"] = True`) before sending.
-
-            Shared by the normal path and by the Rust path's fallback, which
-            builds it only when the Rust call did not serve the request.
+            place (`data["stream"] = True`) before sending. A Rust attempt that
+            declined already emitted pre_call for this request, so skip it there.
             """
-            request_data: Final = config.transform_request(
-                model=model,
-                messages=messages,
-                optional_params={**optional_params, "is_vertex_request": is_vertex_request},
-                litellm_params=litellm_params,
-                headers=headers,
-            )
-            return update_request_with_filtered_beta(
+            request_headers, data = update_request_with_filtered_beta(
                 headers=headers,
                 request_data=request_data,
                 provider=custom_llm_provider,
             )
+            if not serves_via_rust:
+                logging_obj.pre_call(
+                    input=messages,
+                    api_key=api_key,
+                    additional_args={
+                        "complete_input_dict": data,
+                        "api_base": api_base,
+                        "headers": request_headers,
+                    },
+                )
+            print_verbose(f"_is_function_call: {_is_function_call}")
+            return request_headers, data
+
+        async def acompletion_dispatch() -> "ModelResponse | CustomStreamWrapper":
+            """Translate then send, so the provider config can inline remote media off the event loop."""
+            request_headers, data = finish_request(
+                await config.async_transform_request(
+                    model=model,
+                    messages=messages,
+                    optional_params=transform_params,
+                    litellm_params=litellm_params,
+                    headers=headers,
+                )
+            )
+            if (
+                stream is True
+            ):  # if function call - fake the streaming (need complete blocks for output parsing in openai format)
+                print_verbose("makes async anthropic streaming POST request")
+                data["stream"] = stream
+                return await self.acompletion_stream_function(
+                    model=model,
+                    messages=messages,
+                    data=data,
+                    api_base=api_base,
+                    custom_prompt_dict=custom_prompt_dict,
+                    model_response=model_response,
+                    print_verbose=print_verbose,
+                    encoding=encoding,
+                    api_key=api_key,
+                    logging_obj=logging_obj,
+                    optional_params=optional_params,
+                    stream=stream,
+                    _is_function_call=_is_function_call,
+                    json_mode=json_mode,
+                    litellm_params=litellm_params,
+                    logger_fn=logger_fn,
+                    headers=request_headers,
+                    timeout=timeout,
+                    client=(client if client is not None and isinstance(client, AsyncHTTPHandler) else None),
+                )
+            return await self.acompletion_function(
+                model=model,
+                messages=messages,
+                data=data,
+                api_base=api_base,
+                custom_prompt_dict=custom_prompt_dict,
+                model_response=model_response,
+                print_verbose=print_verbose,
+                encoding=encoding,
+                api_key=api_key,
+                provider_config=config,
+                logging_obj=logging_obj,
+                optional_params=optional_params,
+                stream=stream,
+                _is_function_call=_is_function_call,
+                litellm_params=litellm_params,
+                logger_fn=logger_fn,
+                headers=request_headers,
+                client=client,
+                json_mode=json_mode,
+                timeout=timeout,
+            )
 
         # The Rust core owns the whole call for the subset it accepts, so ask
         # before transforming: whichever path runs emits pre_call exactly once.
@@ -424,35 +489,6 @@ class AnthropicChatCompletion(BaseLLM):
                 additional_args=rust_logging_args,
             )
             if acompletion is True:
-
-                async def python_fallback() -> "ModelResponse | CustomStreamWrapper":
-                    # pre_call already fired for this request above. The Rust
-                    # path only declines before the provider is called, so this
-                    # is the same attempt continuing, not a second one.
-                    fallback_headers, fallback_data = build_request()
-                    return await self.acompletion_function(
-                        model=model,
-                        messages=messages,
-                        data=fallback_data,
-                        api_base=api_base,
-                        custom_prompt_dict=custom_prompt_dict,
-                        model_response=model_response,
-                        print_verbose=print_verbose,
-                        encoding=encoding,
-                        api_key=api_key,
-                        provider_config=config,
-                        logging_obj=logging_obj,
-                        optional_params=optional_params,
-                        stream=stream,
-                        _is_function_call=_is_function_call,
-                        litellm_params=litellm_params,
-                        logger_fn=logger_fn,
-                        headers=fallback_headers,
-                        client=client,
-                        json_mode=json_mode,
-                        timeout=timeout,
-                    )
-
                 return rust_chat_completions_bridge.achat_completions_or_fallback(
                     model=model,
                     messages=messages,
@@ -464,7 +500,7 @@ class AnthropicChatCompletion(BaseLLM):
                     extra_headers=headers,
                     timeout=timeout,
                     on_response=log_rust_post_call,
-                    python_fallback=python_fallback,
+                    python_fallback=acompletion_dispatch,
                 )
             rust_response: Final = rust_chat_completions_bridge.chat_completions(
                 model=model,
@@ -481,74 +517,18 @@ class AnthropicChatCompletion(BaseLLM):
             if rust_response is not None:
                 return rust_response
 
-        headers, data = build_request()
-
-        ## LOGGING
-        # Reaching here with `serves_via_rust` set means the Rust attempt
-        # declined at call time, before the provider was called, and already
-        # logged this request. That is the same attempt continuing.
-        if not serves_via_rust:
-            logging_obj.pre_call(
-                input=messages,
-                api_key=api_key,
-                additional_args={
-                    "complete_input_dict": data,
-                    "api_base": api_base,
-                    "headers": headers,
-                },
-            )
-        print_verbose(f"_is_function_call: {_is_function_call}")
         if acompletion is True:
-            if (
-                stream is True
-            ):  # if function call - fake the streaming (need complete blocks for output parsing in openai format)
-                print_verbose("makes async anthropic streaming POST request")
-                data["stream"] = stream
-                return self.acompletion_stream_function(
-                    model=model,
-                    messages=messages,
-                    data=data,
-                    api_base=api_base,
-                    custom_prompt_dict=custom_prompt_dict,
-                    model_response=model_response,
-                    print_verbose=print_verbose,
-                    encoding=encoding,
-                    api_key=api_key,
-                    logging_obj=logging_obj,
-                    optional_params=optional_params,
-                    stream=stream,
-                    _is_function_call=_is_function_call,
-                    json_mode=json_mode,
-                    litellm_params=litellm_params,
-                    logger_fn=logger_fn,
-                    headers=headers,
-                    timeout=timeout,
-                    client=(client if client is not None and isinstance(client, AsyncHTTPHandler) else None),
-                )
-            else:
-                return self.acompletion_function(
-                    model=model,
-                    messages=messages,
-                    data=data,
-                    api_base=api_base,
-                    custom_prompt_dict=custom_prompt_dict,
-                    model_response=model_response,
-                    print_verbose=print_verbose,
-                    encoding=encoding,
-                    api_key=api_key,
-                    provider_config=config,
-                    logging_obj=logging_obj,
-                    optional_params=optional_params,
-                    stream=stream,
-                    _is_function_call=_is_function_call,
-                    litellm_params=litellm_params,
-                    logger_fn=logger_fn,
-                    headers=headers,
-                    client=client,
-                    json_mode=json_mode,
-                    timeout=timeout,
-                )
+            return acompletion_dispatch()
         else:
+            headers, data = finish_request(
+                config.transform_request(
+                    model=model,
+                    messages=messages,
+                    optional_params=transform_params,
+                    litellm_params=litellm_params,
+                    headers=headers,
+                )
+            )
             ## COMPLETION CALL
             if (
                 stream is True
diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py
index 5f7ac73c919..5463f1862ad 100644
--- a/litellm/llms/anthropic/chat/transformation.py
+++ b/litellm/llms/anthropic/chat/transformation.py
@@ -26,6 +26,11 @@ from litellm.litellm_core_utils.core_helpers import map_finish_reason
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     sanitize_input_schema_for_anthropic,
 )
+from litellm.litellm_core_utils.prompt_templates.image_handling import (
+    RemoteMedia,
+    async_inline_remote_media,
+    inline_remote_image_urls,
+)
 from litellm.llms.base_llm.base_utils import type_to_response_format_param
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
 from litellm.types.llms.anthropic import (
@@ -1840,6 +1845,25 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                 break
         return headers
 
+    def inlines_remote_media(self, media: RemoteMedia) -> bool:
+        return inline_remote_image_urls(media) and media.url.startswith("http://")
+
+    async def async_transform_request(
+        self,
+        model: str,
+        messages: list[AllMessageValues],  # mutable-ok: BaseConfig signature
+        optional_params: dict[str, object],  # mutable-ok: BaseConfig signature
+        litellm_params: dict[str, object],  # mutable-ok: BaseConfig signature
+        headers: dict[str, object],  # mutable-ok: BaseConfig signature
+    ) -> dict[str, object]:  # mutable-ok: BaseConfig signature
+        return self.transform_request(
+            model=model,
+            messages=await async_inline_remote_media(messages, should_inline=self.inlines_remote_media),
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            headers=headers,
+        )
+
     def transform_request(
         self,
         model: str,
diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py
index dccc83efed4..a1340ba1952 100644
--- a/litellm/llms/ollama/completion/transformation.py
+++ b/litellm/llms/ollama/completion/transformation.py
@@ -16,6 +16,10 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
     custom_prompt,
     ollama_pt,
 )
+from litellm.litellm_core_utils.prompt_templates.image_handling import (
+    async_inline_remote_media,
+    inline_remote_image_urls,
+)
 from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
 from litellm.types.llms.openai import AllMessageValues, ChatCompletionUsageBlock
@@ -344,6 +348,26 @@ class OllamaConfig(BaseConfig):
         )
         return model_response
 
+    @property
+    def uses_async_transform_request(self) -> bool:
+        return True
+
+    async def async_transform_request(
+        self,
+        model: str,
+        messages: list[AllMessageValues],  # mutable-ok: BaseConfig signature
+        optional_params: dict[str, object],  # mutable-ok: BaseConfig signature
+        litellm_params: dict[str, object],  # mutable-ok: BaseConfig signature
+        headers: dict[str, object],  # mutable-ok: BaseConfig signature
+    ) -> dict[str, object]:  # mutable-ok: BaseConfig signature
+        return self.transform_request(
+            model=model,
+            messages=await async_inline_remote_media(messages, should_inline=inline_remote_image_urls),
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            headers=headers,
+        )
+
     def transform_request(
         self,
         model: str,
diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py
index 7579bc8c02e..508f68b3eca 100644
--- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py
+++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py
@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Final
 import httpx
 
 import litellm
+from litellm.litellm_core_utils.prompt_templates.image_handling import RemoteMedia, inline_remote_image_urls
 from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
 from litellm.types.llms.openai import AllMessageValues
 from litellm.types.utils import ModelResponse
@@ -51,6 +52,9 @@ class VertexAIAnthropicConfig(AnthropicConfig):
     def custom_llm_provider(self) -> str | None:
         return "vertex_ai"
 
+    def inlines_remote_media(self, media: RemoteMedia) -> bool:
+        return inline_remote_image_urls(media)
+
     def should_strip_billing_metadata(self) -> bool:
         return True
 
diff --git a/litellm/llms/watsonx/chat/transformation.py b/litellm/llms/watsonx/chat/transformation.py
index f9e71f9116e..cc616ab5f9f 100644
--- a/litellm/llms/watsonx/chat/transformation.py
+++ b/litellm/llms/watsonx/chat/transformation.py
@@ -157,11 +157,9 @@ class IBMWatsonXChatConfig(IBMWatsonXMixin, OpenAIGPTConfig):
     @staticmethod
     async def aapply_prompt_template(model: str, messages: list[dict[str, str]]) -> str | None:
         """Apply prompt template (async version)"""
-        import litellm
         from litellm.litellm_core_utils.prompt_templates.factory import (
             ahf_chat_template,
             custom_prompt,
-            hf_chat_template,
             ibm_granite_pt,
             mistral_instruct_pt,
         )
@@ -179,11 +177,7 @@ class IBMWatsonXChatConfig(IBMWatsonXMixin, OpenAIGPTConfig):
             else:
                 hf_model = model
             try:
-                # Use sync if cached, async if not
-                if hf_model in litellm.known_tokenizer_config:
-                    result = hf_chat_template(model=hf_model, messages=messages)
-                else:
-                    result = await ahf_chat_template(model=hf_model, messages=messages)
+                result = await ahf_chat_template(model=hf_model, messages=messages)
                 # Return result if it's truthy (not None and not empty string)
                 # The caller (_aconvert_watsonx_messages_core) will handle None/empty by falling back to default
                 if result:
diff --git a/litellm/llms/watsonx/completion/transformation.py b/litellm/llms/watsonx/completion/transformation.py
index 0b4c9ae917a..2be007336b4 100644
--- a/litellm/llms/watsonx/completion/transformation.py
+++ b/litellm/llms/watsonx/completion/transformation.py
@@ -16,6 +16,7 @@ from ..common_utils import (
     IBMWatsonXMixin,
     WatsonXAIError,
     _get_api_params,
+    aconvert_watsonx_messages_to_prompt,
     convert_watsonx_messages_to_prompt,
 )
 
@@ -236,7 +237,11 @@ class IBMWatsonXAIConfig(IBMWatsonXMixin, BaseConfig):
             **watsonx_auth_payload,
         }
 
-    async def atransform_request(
+    @property
+    def uses_async_transform_request(self) -> bool:
+        return True
+
+    async def async_transform_request(
         self,
         model: str,
         messages: list[AllMessageValues],
@@ -244,11 +249,6 @@ class IBMWatsonXAIConfig(IBMWatsonXMixin, BaseConfig):
         litellm_params: dict,
         headers: dict,
     ) -> dict:
-        """Async version of transform_request"""
-        from litellm.llms.watsonx.common_utils import (
-            aconvert_watsonx_messages_to_prompt,
-        )
-
         provider: Final = model.split("/")[0]
         prompt: Final = await aconvert_watsonx_messages_to_prompt(
             model=model, messages=messages, provider=provider, custom_prompt_dict={}
diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py
index b4b173b20c3..18309595414 100644
--- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py
+++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_handler.py
@@ -9,7 +9,8 @@ import pytest
 import litellm
 from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
 from litellm.llms.anthropic.chat.handler import ModelResponseIterator, make_call
-from litellm.llms.custom_httpx.http_handler import HTTPHandler
+from litellm._uuid import uuid
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from litellm.types.llms.openai import (
     ChatCompletionToolCallChunk,
     ChatCompletionToolCallFunctionChunk,
@@ -85,6 +86,54 @@ def test_anthropic_completion_does_not_send_deployment_default_limits():
     assert "default_api_key_tpm_limit" not in request_body
 
 
+async def test_anthropic_async_completion_inlines_http_images_off_the_event_loop(async_only_image_fetch):
+    http_image_url = f"http://img.example/{uuid.uuid4()}.png"
+    https_image_url = f"https://img.example/{uuid.uuid4()}.png"
+    captured = {}
+
+    def handle(request):
+        captured["body"] = json.loads(request.content)
+        return httpx.Response(
+            200,
+            json={
+                "id": "msg_1",
+                "type": "message",
+                "role": "assistant",
+                "model": "claude-sonnet-4-6",
+                "content": [{"type": "text", "text": "Green"}],
+                "stop_reason": "end_turn",
+                "usage": {"input_tokens": 1, "output_tokens": 1},
+            },
+        )
+
+    client = AsyncHTTPHandler()
+    client.client = httpx.AsyncClient(transport=httpx.MockTransport(handle))
+
+    response = await litellm.acompletion(
+        model="anthropic/claude-sonnet-4-6",
+        messages=[
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": "What colour is this?"},
+                    {"type": "image_url", "image_url": {"url": http_image_url}},
+                    {"type": "image_url", "image_url": {"url": https_image_url}},
+                ],
+            }
+        ],
+        api_key="test-key",
+        client=client,
+    )
+
+    assert response.choices[0].message.content == "Green"
+    assert async_only_image_fetch.fetched == [http_image_url]
+    sources = [part["source"] for part in captured["body"]["messages"][0]["content"] if part["type"] == "image"]
+    assert sources == [
+        {"type": "base64", "media_type": "image/png", "data": async_only_image_fetch.base64_png},
+        {"type": "url", "url": https_image_url},
+    ]
+
+
 def test_redacted_thinking_content_block_delta():
     chunk = {
         "type": "content_block_start",
diff --git a/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py b/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py
index eadc2bc9541..b2071155f3f 100644
--- a/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py
+++ b/tests/test_litellm/llms/ollama/test_ollama_completion_transformation.py
@@ -2,9 +2,11 @@ import json
 from litellm._uuid import uuid
 from unittest.mock import MagicMock, patch
 
+import httpx
 import pytest
 
-
+import litellm
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
 from litellm.llms.ollama.completion.transformation import (
     OllamaConfig,
     OllamaTextCompletionResponseIterator,
@@ -502,3 +504,43 @@ class TestOllamaTextCompletionResponseIterator:
         assert result["usage"]["prompt_tokens"] == 10
         assert result["usage"]["completion_tokens"] == 5
         assert result["usage"]["total_tokens"] == 15
+
+
+async def test_ollama_async_completion_inlines_remote_images_off_the_event_loop(async_only_image_fetch):
+    image_url = f"https://img.example/{uuid.uuid4()}.png"
+    captured = {}
+
+    def handle(request):
+        captured["body"] = json.loads(request.content)
+        return httpx.Response(
+            200,
+            json={
+                "model": "llava",
+                "response": "Green",
+                "done": True,
+                "prompt_eval_count": 1,
+                "eval_count": 1,
+            },
+        )
+
+    client = AsyncHTTPHandler()
+    client.client = httpx.AsyncClient(transport=httpx.MockTransport(handle))
+
+    response = await litellm.acompletion(
+        model="ollama/llava",
+        messages=[
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": "What colour is this?"},
+                    {"type": "image_url", "image_url": {"url": image_url}},
+                ],
+            }
+        ],
+        api_base="http://ollama.example:11434",
+        client=client,
+    )
+
+    assert response.choices[0].message.content == "Green"
+    assert async_only_image_fetch.fetched == [image_url]
+    assert captured["body"]["images"] == [async_only_image_fetch.base64_png]
diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py
index fa286f6f609..98071594ebf 100644
--- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py
+++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py
@@ -6,11 +6,15 @@ Vertex AI Anthropic models don't support URL sources for images.
 LiteLLM should convert image URLs to base64 when using Vertex AI Anthropic.
 """
 
+import json
+import sys
 from unittest.mock import patch, MagicMock
 
+import httpx
 import pytest
 
-
+import litellm
+from litellm._uuid import uuid
 from litellm.litellm_core_utils.prompt_templates.factory import (
     anthropic_messages_pt,
     convert_to_anthropic_tool_result,
@@ -371,3 +375,59 @@ class TestToolMessageImageURLHandling:
                                 assert item["source"]["type"] == "url"
                                 return
         pytest.fail("Could not find image in tool result")
+
+
+async def test_vertex_ai_anthropic_async_completion_inlines_https_images_off_the_event_loop(async_only_image_fetch):
+    from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+
+    image_url = f"https://img.example/{uuid.uuid4()}.png"
+    captured = {}
+
+    def handle(request):
+        captured["body"] = json.loads(request.content)
+        return httpx.Response(
+            200,
+            json={
+                "id": "msg_1",
+                "type": "message",
+                "role": "assistant",
+                "model": "claude-sonnet-4-6",
+                "content": [{"type": "text", "text": "Green"}],
+                "stop_reason": "end_turn",
+                "usage": {"input_tokens": 1, "output_tokens": 1},
+            },
+        )
+
+    client = AsyncHTTPHandler()
+    client.client = httpx.AsyncClient(transport=httpx.MockTransport(handle))
+    vertexai = MagicMock()
+    vertexai.preview.language_models = MagicMock()
+
+    with (
+        patch.dict(sys.modules, {"vertexai": vertexai}),
+        patch.object(  # test-quality-ok: litellm.acompletion has no seam for Vertex token minting
+            litellm.main.vertex_partner_models_chat_completion,
+            "_ensure_access_token",
+            return_value=("token", "test-project"),
+        ),
+    ):
+        response = await litellm.acompletion(
+            model="vertex_ai/claude-sonnet-4-6",
+            messages=[
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": "What colour is this?"},
+                        {"type": "image_url", "image_url": {"url": image_url}},
+                    ],
+                }
+            ],
+            vertex_project="test-project",
+            vertex_location="us-east5",
+            client=client,
+        )
+
+    assert response.choices[0].message.content == "Green"
+    assert async_only_image_fetch.fetched == [image_url]
+    sources = [part["source"] for part in captured["body"]["messages"][0]["content"] if part["type"] == "image"]
+    assert sources == [{"type": "base64", "media_type": "image/png", "data": async_only_image_fetch.base64_png}]
diff --git a/tests/test_litellm/llms/watsonx/test_watsonx.py b/tests/test_litellm/llms/watsonx/test_watsonx.py
index 8ac4472b22d..285afffefc0 100644
--- a/tests/test_litellm/llms/watsonx/test_watsonx.py
+++ b/tests/test_litellm/llms/watsonx/test_watsonx.py
@@ -356,6 +356,74 @@ async def test_watsonx_gpt_oss_uses_async_http_handler():
         assert result["status"] == "success", "Should return success status"
 
 
+@pytest.mark.parametrize("tokenizer_config_cached", [False, True], ids=["tokenizer_config", "cached_config_jinja"])
+async def test_watsonx_text_gpt_oss_async_completion_fetches_hf_template_off_the_event_loop(
+    monkeypatch, tokenizer_config_cached
+):
+    import httpx
+
+    from litellm._uuid import uuid
+    from litellm.litellm_core_utils.prompt_templates import huggingface_template_handler
+    from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
+
+    hf_model = f"openai/gpt-oss-{uuid.uuid4()}"
+    chat_template = "{% for m in messages %}<|{{ m['role'] }}|>{{ m['content'] }}{% endfor %}"
+    if tokenizer_config_cached:
+        cached_config = {"status": "success", "tokenizer": {"bos_token": None, "eos_token": None}}
+        monkeypatch.setattr(litellm, "known_tokenizer_config", {hf_model: cached_config})
+        expected_fetch = f"https://huggingface.co/{hf_model}/raw/main/chat_template.jinja"
+    else:
+        monkeypatch.setattr(litellm, "known_tokenizer_config", {})
+        expected_fetch = f"https://huggingface.co/{hf_model}/raw/main/tokenizer_config.json"
+    hf_fetched = []
+    captured = {}
+
+    def forbid_sync_client():
+        raise AssertionError("sync HuggingFace fetch ran on the request path")
+
+    async def serve_hf_file(url, **kwargs):
+        hf_fetched.append(url)
+        if url.endswith(".jinja"):
+            return httpx.Response(200, content=chat_template.encode())
+        return httpx.Response(200, json={"chat_template": chat_template, "bos_token": None, "eos_token": None})
+
+    monkeypatch.setattr(huggingface_template_handler, "_get_httpx_client", forbid_sync_client)
+    monkeypatch.setattr(huggingface_template_handler, "get_async_httpx_client", lambda **kwargs: Mock(get=serve_hf_file))
+
+    def handle(request):
+        captured["body"] = json.loads(request.content)
+        return httpx.Response(
+            200,
+            json={
+                "model_id": hf_model,
+                "results": [
+                    {
+                        "generated_text": "Hi",
+                        "generated_token_count": 1,
+                        "input_token_count": 1,
+                        "stop_reason": "eos_token",
+                    }
+                ],
+            },
+        )
+
+    client = AsyncHTTPHandler()
+    client.client = httpx.AsyncClient(transport=httpx.MockTransport(handle))
+
+    response = await litellm.acompletion(
+        model=f"watsonx_text/{hf_model}",
+        messages=[{"role": "user", "content": "Hi there"}],
+        api_base="https://test-api.watsonx.ai",
+        project_id="test-project-id",
+        token="test-token",
+        client=client,
+    )
+
+    assert response.choices[0].message.content == "Hi"
+    assert hf_fetched == [expected_fetch]
+    assert captured["body"]["input"] == "<|user|>Hi there"
+
+
 def test_watsonx_chat_completion_with_reasoning_effort(monkeypatch):
     """
     Test that 'reasoning_effort' is correctly passed through to the WatsonX API payload.

From 8ce4c050198b957d742fcac3b7a9ee3a407e5738 Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 8 Sep 2026 15:54:11 -0700
Subject: [PATCH 82/85] fix(router): move retry-policy retries off the refusing
 deployment on every router entrypoint (#40306)

Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/router.py                 |  24 +++-
 tests/test_litellm/test_router.py | 227 ++++++++++++++++++++++++++++++
 2 files changed, 250 insertions(+), 1 deletion(-)

diff --git a/litellm/router.py b/litellm/router.py
index e1095fe26f3..934a4ac86a9 100644
--- a/litellm/router.py
+++ b/litellm/router.py
@@ -3656,6 +3656,13 @@ class Router:
         effective_model_info: Final = kwargs.get("model_info") or deployment.get("model_info") or MappingProxyType({})
         self._set_failed_deployment_id_on_exception(exception, MappingProxyType({"model_info": effective_model_info}))
 
+    @staticmethod
+    def _stamp_retry_skip_deployment_id(exception: Exception, kwargs: Mapping[str, object]) -> None:
+        effective_model_info: Final = kwargs.get("model_info")
+        deployment_id: Final = effective_model_info.get("id") if isinstance(effective_model_info, Mapping) else None
+        if isinstance(deployment_id, str) and deployment_id:
+            exception.retry_skip_deployment_id = deployment_id  # pyright: ignore[reportAttributeAccessIssue]  # dynamic stamp, read by _deployment_ids_to_skip_on_retry
+
     def _update_kwargs_with_default_litellm_params(
         self, kwargs: dict, metadata_variable_name: str | None = "metadata"
     ) -> None:
@@ -4358,6 +4365,7 @@ class Router:
                 model=model,
                 messages=[{"role": "user", "content": "prompt"}],
                 specific_deployment=kwargs.pop("specific_deployment", None),
+                request_kwargs=kwargs,
             )
             self._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs)
             data: Final = deployment["litellm_params"].copy()
@@ -4388,6 +4396,7 @@ class Router:
             verbose_router_logger.info("litellm.image_generation(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e)
             if model_name is not None:
                 self.fail_calls[model_name] += 1
+            self._stamp_retry_skip_deployment_id(e, kwargs)
             raise e
 
     async def aimage_generation(self, prompt: str, model: str, **kwargs):
@@ -4472,6 +4481,7 @@ class Router:
             verbose_router_logger.info("litellm.aimage_generation(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e)
             if model_name is not None:
                 self.fail_calls[model_name] += 1
+            self._stamp_retry_skip_deployment_id(e, kwargs)
             raise e
 
     async def atranscription(self, file: FileTypes, model: str, **kwargs):
@@ -4576,6 +4586,7 @@ class Router:
             verbose_router_logger.info("litellm.atranscription(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e)
             if model_name is not None:
                 self.fail_calls[model_name] += 1
+            self._stamp_retry_skip_deployment_id(e, kwargs)
             raise e
 
     async def aspeech(self, model: str, input: str, voice: str | None = None, **kwargs):
@@ -4690,6 +4701,7 @@ class Router:
             verbose_router_logger.info("litellm.aspeech(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e)
             if model_name is not None:
                 self.fail_calls[model_name] += 1
+            self._stamp_retry_skip_deployment_id(e, kwargs)
             raise e
 
     async def arerank(self, model: str, **kwargs):
@@ -4748,6 +4760,7 @@ class Router:
             verbose_router_logger.info("litellm.arerank(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e)
             if model_name is not None:
                 self.fail_calls[model_name] += 1
+            self._stamp_retry_skip_deployment_id(e, kwargs)
             raise e
 
     def text_completion(
@@ -4882,6 +4895,7 @@ class Router:
             verbose_router_logger.info("litellm.atext_completion(model=%s)\x1b[31m Exception %s\x1b[0m", model, e)
             if model is not None:
                 self.fail_calls[model] += 1
+            self._stamp_retry_skip_deployment_id(e, kwargs)
             raise e
 
     async def aadapter_completion(
@@ -4972,6 +4986,7 @@ class Router:
             verbose_router_logger.info("litellm.aadapter_completion(model=%s)\x1b[31m Exception %s\x1b[0m", model, e)
             if model is not None:
                 self.fail_calls[model] += 1
+            self._stamp_retry_skip_deployment_id(e, kwargs)
             raise e
 
     async def _asearch_with_fallbacks(self, original_function: Callable, **kwargs):
@@ -5754,6 +5769,7 @@ class Router:
                 model=model,
                 input=input,
                 specific_deployment=kwargs.pop("specific_deployment", None),
+                request_kwargs=kwargs,
             )
             self._update_kwargs_with_deployment(deployment=deployment, kwargs=kwargs)
             data: Final = deployment["litellm_params"].copy()
@@ -5792,6 +5808,7 @@ class Router:
             verbose_router_logger.info("litellm.embedding(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e)
             if model_name is not None:
                 self.fail_calls[model_name] += 1
+            self._stamp_retry_skip_deployment_id(e, kwargs)
             raise e
 
     async def aembedding(
@@ -5879,6 +5896,7 @@ class Router:
             verbose_router_logger.info("litellm.aembedding(model=%s)\x1b[31m Exception %s\x1b[0m", model_name, e)
             if model_name is not None:
                 self.fail_calls[model_name] += 1
+            self._stamp_retry_skip_deployment_id(e, kwargs)
             raise e
 
     #### FILES API ####
@@ -6252,6 +6270,7 @@ class Router:
             )
             if model is not None:
                 self.fail_calls[model] += 1
+            self._stamp_retry_skip_deployment_id(e, kwargs)
             raise e
 
     async def aretrieve_batch(
@@ -6472,6 +6491,7 @@ class Router:
             )
             if model is not None:
                 self.fail_calls[model] += 1
+            self._stamp_retry_skip_deployment_id(e, kwargs)
             raise e
 
     async def alist_batches(
@@ -7589,7 +7609,9 @@ class Router:
 
     @staticmethod
     def _deployment_ids_to_skip_on_retry(exception: Exception, already_skipped: object) -> tuple[str, ...]:
-        failed_deployment_id: Final[str | None] = getattr(exception, "failed_deployment_id", None)
+        failed_deployment_id: Final[str | None] = getattr(exception, "retry_skip_deployment_id", None) or getattr(
+            exception, "failed_deployment_id", None
+        )
         status_code: Final = getattr(exception, "status_code", None)
         if not failed_deployment_id or not isinstance(status_code, int):
             return ()
diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py
index 7bfde0def09..bc79c5f6589 100644
--- a/tests/test_litellm/test_router.py
+++ b/tests/test_litellm/test_router.py
@@ -6,6 +6,7 @@ import logging
 import os
 import threading
 from datetime import datetime
+from collections.abc import Awaitable, Callable, Mapping
 from types import SimpleNamespace
 from typing import Final
 from unittest.mock import AsyncMock, MagicMock, patch
@@ -27,6 +28,7 @@ from litellm.llms.bedrock.common_utils import BedrockError
 from litellm.llms.anthropic.experimental_pass_through.messages.agentic_streaming_iterator import (
     SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES,
 )
+from litellm.types.llms.openai import ChatCompletionRequest
 from litellm.router import (
     MAX_BUFFERED_PRE_CONTENT_ANTHROPIC_CHUNKS,
     FallbackAwareAnthropicMessagesStream,
@@ -13972,6 +13974,31 @@ def test_router_deployment_ids_to_skip_on_retry(status_code, failed_deployment_i
     assert litellm.Router._deployment_ids_to_skip_on_retry(exception, already_skipped) == expected
 
 
+@pytest.mark.parametrize(
+    "kwargs,failed_deployment_id,expected",
+    [
+        ({"model_info": {"id": "rejecting"}}, None, ("rejecting",)),
+        ({"model_info": {"id": "rejecting"}}, "cooldown-target", ("rejecting",)),
+        ({"model_info": {"id": ""}}, None, ()),
+        ({"model_info": {"id": 7}}, None, ()),
+        ({"model_info": "rejecting"}, None, ()),
+        ({}, None, ()),
+        ({}, "cooldown-target", ("cooldown-target",)),
+    ],
+)
+def test_router_retry_skip_stamp_feeds_deployment_ids_to_skip_on_retry(
+    kwargs: Mapping[str, object], failed_deployment_id: str | None, expected: tuple[str, ...]
+):
+    exception = Exception("upstream refused this request")
+    exception.status_code = 400
+    exception.failed_deployment_id = failed_deployment_id
+
+    litellm.Router._stamp_retry_skip_deployment_id(exception, kwargs)
+
+    assert litellm.Router._deployment_ids_to_skip_on_retry(exception, None) == expected
+    assert exception.failed_deployment_id == failed_deployment_id
+
+
 @pytest.mark.parametrize(
     "value,expected",
     [
@@ -14165,6 +14192,206 @@ async def test_router_retry_policy_400_never_returns_to_a_deployment_that_alread
     assert response.choices[0].message.content == "hi back"
 
 
+_LIT_7114_CHAT_OK = {
+    "id": "chatcmpl-lit-7114",
+    "object": "chat.completion",
+    "created": 1,
+    "model": "gpt-5.6",
+    "choices": [{"index": 0, "message": {"role": "assistant", "content": "hi back"}, "finish_reason": "stop"}],
+    "usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3},
+}
+_LIT_7114_EMBEDDING_OK = {
+    "object": "list",
+    "data": [{"object": "embedding", "index": 0, "embedding": [0.1]}],
+    "model": "text-embedding-3-large",
+    "usage": {"prompt_tokens": 1, "total_tokens": 1},
+}
+_LIT_7114_IMAGE_OK = {"created": 1, "data": [{"b64_json": "aGk="}]}
+_LIT_7114_BATCH_OK = {
+    "id": "batch_lit_7114",
+    "object": "batch",
+    "endpoint": "/v1/chat/completions",
+    "input_file_id": "file-lit-7114",
+    "completion_window": "24h",
+    "status": "validating",
+    "created_at": 1,
+}
+
+
+class _PassthroughAdapter(CustomLogger):
+    def translate_completion_input_params(self, kwargs: ChatCompletionRequest) -> ChatCompletionRequest:
+        return ChatCompletionRequest(**kwargs)
+
+    def translate_completion_output_params(self, response: litellm.ModelResponse) -> litellm.ModelResponse:
+        return response
+
+
+def _lit_7114_router(litellm_model: str) -> litellm.Router:
+    api_base_suffix: Final = "" if litellm_model.startswith("cohere/") else "/v1"
+    return litellm.Router(
+        model_list=[
+            {
+                "model_name": "gpt-5.6",
+                "litellm_params": {
+                    "model": litellm_model,
+                    "api_key": "sk-fake",
+                    "api_base": f"https://{host}.local{api_base_suffix}",
+                    "weight": weight,
+                },
+                "model_info": {"id": host},
+            }
+            for host, weight in (("rejecting", 1), ("accepting", 0))
+        ],
+        num_retries=2,
+        retry_policy={"BadRequestErrorRetries": 2},
+        disable_cooldowns=True,
+    )
+
+
+def _lit_7114_mock_upstreams(
+    respx_mock: respx.MockRouter, path: str, refusal_status: int, success_body: Mapping[str, object] | bytes
+) -> tuple[respx.Route, respx.Route]:
+    ok_response: Final = (
+        httpx.Response(200, content=success_body)
+        if isinstance(success_body, bytes)
+        else httpx.Response(200, json=success_body)
+    )
+    rejecting: Final = respx_mock.post(f"https://rejecting.local{path}").mock(
+        return_value=httpx.Response(
+            refusal_status, json={"error": _UPSTREAM_400, "message": "upstream refused this request"}
+        )
+    )
+    accepting: Final = respx_mock.post(f"https://accepting.local{path}").mock(return_value=ok_response)
+    return rejecting, accepting
+
+
+_LIT_7114_ASYNC_ENTRYPOINTS: Final[
+    Mapping[str, tuple[str, str, int, Mapping[str, object] | bytes, Callable[[litellm.Router], Awaitable[object]]]]
+] = {
+    "aembedding": (
+        "openai/text-embedding-3-large",
+        "/v1/embeddings",
+        400,
+        _LIT_7114_EMBEDDING_OK,
+        lambda router: router.aembedding(model="gpt-5.6", input="hi"),
+    ),
+    "aimage_generation": (
+        "openai/gpt-image-1",
+        "/v1/images/generations",
+        400,
+        _LIT_7114_IMAGE_OK,
+        lambda router: router.aimage_generation(model="gpt-5.6", prompt="a cat"),
+    ),
+    "atext_completion": (
+        "text-completion-openai/gpt-3.5-turbo-instruct",
+        "/v1/completions",
+        400,
+        {"id": "c", "object": "text_completion", "created": 1, "model": "i", "choices": [{"text": "hi", "index": 0}]},
+        lambda router: router.atext_completion(model="gpt-5.6", prompt="hi"),
+    ),
+    "aspeech": (
+        "openai/gpt-4o-mini-tts",
+        "/v1/audio/speech",
+        400,
+        b"RIFF",
+        lambda router: router.aspeech(model="gpt-5.6", input="hi", voice="alloy"),
+    ),
+    "atranscription": (
+        "openai/gpt-4o-transcribe",
+        "/v1/audio/transcriptions",
+        400,
+        {"text": "hi"},
+        lambda router: router.atranscription(model="gpt-5.6", file=("hi.wav", b"RIFF", "audio/wav")),
+    ),
+    "arerank": (
+        "cohere/rerank-v3.5",
+        "/v2/rerank",
+        400,
+        {"id": "r", "results": [{"index": 0, "relevance_score": 0.9}], "meta": {}},
+        lambda router: router.arerank(model="gpt-5.6", query="hi", documents=["hi"]),
+    ),
+    "aadapter_completion": (
+        "openai/gpt-5.6",
+        "/v1/chat/completions",
+        400,
+        _LIT_7114_CHAT_OK,
+        lambda router: router.aadapter_completion(
+            adapter_id="lit-7114", model="gpt-5.6", messages=[{"role": "user", "content": "hi"}]
+        ),
+    ),
+    "acreate_batch": (
+        "openai/gpt-5.6",
+        "/v1/batches",
+        401,
+        _LIT_7114_BATCH_OK,
+        lambda router: router.acreate_batch(
+            model="gpt-5.6", completion_window="24h", endpoint="/v1/chat/completions", input_file_id="file-lit-7114"
+        ),
+    ),
+    "acancel_batch": (
+        "openai/gpt-5.6",
+        "/v1/batches/batch_lit_7114/cancel",
+        401,
+        {**_LIT_7114_BATCH_OK, "status": "cancelling"},
+        lambda router: router.acancel_batch(model="gpt-5.6", batch_id="batch_lit_7114"),
+    ),
+}
+
+
+@pytest.mark.parametrize("entrypoint", sorted(_LIT_7114_ASYNC_ENTRYPOINTS))
+@pytest.mark.asyncio
+async def test_router_retry_moves_off_the_refusing_deployment_on_every_async_entrypoint(
+    monkeypatch: pytest.MonkeyPatch, entrypoint: str
+):
+    litellm_model, path, refusal_status, success_body, call = _LIT_7114_ASYNC_ENTRYPOINTS[entrypoint]
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    monkeypatch.setattr(litellm, "adapters", [{"id": "lit-7114", "adapter": _PassthroughAdapter()}])
+    router: Final = _lit_7114_router(litellm_model)
+
+    with respx.mock as respx_mock:
+        rejecting, accepting = _lit_7114_mock_upstreams(respx_mock, path, refusal_status, success_body)
+        response: Final = await call(router)
+
+    assert response is not None
+    assert rejecting.call_count == 1
+    assert accepting.call_count == 1
+
+
+_LIT_7114_SYNC_ENTRYPOINTS: Final[
+    Mapping[str, tuple[str, str, Mapping[str, object], Callable[[litellm.Router], object]]]
+] = {
+    "embedding": (
+        "openai/text-embedding-3-large",
+        "/v1/embeddings",
+        _LIT_7114_EMBEDDING_OK,
+        lambda router: router.embedding(model="gpt-5.6", input="hi"),
+    ),
+    "image_generation": (
+        "openai/gpt-image-1",
+        "/v1/images/generations",
+        _LIT_7114_IMAGE_OK,
+        lambda router: router.image_generation(model="gpt-5.6", prompt="a cat"),
+    ),
+}
+
+
+@pytest.mark.parametrize("entrypoint", sorted(_LIT_7114_SYNC_ENTRYPOINTS))
+def test_router_retry_policy_400_moves_off_the_refusing_deployment_on_every_sync_entrypoint(
+    monkeypatch: pytest.MonkeyPatch, entrypoint: str
+):
+    litellm_model, path, success_body, call = _LIT_7114_SYNC_ENTRYPOINTS[entrypoint]
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    router: Final = _lit_7114_router(litellm_model)
+
+    with respx.mock as respx_mock:
+        rejecting, accepting = _lit_7114_mock_upstreams(respx_mock, path, 400, success_body)
+        response: Final = call(router)
+
+    assert response is not None
+    assert rejecting.call_count == 1
+    assert accepting.call_count == 1
+
+
 def _make_failure_logging_obj():
     return LiteLLMLogging(
         model="gpt-5.6",

From c6a938731998779f48a086551dd9caa7fea71dcf Mon Sep 17 00:00:00 2001
From: "devin-ai-integration[bot]"
 <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Tue, 8 Sep 2026 15:54:31 -0700
Subject: [PATCH 83/85] fix(proxy): default max_idle_connection_lifetime on
 componentized DB URLs (#40285)

* fix(proxy): default max_idle_connection_lifetime on componentized DB URLs

DatabaseURLSettings.apply_to_env() now appends max_idle_connection_lifetime=60
(or DATABASE_MAX_IDLE_CONNECTION_LIFETIME) to DATABASE_URL and DIRECT_URL before
the reader inherits the writer's connection params, so the gateway, backend and
migrations entrypoints get the same idle-connection reaping as the classic CLI.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* fix(proxy): keep DB URL connection params across IAM/Entra token refresh

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* ci: retrigger proxy-infra after process-tree test flake

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 litellm/proxy/db/db_url_settings.py           |  10 ++
 litellm/proxy/db/prisma_client.py             |   6 +-
 .../proxy/db/test_db_url_settings.py          | 152 +++++++++++++++---
 .../proxy/db/test_prisma_client.py            |  50 ++++++
 4 files changed, 192 insertions(+), 26 deletions(-)

diff --git a/litellm/proxy/db/db_url_settings.py b/litellm/proxy/db/db_url_settings.py
index f28e505246a..4a0231ad9df 100644
--- a/litellm/proxy/db/db_url_settings.py
+++ b/litellm/proxy/db/db_url_settings.py
@@ -64,6 +64,7 @@ DISABLE_PREPARED_STATEMENTS_ENV_VAR: Final = "DATABASE_DISABLE_PREPARED_STATEMEN
 DisablePreparedStatementsFlag = Annotated[
     bool, BeforeValidator(partial(token_auth_flag_enabled, env_var=DISABLE_PREPARED_STATEMENTS_ENV_VAR))
 ]
+MAX_IDLE_CONNECTION_LIFETIME_ENV_VAR: Final = "DATABASE_MAX_IDLE_CONNECTION_LIFETIME"
 
 # schema.prisma pins `provider = "postgresql"`, so these are the only schemes
 # Prisma can actually connect with.
@@ -217,6 +218,9 @@ class DatabaseURLSettings(BaseSettings):
     disable_prepared_statements: DisablePreparedStatementsFlag = Field(
         default=False, validation_alias=DISABLE_PREPARED_STATEMENTS_ENV_VAR
     )
+    max_idle_connection_lifetime: int | None = Field(
+        default=None, validation_alias=MAX_IDLE_CONNECTION_LIFETIME_ENV_VAR
+    )
 
     # Writer
     database_url: str | None = Field(default=None, validation_alias="DATABASE_URL")
@@ -453,6 +457,12 @@ class DatabaseURLSettings(BaseSettings):
                 if url:
                     os.environ[env_var] = add_missing_query_params(url, MappingProxyType({"pgbouncer": "true"}))
 
+        lifetime_params: Final = idle_lifetime_params(self.max_idle_connection_lifetime)
+        for env_var in ("DATABASE_URL", "DIRECT_URL"):
+            url = os.environ.get(env_var)
+            if url:
+                os.environ[env_var] = add_missing_query_params(url, lifetime_params)
+
         # The reader inherits the writer's connection params (pool size, timeouts,
         # pgbouncer mode). Without this the reader pool ignores the configured cap
         # and falls back to Prisma's `num_physical_cpus * 2 + 1` default.
diff --git a/litellm/proxy/db/prisma_client.py b/litellm/proxy/db/prisma_client.py
index 9ea2432f2f5..21b73f27a80 100644
--- a/litellm/proxy/db/prisma_client.py
+++ b/litellm/proxy/db/prisma_client.py
@@ -16,6 +16,7 @@ from datetime import datetime, timedelta
 from typing import Any, Final, Protocol
 
 from litellm._logging import verbose_proxy_logger
+from litellm.proxy.db.db_url_settings import add_missing_query_params, connection_params_from_url
 from litellm.proxy.db.token_auth import (
     DEFAULT_POSTGRES_PORT,
     DatabaseTokenAuth,
@@ -438,7 +439,10 @@ class PrismaWrapper:
             return None
 
         endpoint: Final = self._iam_endpoint if self._iam_endpoint is not None else self._endpoint_from_env()
-        db_url: Final = endpoint.build_url(mint_database_token(auth, endpoint))
+        db_url: Final = add_missing_query_params(
+            endpoint.build_url(mint_database_token(auth, endpoint)),
+            connection_params_from_url(os.environ.get(self._db_url_env_var, "")),
+        )
         os.environ[self._db_url_env_var] = db_url
         return db_url
 
diff --git a/tests/test_litellm/proxy/db/test_db_url_settings.py b/tests/test_litellm/proxy/db/test_db_url_settings.py
index db524625a93..ba342342366 100644
--- a/tests/test_litellm/proxy/db/test_db_url_settings.py
+++ b/tests/test_litellm/proxy/db/test_db_url_settings.py
@@ -35,6 +35,7 @@ _MANAGED_DB_ENV_VARS = (
     "IAM_TOKEN_DB_AUTH",
     "AZURE_POSTGRESQL_AUTH",
     "DATABASE_DISABLE_PREPARED_STATEMENTS",
+    "DATABASE_MAX_IDLE_CONNECTION_LIFETIME",
     "DATABASE_URL",
     "DIRECT_URL",
     "DATABASE_URL_READ_REPLICA",
@@ -111,7 +112,7 @@ def test_assembles_writer_url_when_iam_enabled(monkeypatch):
 
     assert (
         os.environ["DATABASE_URL"]
-        == "postgresql://litellm:WRITER_TOKEN@writer.example.com:5432/litellm_db"
+        == "postgresql://litellm:WRITER_TOKEN@writer.example.com:5432/litellm_db?max_idle_connection_lifetime=60"
     )
     # Reader was never configured, so it must not have been set.
     assert "DATABASE_URL_READ_REPLICA" not in os.environ
@@ -130,7 +131,9 @@ def test_a_pre_encoded_iam_user_survives_url_assembly(monkeypatch):
     with _stub_iam_token("WRITER_TOKEN"):
         assert _apply() is True
 
-    assert os.environ["DATABASE_URL"] == "postgresql://svc%40corp:WRITER_TOKEN@writer.example.com:5432/litellm_db"
+    assert os.environ["DATABASE_URL"] == (
+        "postgresql://svc%40corp:WRITER_TOKEN@writer.example.com:5432/litellm_db?max_idle_connection_lifetime=60"
+    )
 
 
 def test_an_unreadable_toggle_fails_the_settings_model(monkeypatch):
@@ -168,7 +171,7 @@ def test_reader_url_assembled_when_host_set_and_url_unset(monkeypatch):
 
     assert (
         os.environ["DATABASE_URL_READ_REPLICA"]
-        == "postgresql://litellm:READER_TOKEN@reader.example.com:5432/litellm_db"
+        == "postgresql://litellm:READER_TOKEN@reader.example.com:5432/litellm_db?max_idle_connection_lifetime=60"
     )
 
 
@@ -191,7 +194,7 @@ def test_reader_url_not_clobbered_when_already_set(monkeypatch):
 
     assert (
         os.environ["DATABASE_URL_READ_REPLICA"]
-        == "postgresql://app:secret@reader.example.com:5432/litellm_db"
+        == "postgresql://app:secret@reader.example.com:5432/litellm_db?max_idle_connection_lifetime=60"
     )
 
 
@@ -222,7 +225,8 @@ def test_reader_field_fallbacks_default_to_writer_values(monkeypatch):
 
     assert (
         os.environ["DATABASE_URL_READ_REPLICA"]
-        == "postgresql://litellm:READER_TOKEN@reader.example.com:5432/litellm_db?schema=public"
+        == "postgresql://litellm:READER_TOKEN@reader.example.com:5432/litellm_db"
+        "?schema=public&max_idle_connection_lifetime=60"
     )
 
 
@@ -242,7 +246,7 @@ def test_assembles_writer_url_when_azure_entra_enabled(monkeypatch):
 
     assert os.environ["DATABASE_URL"] == (
         "postgresql://litellm%40contoso.onmicrosoft.com:ENTRA_TOKEN"
-        "@writer.postgres.database.azure.com:5432/litellm_db"
+        "@writer.postgres.database.azure.com:5432/litellm_db?max_idle_connection_lifetime=60"
     )
     assert os.environ["AZURE_POSTGRESQL_AUTH"] == "True"
     assert "IAM_TOKEN_DB_AUTH" not in os.environ
@@ -261,7 +265,7 @@ def test_azure_reader_url_assembled_from_writer_fallbacks(monkeypatch):
 
     assert os.environ["DATABASE_URL_READ_REPLICA"] == (
         "postgresql://litellm%40contoso.onmicrosoft.com:ENTRA_TOKEN"
-        "@reader.postgres.database.azure.com:5432/litellm_db?schema=public"
+        "@reader.postgres.database.azure.com:5432/litellm_db?schema=public&max_idle_connection_lifetime=60"
     )
 
 
@@ -357,7 +361,7 @@ def test_assembles_writer_url_from_password(monkeypatch):
     assert _apply() is True
     assert (
         os.environ["DATABASE_URL"]
-        == "postgresql://litellm:s3cr3t@writer.example.com:5432/litellm_db"
+        == "postgresql://litellm:s3cr3t@writer.example.com:5432/litellm_db?max_idle_connection_lifetime=60"
     )
 
 
@@ -370,7 +374,7 @@ def test_writer_password_is_percent_encoded(monkeypatch):
     assert _apply() is True
     assert (
         os.environ["DATABASE_URL"]
-        == "postgresql://litellm:p%40ss%2Fw%3Ard@writer.example.com:5432/litellm_db"
+        == "postgresql://litellm:p%40ss%2Fw%3Ard@writer.example.com:5432/litellm_db?max_idle_connection_lifetime=60"
     )
 
 
@@ -388,7 +392,7 @@ def test_writer_url_not_clobbered_when_already_set(monkeypatch):
     assert _apply() is False
     assert (
         os.environ["DATABASE_URL"]
-        == "postgresql://pinned:url@db.example.com:5432/litellm_db"
+        == "postgresql://pinned:url@db.example.com:5432/litellm_db?max_idle_connection_lifetime=60"
     )
 
 
@@ -400,7 +404,7 @@ def test_writer_url_passwordless(monkeypatch):
     assert _apply() is True
     assert (
         os.environ["DATABASE_URL"]
-        == "postgresql://litellm@writer.example.com:5432/litellm_db"
+        == "postgresql://litellm@writer.example.com:5432/litellm_db?max_idle_connection_lifetime=60"
     )
 
 
@@ -415,7 +419,7 @@ def test_database_username_alias(monkeypatch):
     assert _apply() is True
     assert (
         os.environ["DATABASE_URL"]
-        == "postgresql://litellm:s3cr3t@writer.example.com:5432/litellm_db"
+        == "postgresql://litellm:s3cr3t@writer.example.com:5432/litellm_db?max_idle_connection_lifetime=60"
     )
 
 
@@ -429,7 +433,7 @@ def test_password_reader_falls_back_to_writer_password(monkeypatch):
     assert _apply() is True
     assert (
         os.environ["DATABASE_URL_READ_REPLICA"]
-        == "postgresql://litellm:s3cr3t@reader.example.com:5432/litellm_db"
+        == "postgresql://litellm:s3cr3t@reader.example.com:5432/litellm_db?max_idle_connection_lifetime=60"
     )
 
 
@@ -445,7 +449,7 @@ def test_password_reader_uses_own_credentials(monkeypatch):
     assert _apply() is True
     assert (
         os.environ["DATABASE_URL_READ_REPLICA"]
-        == "postgresql://litellm_ro:ro_pw@reader.example.com:5432/litellm_db"
+        == "postgresql://litellm_ro:ro_pw@reader.example.com:5432/litellm_db?max_idle_connection_lifetime=60"
     )
 
 
@@ -641,19 +645,19 @@ def test_reader_keeps_its_own_options_when_writer_params_are_appended(monkeypatc
     assert query["connection_limit"] == ["3"]
 
 
-def test_reader_url_left_alone_when_writer_has_no_params(monkeypatch):
+def test_reader_url_left_alone_when_nothing_is_missing(monkeypatch):
     """No params to inherit must mean the reader URL is not rewritten at all."""
     monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@writer.example.com:5432/db")
     monkeypatch.setenv(
         "DATABASE_URL_READ_REPLICA",
-        "postgresql://u:p@reader.example.com:5432/db?options=-c%20search_path%3Dapp",
+        "postgresql://u:p@reader.example.com:5432/db?options=-c%20search_path%3Dapp&max_idle_connection_lifetime=45",
     )
 
     _apply()
 
     assert (
         os.environ["DATABASE_URL_READ_REPLICA"]
-        == "postgresql://u:p@reader.example.com:5432/db?options=-c%20search_path%3Dapp"
+        == "postgresql://u:p@reader.example.com:5432/db?options=-c%20search_path%3Dapp&max_idle_connection_lifetime=45"
     )
 
 
@@ -671,7 +675,7 @@ def test_disable_prepared_statements_appends_pgbouncer_to_assembled_writer(monke
 
     assert _apply() is True
     assert os.environ["DATABASE_URL"] == (
-        "postgresql://litellm:s3cr3t@writer.example.com:5432/litellm_db?pgbouncer=true"
+        "postgresql://litellm:s3cr3t@writer.example.com:5432/litellm_db?pgbouncer=true&max_idle_connection_lifetime=60"
     )
     assert "DIRECT_URL" not in os.environ
 
@@ -685,7 +689,9 @@ def test_disable_prepared_statements_appends_pgbouncer_to_pinned_writer(monkeypa
     monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@db.example.com:5432/litellm_db")
 
     assert _apply() is False
-    assert os.environ["DATABASE_URL"] == "postgresql://u:p@db.example.com:5432/litellm_db?pgbouncer=true"
+    assert os.environ["DATABASE_URL"] == (
+        "postgresql://u:p@db.example.com:5432/litellm_db?pgbouncer=true&max_idle_connection_lifetime=60"
+    )
 
 
 def test_disable_prepared_statements_respects_a_pinned_pgbouncer_value(monkeypatch):
@@ -694,7 +700,9 @@ def test_disable_prepared_statements_respects_a_pinned_pgbouncer_value(monkeypat
 
     _apply()
 
-    assert os.environ["DATABASE_URL"] == "postgresql://u:p@db.example.com:5432/litellm_db?pgbouncer=false"
+    assert os.environ["DATABASE_URL"] == (
+        "postgresql://u:p@db.example.com:5432/litellm_db?pgbouncer=false&max_idle_connection_lifetime=60"
+    )
 
 
 def test_disable_prepared_statements_applies_to_direct_url(monkeypatch):
@@ -704,7 +712,9 @@ def test_disable_prepared_statements_applies_to_direct_url(monkeypatch):
 
     _apply()
 
-    assert os.environ["DIRECT_URL"] == "postgresql://u:p@direct.example.com:5432/litellm_db?pgbouncer=true"
+    assert os.environ["DIRECT_URL"] == (
+        "postgresql://u:p@direct.example.com:5432/litellm_db?pgbouncer=true&max_idle_connection_lifetime=60"
+    )
 
 
 def test_reader_inherits_pgbouncer_from_disable_prepared_statements(monkeypatch):
@@ -724,7 +734,9 @@ def test_disable_prepared_statements_off_leaves_urls_alone(monkeypatch):
 
     _apply()
 
-    assert os.environ["DATABASE_URL"] == "postgresql://u:p@db.example.com:5432/litellm_db"
+    assert os.environ["DATABASE_URL"] == (
+        "postgresql://u:p@db.example.com:5432/litellm_db?max_idle_connection_lifetime=60"
+    )
 
 
 def test_disable_prepared_statements_rejects_an_unreadable_value(monkeypatch):
@@ -760,6 +772,7 @@ def test_libpq_verify_full_and_sslrootcert_become_prisma_strict_sslcert(monkeypa
         "sslmode": ["require"],
         "sslcert": ["/certs/rds-bundle.pem"],
         "sslaccept": ["strict"],
+        "max_idle_connection_lifetime": ["60"],
     }
 
 
@@ -768,7 +781,11 @@ def test_libpq_verify_ca_becomes_prisma_strict(monkeypatch):
 
     _apply()
 
-    assert _query(os.environ["DATABASE_URL"]) == {"sslmode": ["require"], "sslaccept": ["strict"]}
+    assert _query(os.environ["DATABASE_URL"]) == {
+        "sslmode": ["require"],
+        "sslaccept": ["strict"],
+        "max_idle_connection_lifetime": ["60"],
+    }
 
 
 def test_sslrootcert_alone_turns_on_strict_verification(monkeypatch):
@@ -784,6 +801,7 @@ def test_sslrootcert_alone_turns_on_strict_verification(monkeypatch):
         "sslmode": ["require"],
         "sslcert": ["/certs/ca.pem"],
         "sslaccept": ["strict"],
+        "max_idle_connection_lifetime": ["60"],
     }
 
 
@@ -800,11 +818,15 @@ def test_pinned_prisma_ssl_params_win_over_libpq_translation(monkeypatch):
         "sslmode": ["require"],
         "sslcert": ["/pinned.pem"],
         "sslaccept": ["accept_invalid_certs"],
+        "max_idle_connection_lifetime": ["60"],
     }
 
 
 def test_prisma_native_ssl_url_is_left_untouched(monkeypatch):
-    url = "postgresql://u:p@db.example.com:5432/litellm_db?sslmode=require&sslcert=/certs/ca.pem&sslaccept=strict"
+    url = (
+        "postgresql://u:p@db.example.com:5432/litellm_db"
+        "?sslmode=require&sslcert=/certs/ca.pem&sslaccept=strict&max_idle_connection_lifetime=60"
+    )
     monkeypatch.setenv("DATABASE_URL", url)
 
     _apply()
@@ -820,4 +842,84 @@ def test_libpq_ssl_translation_covers_direct_url_and_read_replica(monkeypatch):
     _apply()
 
     for env_var in ("DATABASE_URL", "DIRECT_URL", "DATABASE_URL_READ_REPLICA"):
-        assert _query(os.environ[env_var]) == {"sslmode": ["require"], "sslaccept": ["strict"]}, env_var
+        assert _query(os.environ[env_var]) == {
+            "sslmode": ["require"],
+            "sslaccept": ["strict"],
+            "max_idle_connection_lifetime": ["60"],
+        }, env_var
+
+
+def test_default_idle_lifetime_applied_to_pinned_writer_and_direct_url(monkeypatch):
+    monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@db.example.com:5432/litellm_db")
+    monkeypatch.setenv("DIRECT_URL", "postgresql://u:p@direct.example.com:5432/litellm_db")
+
+    assert _apply() is False
+
+    assert os.environ["DATABASE_URL"] == (
+        "postgresql://u:p@db.example.com:5432/litellm_db?max_idle_connection_lifetime=60"
+    )
+    assert os.environ["DIRECT_URL"] == (
+        "postgresql://u:p@direct.example.com:5432/litellm_db?max_idle_connection_lifetime=60"
+    )
+
+
+def test_url_pinned_idle_lifetime_wins_over_default_and_env_knob(monkeypatch):
+    monkeypatch.setenv("DATABASE_MAX_IDLE_CONNECTION_LIFETIME", "45")
+    monkeypatch.setenv(
+        "DATABASE_URL", "postgresql://u:p@db.example.com:5432/litellm_db?max_idle_connection_lifetime=300"
+    )
+
+    _apply()
+
+    assert os.environ["DATABASE_URL"] == (
+        "postgresql://u:p@db.example.com:5432/litellm_db?max_idle_connection_lifetime=300"
+    )
+
+
+def test_env_knob_overrides_default_idle_lifetime(monkeypatch):
+    monkeypatch.setenv("DATABASE_MAX_IDLE_CONNECTION_LIFETIME", "45")
+    monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@db.example.com:5432/litellm_db")
+    monkeypatch.setenv("DIRECT_URL", "postgresql://u:p@direct.example.com:5432/litellm_db")
+
+    _apply()
+
+    assert os.environ["DATABASE_URL"] == (
+        "postgresql://u:p@db.example.com:5432/litellm_db?max_idle_connection_lifetime=45"
+    )
+    assert os.environ["DIRECT_URL"] == (
+        "postgresql://u:p@direct.example.com:5432/litellm_db?max_idle_connection_lifetime=45"
+    )
+
+
+def test_env_knob_rejects_a_non_integer_value(monkeypatch):
+    monkeypatch.setenv("DATABASE_MAX_IDLE_CONNECTION_LIFETIME", "soon")
+
+    with pytest.raises(ValidationError, match="DATABASE_MAX_IDLE_CONNECTION_LIFETIME"):
+        DatabaseURLSettings.from_env()
+
+
+@pytest.mark.parametrize(("knob", "expected"), [(None, "60"), ("45", "45")])
+def test_reader_inherits_the_writer_idle_lifetime(monkeypatch, knob, expected):
+    if knob is not None:
+        monkeypatch.setenv("DATABASE_MAX_IDLE_CONNECTION_LIFETIME", knob)
+    monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@writer.example.com:5432/db")
+    monkeypatch.setenv("DATABASE_URL_READ_REPLICA", "postgresql://u:p@reader.example.com:5432/db")
+
+    _apply()
+
+    assert os.environ["DATABASE_URL_READ_REPLICA"] == (
+        f"postgresql://u:p@reader.example.com:5432/db?max_idle_connection_lifetime={expected}"
+    )
+
+
+def test_reader_keeps_its_own_pinned_idle_lifetime(monkeypatch):
+    monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@writer.example.com:5432/db")
+    monkeypatch.setenv(
+        "DATABASE_URL_READ_REPLICA", "postgresql://u:p@reader.example.com:5432/db?max_idle_connection_lifetime=120"
+    )
+
+    _apply()
+
+    assert os.environ["DATABASE_URL_READ_REPLICA"] == (
+        "postgresql://u:p@reader.example.com:5432/db?max_idle_connection_lifetime=120"
+    )
diff --git a/tests/test_litellm/proxy/db/test_prisma_client.py b/tests/test_litellm/proxy/db/test_prisma_client.py
index 7f43e483557..963f6a5640f 100644
--- a/tests/test_litellm/proxy/db/test_prisma_client.py
+++ b/tests/test_litellm/proxy/db/test_prisma_client.py
@@ -291,6 +291,56 @@ def test_azure_entra_mint_writes_an_encoded_url_into_the_db_url_env_var(azure_en
     assert os.environ["DATABASE_URL"] == db_url
 
 
+@pytest.mark.parametrize(
+    ("previous_query", "expected_query"),
+    [
+        ("max_idle_connection_lifetime=60", {"max_idle_connection_lifetime": ["60"]}),
+        (
+            "connection_limit=20&pgbouncer=true&max_idle_connection_lifetime=45",
+            {"connection_limit": ["20"], "pgbouncer": ["true"], "max_idle_connection_lifetime": ["45"]},
+        ),
+    ],
+)
+def test_token_refresh_keeps_the_connection_params_of_the_url_it_replaces(
+    azure_env, monkeypatch, previous_query, expected_query
+):
+    old_token = _entra_jwt(60)
+    monkeypatch.setenv(
+        "DATABASE_URL",
+        f"postgresql://litellm%40contoso.onmicrosoft.com:{urllib.parse.quote(old_token, safe='')}"
+        f"@pg.postgres.database.azure.com:5432/litellm_db?{previous_query}",
+    )
+    new_token = _entra_jwt(3600)
+
+    db_url = _azure_wrapper(new_token).get_rds_iam_token()
+
+    assert db_url is not None
+    assert os.environ["DATABASE_URL"] == db_url
+    assert urllib.parse.quote(new_token, safe="") in db_url
+    assert urllib.parse.parse_qs(urllib.parse.urlsplit(db_url).query) == expected_query
+
+
+def test_token_refresh_keeps_the_reader_url_params_separate_from_the_writer(azure_env, monkeypatch):
+    from litellm.proxy.db.token_auth import IAMEndpoint
+
+    monkeypatch.setenv("DATABASE_URL", "postgresql://w:t@pg:5432/litellm_db?max_idle_connection_lifetime=45")
+    monkeypatch.setenv(
+        "DATABASE_URL_READ_REPLICA", "postgresql://r:t@replica:5432/litellm_db?max_idle_connection_lifetime=60"
+    )
+    reader = _azure_wrapper(
+        _entra_jwt(3600),
+        db_url_env_var="DATABASE_URL_READ_REPLICA",
+        iam_endpoint=IAMEndpoint(host="replica", port="5432", user="r", name="litellm_db", schema=None),
+    )
+
+    reader_url = reader.get_rds_iam_token()
+
+    assert reader_url is not None
+    assert reader_url.startswith("postgresql://r:")
+    assert urllib.parse.parse_qs(urllib.parse.urlsplit(reader_url).query) == {"max_idle_connection_lifetime": ["60"]}
+    assert os.environ["DATABASE_URL"].endswith("?max_idle_connection_lifetime=45")
+
+
 def test_azure_entra_refresh_is_scheduled_off_the_jwt_expiry(azure_env):
     """Without reading `exp` this falls back to a fixed 600s interval, which silently
     outlives a token and breaks every reconnect after it lapses (issue #29661)."""

From 4b9c289a7219400317ec41e4dc2816ba7559a542 Mon Sep 17 00:00:00 2001
From: Yassin Kortam <yassin@berri.ai>
Date: Tue, 8 Sep 2026 15:56:52 -0700
Subject: [PATCH 84/85] fix(a2a): forward caller identity headers on
 message/send and message/stream (#40305)

* fix(a2a): forward caller identity headers on message/send and message/stream

_forwarding_headers() stamped X-LiteLLM-User-Id/-Team-Id from the authenticated
caller, but was only wired into the tasks/* and tasks/resubscribe branches. The
primary message/send and message/stream conversational path forwarded
agent_extra_headers unchanged, so a downstream agent never learned which end
user was calling it except on secondary task-management calls. This broke
per-user MCP scoping and per-customer FinOps budget enforcement for any agent
invoked through the normal conversational flow.

Resolves LIT-7342

* fix(a2a): snapshot key-bound caller identity before pre-call processing

user_header_mappings lets add_litellm_data_to_request rewrite
user_api_key_dict.user_id from a client header, so the identity stamped
onto X-LiteLLM-User-Id is now captured before that step runs. Header
tests updated to expect the caller identity on message/send.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(a2a): drop redundant docstrings from message identity tests

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(a2a): type the message method test helpers

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

* test(a2a): make message method test helpers immutable

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

---------

Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
---
 .../proxy/agent_endpoints/a2a_endpoints.py    |  62 +++---
 .../agent_endpoints/test_a2a_endpoints.py     | 187 ++++++++++++------
 .../agent_endpoints/test_agent_headers.py     |  10 +-
 3 files changed, 164 insertions(+), 95 deletions(-)

diff --git a/litellm/proxy/agent_endpoints/a2a_endpoints.py b/litellm/proxy/agent_endpoints/a2a_endpoints.py
index 28882484db4..95c34f70d7b 100644
--- a/litellm/proxy/agent_endpoints/a2a_endpoints.py
+++ b/litellm/proxy/agent_endpoints/a2a_endpoints.py
@@ -144,31 +144,32 @@ def _validate_push_notification_url(url: str) -> None:
         raise HTTPException(status_code=400, detail=str(e)) from e
 
 
-def _caller_identity_headers(user_api_key_dict: UserAPIKeyAuth) -> dict[str, str]:
-    headers: Final[dict[str, str]] = {}
-    if user_api_key_dict.user_id:
-        headers["X-LiteLLM-User-Id"] = user_api_key_dict.user_id
-    if user_api_key_dict.team_id:
-        headers["X-LiteLLM-Team-Id"] = user_api_key_dict.team_id
-    return headers
+def _caller_identity_headers(user_api_key_dict: UserAPIKeyAuth) -> Mapping[str, str]:
+    return MappingProxyType(
+        {
+            name: value
+            for name, value in (
+                ("X-LiteLLM-User-Id", user_api_key_dict.user_id),
+                ("X-LiteLLM-Team-Id", user_api_key_dict.team_id),
+            )
+            if value
+        }
+    )
 
 
 def _forwarding_headers(
-    user_api_key_dict: UserAPIKeyAuth,
+    caller_identity: Mapping[str, str],
     request_data: Mapping[str, object],
     agent_extra_headers: Mapping[str, str] | None,
-) -> Mapping[str, str] | None:
-    sanitized: Final = (
-        {k: v for k, v in agent_extra_headers.items() if not k.lower().startswith("x-litellm-")}
-        if agent_extra_headers
-        else None
+) -> dict[str, str] | None:
+    passthrough: Final = tuple(
+        (name, value)
+        for name, value in (agent_extra_headers.items() if agent_extra_headers else ())
+        if not name.lower().startswith("x-litellm-")
     )
-    merged: Final = merge_agent_headers(dynamic_headers=sanitized, static_headers=None) or {}
-    identity: Final = _caller_identity_headers(user_api_key_dict)
     trace_id: Final = request_data.get("litellm_trace_id")
-    if trace_id:
-        identity["X-LiteLLM-Trace-Id"] = str(trace_id)
-    merged.update(identity)
+    trace: Final = (("X-LiteLLM-Trace-Id", str(trace_id)),) if trace_id else ()
+    merged: Final = dict((*passthrough, *caller_identity.items(), *trace))
     return merged or None
 
 
@@ -755,6 +756,7 @@ async def invoke_agent_a2a(
             ProxyBaseLLMRequestProcessing,
         )
 
+        caller_identity: Final = _caller_identity_headers(user_api_key_dict)
         processor: Final = ProxyBaseLLMRequestProcessing(data=body)
         data, logging_obj = await processor.common_processing_pre_call_logic(
             request=request,
@@ -793,9 +795,13 @@ async def invoke_agent_a2a(
                     if header_name:
                         dynamic_headers[header_name] = val
 
-        agent_extra_headers = merge_agent_headers(
-            dynamic_headers=dynamic_headers or None,
-            static_headers=static_headers or None,
+        agent_extra_headers = _forwarding_headers(
+            caller_identity=caller_identity,
+            request_data=data,
+            agent_extra_headers=merge_agent_headers(
+                dynamic_headers=dynamic_headers or None,
+                static_headers=static_headers or None,
+            ),
         )
 
         # Databricks App endpoints require a short-lived OAuth M2M token rather
@@ -942,12 +948,7 @@ async def invoke_agent_a2a(
                 "method": method,
                 "params": params,
             }
-            caller_headers: Final = _forwarding_headers(
-                user_api_key_dict=user_api_key_dict,
-                request_data=data,
-                agent_extra_headers=agent_extra_headers,
-            )
-            result = await _forward_jsonrpc(agent_url, forward_body, extra_headers=caller_headers)
+            result = await _forward_jsonrpc(agent_url, forward_body, extra_headers=agent_extra_headers)
             if method == "agent/getAuthenticatedExtendedCard":
                 card: Final = result.get("result")
                 if isinstance(card, dict):
@@ -988,16 +989,11 @@ async def invoke_agent_a2a(
                 "method": method,
                 "params": params,
             }
-            sse_caller_headers: Final = _forwarding_headers(
-                user_api_key_dict=user_api_key_dict,
-                request_data=data,
-                agent_extra_headers=agent_extra_headers,
-            )
             return await _forward_jsonrpc_sse(
                 agent_url,
                 forward_body,
                 request_id=request_id,
-                extra_headers=sse_caller_headers,
+                extra_headers=agent_extra_headers,
                 proxy_logging_obj=proxy_logging_obj,
                 user_api_key_dict=user_api_key_dict,
                 request_data=data,
diff --git a/tests/test_litellm/proxy/agent_endpoints/test_a2a_endpoints.py b/tests/test_litellm/proxy/agent_endpoints/test_a2a_endpoints.py
index 43034f889f6..e0476361074 100644
--- a/tests/test_litellm/proxy/agent_endpoints/test_a2a_endpoints.py
+++ b/tests/test_litellm/proxy/agent_endpoints/test_a2a_endpoints.py
@@ -7,11 +7,24 @@ Tests that invoke_agent_a2a properly integrates with add_litellm_data_to_request
 import json
 import socket
 import sys
-from contextlib import ExitStack
+from collections.abc import Awaitable, Callable, Mapping
+from contextlib import AbstractContextManager, ExitStack
+from dataclasses import dataclass
+from typing import Final
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
 
+from litellm.proxy._types import UserAPIKeyAuth
+
+AddLiteLLMData = Callable[..., Awaitable[dict[str, object]]]
+
+
+@dataclass(frozen=True, slots=True)
+class CapturedAgentCall:
+    request_id: object
+    agent_extra_headers: dict[str, str] | None
+
 
 @pytest.mark.asyncio
 async def test_invoke_agent_a2a_adds_litellm_data():
@@ -364,7 +377,7 @@ def _make_agent_mock(url: str = "http://backend-agent:10001") -> MagicMock:
 
 
 def _make_request_mock(
-    method: str, params: dict, request_id: object = "req-1"
+    method: str, params: Mapping[str, object], request_id: object = "req-1"
 ) -> MagicMock:
     req = MagicMock()
     req.headers = {}
@@ -379,7 +392,9 @@ def _make_request_mock(
     return req
 
 
-def _base_patches(agent: MagicMock):
+def _base_patches(
+    agent: MagicMock, add_litellm_data: AddLiteLLMData | None = None
+) -> list[AbstractContextManager[object]]:
     return [
         patch(
             "litellm.proxy.agent_endpoints.a2a_endpoints._get_agent",
@@ -391,7 +406,7 @@ def _base_patches(agent: MagicMock):
         ),
         patch(
             "litellm.proxy.common_request_processing.add_litellm_data_to_request",
-            new=AsyncMock(side_effect=_add_proxy_data),
+            new=AsyncMock(side_effect=add_litellm_data or _add_proxy_data),
         ),
         patch("litellm.proxy.proxy_server.general_settings", {}),
         patch("litellm.proxy.proxy_server.proxy_config", MagicMock()),
@@ -399,84 +414,67 @@ def _base_patches(agent: MagicMock):
     ]
 
 
-async def _add_proxy_data(data, **kwargs):
-    data["proxy_server_request"] = {
-        "url": "http://localhost:4000",
-        "method": "POST",
-        "headers": {},
-        "body": {},
+async def _add_proxy_data(data: dict[str, object], **kwargs: object) -> dict[str, object]:
+    return {
+        **data,
+        "proxy_server_request": {"url": "http://localhost:4000", "method": "POST", "headers": {}, "body": {}},
+        "metadata": data.get("metadata", {}),
     }
-    data.setdefault("metadata", {})
-    return data
 
 
-@pytest.mark.asyncio
-@pytest.mark.parametrize("method", ["message/send", "message/stream"])
-async def test_message_methods_preserve_numeric_zero_request_id(method: str):
+_HELLO_MESSAGE_PARAMS = {
+    "message": {
+        "role": "user",
+        "parts": [{"kind": "text", "text": "Hello"}],
+        "messageId": "msg-123",
+    }
+}
+
+
+async def _invoke_message_method(
+    method: str,
+    mock_request: MagicMock,
+    user_api_key_dict: UserAPIKeyAuth,
+    add_litellm_data: AddLiteLLMData | None = None,
+) -> CapturedAgentCall:
     from fastapi.responses import JSONResponse
-    from litellm.proxy._types import UserAPIKeyAuth
 
     class MessageSendParams:
-        def __init__(self, **kwargs):
+        def __init__(self, **kwargs: object) -> None:
             self.__dict__.update(kwargs)
 
     class SendMessageRequest:
-        def __init__(self, **kwargs):
+        def __init__(self, **kwargs: object) -> None:
             self.__dict__.update(kwargs)
 
-    agent = _make_agent_mock()
-    params = {
-        "message": {
-            "role": "user",
-            "parts": [{"kind": "text", "text": "Hello"}],
-            "messageId": "msg-123",
-        }
-    }
-    mock_request = _make_request_mock(method, params, request_id=0)
-    user_api_key_dict = UserAPIKeyAuth(api_key="sk-test", user_id="u1", team_id="t1")
-    captured = {}
-
-    async def capture_asend_message(request, **kwargs):
-        captured["request_id"] = request.id
-        response = MagicMock()
+    async def fake_asend_message(request: SendMessageRequest, **kwargs: object) -> MagicMock:
+        response: Final = MagicMock()
         response.model_dump.return_value = {
             "jsonrpc": "2.0",
-            "id": request.id,
+            "id": request.__dict__["id"],
             "result": {"status": "success"},
         }
         return response
 
-    async def capture_stream_message(**kwargs):
-        captured["request_id"] = kwargs["request_id"]
-        return JSONResponse({"jsonrpc": "2.0", "id": kwargs["request_id"]})
+    async def fake_stream_message(request_id: object, **kwargs: object) -> JSONResponse:
+        return JSONResponse({"jsonrpc": "2.0", "id": request_id})
 
-    mock_a2a_types = MagicMock()
+    mock_a2a_types: Final = MagicMock()
     mock_a2a_types.MessageSendParams = MessageSendParams
     mock_a2a_types.SendMessageRequest = SendMessageRequest
+    is_send: Final = method == "message/send"
+    downstream: Final = AsyncMock(side_effect=fake_asend_message if is_send else fake_stream_message)
 
     with ExitStack() as stack:
-        for p in _base_patches(agent):
+        for p in _base_patches(_make_agent_mock(), add_litellm_data):
             stack.enter_context(p)
         stack.enter_context(patch("litellm.a2a_protocol.main.A2A_SDK_AVAILABLE", True))
-        if method == "message/send":
-            stack.enter_context(
-                patch.dict(
-                    sys.modules,
-                    {"a2a": MagicMock(), "a2a.types": mock_a2a_types},
-                )
-            )
-            stack.enter_context(
-                patch(
-                    "litellm.a2a_protocol.asend_message",
-                    new=AsyncMock(side_effect=capture_asend_message),
-                )
-            )
+        if is_send:
+            stack.enter_context(patch.dict(sys.modules, {"a2a": MagicMock(), "a2a.types": mock_a2a_types}))
+            stack.enter_context(patch("litellm.a2a_protocol.asend_message", new=downstream))
         else:
             stack.enter_context(
-                patch(
-                    "litellm.proxy.agent_endpoints.a2a_endpoints._handle_stream_message",
-                    new=AsyncMock(side_effect=capture_stream_message),
-                )
+                patch("litellm.proxy.agent_endpoints.a2a_endpoints._handle_stream_message", new=downstream)
             )
 
         from litellm.proxy.agent_endpoints.a2a_endpoints import invoke_agent_a2a
@@ -488,7 +486,82 @@ async def test_message_methods_preserve_numeric_zero_request_id(method: str):
             user_api_key_dict=user_api_key_dict,
         )
 
-    assert captured["request_id"] == 0
+    kwargs: Final = downstream.call_args.kwargs
+    request_id: Final = kwargs["request"].__dict__["id"] if is_send else kwargs["request_id"]
+    return CapturedAgentCall(request_id=request_id, agent_extra_headers=kwargs.get("agent_extra_headers"))
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("method", ["message/send", "message/stream"])
+async def test_message_methods_preserve_numeric_zero_request_id(method: str):
+    mock_request = _make_request_mock(method, _HELLO_MESSAGE_PARAMS, request_id=0)
+    user_api_key_dict = UserAPIKeyAuth(api_key="sk-test", user_id="u1", team_id="t1")
+
+    captured = await _invoke_message_method(method, mock_request, user_api_key_dict)
+
+    assert captured.request_id == 0
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("method", ["message/send", "message/stream"])
+async def test_message_methods_forward_caller_identity_headers(method: str):
+    mock_request = _make_request_mock(method, _HELLO_MESSAGE_PARAMS)
+    user_api_key_dict = UserAPIKeyAuth(api_key="sk-test", user_id="user-abc", team_id="team-xyz")
+
+    captured = await _invoke_message_method(method, mock_request, user_api_key_dict)
+
+    forwarded_headers = captured.agent_extra_headers or {}
+    assert forwarded_headers.get("X-LiteLLM-User-Id") == "user-abc"
+    assert forwarded_headers.get("X-LiteLLM-Team-Id") == "team-xyz"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("method", ["message/send", "message/stream"])
+async def test_message_methods_caller_identity_headers_cannot_be_spoofed(method: str):
+    mock_request = _make_request_mock(method, _HELLO_MESSAGE_PARAMS)
+    mock_request.headers = {
+        "x-a2a-test-agent-x-litellm-user-id": "attacker-user",
+        "x-a2a-test-agent-x-litellm-team-id": "attacker-team",
+    }
+    user_api_key_dict = UserAPIKeyAuth(api_key="sk-test", user_id="real-user", team_id="real-team")
+
+    captured = await _invoke_message_method(method, mock_request, user_api_key_dict)
+
+    forwarded_headers = captured.agent_extra_headers or {}
+    assert (
+        forwarded_headers.get("X-LiteLLM-User-Id") == "real-user"
+    ), "authenticated user id must not be overridden by forwarded client headers"
+    assert (
+        forwarded_headers.get("X-LiteLLM-Team-Id") == "real-team"
+    ), "authenticated team id must not be overridden by forwarded client headers"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("method", ["message/send", "message/stream"])
+async def test_message_methods_forward_key_bound_identity_not_pre_call_rewrite(method: str):
+    from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
+
+    mock_request = _make_request_mock(method, _HELLO_MESSAGE_PARAMS)
+    mock_request.headers = {"X-OpenWebUI-User-Id": "header-mapped-user"}
+    user_api_key_dict = UserAPIKeyAuth(api_key="sk-test", user_id="key-user", team_id="key-team")
+    general_settings: Final = {
+        "user_header_mappings": [{"header_name": "X-OpenWebUI-User-Id", "litellm_user_role": "internal_user"}]
+    }
+
+    async def apply_user_header_mapping(data: dict[str, object], **kwargs: object) -> dict[str, object]:
+        LiteLLMProxyRequestSetup.add_internal_user_from_user_mapping(
+            general_settings, user_api_key_dict, dict(mock_request.headers)
+        )
+        return await _add_proxy_data(data, **kwargs)
+
+    captured = await _invoke_message_method(
+        method, mock_request, user_api_key_dict, add_litellm_data=apply_user_header_mapping
+    )
+
+    assert user_api_key_dict.user_id == "header-mapped-user", "precondition: pre-call rewrite ran"
+    forwarded_headers = captured.agent_extra_headers or {}
+    assert forwarded_headers.get("X-LiteLLM-User-Id") == "key-user"
+    assert forwarded_headers.get("X-LiteLLM-Team-Id") == "key-team"
 
 
 @pytest.mark.asyncio
diff --git a/tests/test_litellm/proxy/agent_endpoints/test_agent_headers.py b/tests/test_litellm/proxy/agent_endpoints/test_agent_headers.py
index 15864417489..e894f4ad69a 100644
--- a/tests/test_litellm/proxy/agent_endpoints/test_agent_headers.py
+++ b/tests/test_litellm/proxy/agent_endpoints/test_agent_headers.py
@@ -223,7 +223,7 @@ async def test_static_overrides_dynamic():
 
 @pytest.mark.asyncio
 async def test_no_headers():
-    """When no headers are configured, agent_extra_headers is None and behaviour is unchanged."""
+    """When no headers are configured, only the caller identity is forwarded."""
     mock_agent = _make_mock_agent()  # no static_headers or extra_headers
     mock_request = _make_mock_request()
 
@@ -231,7 +231,7 @@ async def test_no_headers():
 
     call_kwargs = mock_asend.call_args.kwargs
     headers = call_kwargs.get("agent_extra_headers")
-    assert headers is None
+    assert headers == {"X-LiteLLM-User-Id": "u1"}
 
 
 # ---------------------------------------------------------------------------
@@ -303,7 +303,7 @@ async def test_convention_unrelated_prefix_not_forwarded():
     mock_asend = await _invoke(mock_agent, mock_request, None)
 
     headers = mock_asend.call_args.kwargs.get("agent_extra_headers")
-    assert headers is None
+    assert headers == {"X-LiteLLM-User-Id": "u1"}
 
 
 # ---------------------------------------------------------------------------
@@ -393,7 +393,7 @@ async def test_non_databricks_agent_skips_oauth_resolution():
 
     mock_resolve.assert_not_called()
     headers = mock_asend.call_args.kwargs.get("agent_extra_headers")
-    assert headers == {"x-custom": "v"}
+    assert headers == {"x-custom": "v", "X-LiteLLM-User-Id": "u1"}
     assert "Authorization" not in headers
 
 
@@ -477,7 +477,7 @@ async def test_convention_header_blocked_by_case_variant_static():
 
     headers = mock_asend.call_args.kwargs.get("agent_extra_headers")
     assert headers is not None
-    assert headers == {"Authorization": "Bearer admin-token"}
+    assert headers == {"Authorization": "Bearer admin-token", "X-LiteLLM-User-Id": "u1"}
     assert "authorization" not in headers
 
 

From 90731576e3e970543633fd2bfd2b032a89a42e48 Mon Sep 17 00:00:00 2001
From: yucheng-berri <yucheng@berri.ai>
Date: Tue, 8 Sep 2026 15:58:58 -0700
Subject: [PATCH 85/85] feat(team): let a team admin manage their own team's
 logging callbacks (#37667)

* feat(team): let a team admin manage their own team's logging callbacks

The team callback endpoints already authorize correctly: POST, GET and DELETE
each call _verify_team_access, which admits a proxy admin, an org admin for the
team, or an admin of that team, and 403s everyone else. The route-permission
layer never let a team admin reach them, so it answered 401 naming proxy admin
and the handler's own check was dead code for the caller it was written for.

Adding the two paths to self_managed_routes is how every other team-admin route
works: /team/member_add, /team/member_delete, /team/member_update and
/team/permissions_update all sit in that list and scope per team inside the
handler. The entries use the :path converter the routes are registered with, so
a team id containing a slash resolves the same way at the gate as at the router.

Because any authenticated caller now reaches these handlers, an unknown team had
to stop being distinguishable from one the caller may not manage. All three
handlers looked the team up and raised a distinct 'does not exist' before the
access check, which would have let any valid key probe for team ids. That branch
now returns the same 403 body _verify_team_access raises, and keeps the
diagnosable error for a proxy admin.

disable_logging stays out of the grant. That is a scope decision rather than a
security boundary, since a team admin holding DELETE can clear callbacks one at
a time; it differs only in also clearing the deprecated callback_settings shape.

* fix(team): reach the callback routes for a team id containing a colon

The route gate expands {team_id:path} to "[^:]+" so a colon-suffixed provider
route is not swallowed, which means the two entries added here matched a team
id with a slash but not one with a colon, while the router accepts both.
team_id is a free-form string, so a team whose id contains a colon kept the
old proxy-admin-only denial and its admin could not manage its own callbacks.
List both spellings rather than relaxing the shared matcher, which every
":path" route depends on. The comment claimed the two matchers agree; they do
not, so it now says what each placeholder actually accepts.

* fix(auth): match a :path placeholder the way the router's converter does

A team id may carry a slash, a colon, or both. The gate expanded {x:path}
to "[^:]+", so an id with a colon in it matched no self_managed_routes
entry and its team admin got the proxy-admin-only denial on a route the
router had already resolved for them. Listing a second {x} spelling covered
a colon or a slash but never both.

Expand {x:path} to ".+" instead, except when the template puts a ":"
literal of its own after the placeholder, which is where the narrower form
was earning its keep: the Google routes end in ":generateContent" and
friends, and there the value has to stop before that suffix rather than
swallow it and match a different verb.

That lets self_managed_routes drop back to the two :path spellings the
router itself mounts.

* test(auth): pin that the callback grant reaches no neighbouring team route

The grant is two templates ending in the callback suffix, and the
placeholder now takes slashes and colons. Every other route under
/team/{team_id} registers an ordinary single-segment placeholder, so no
URL the router sends to one of them can end in the callback suffix.

Pin that, so adding a path-converter route beside these fails here
rather than by handing a caller a handler the grant never covered.

* fix(team): make one entry own a credential family end to end

Every stored entry's callback_vars are flattened into one dict before a
request reads them, and that dict is what the exporter authenticates and
addresses with. So an entry naming only a destination is enough to
redirect a credential written somewhere else: a host on a second entry
pairs with the key pair from the first, and the request carries that key
pair to the new host. A team admin cannot read the team's masked Langfuse
secret, but could add such an entry and receive it.

Reject, for writers who are not proxy admins, an entry using a credential
family another entry already holds. Family rather than callback name,
because langfuse and langfuse_otel configure one Langfuse project and
would otherwise redirect each other, and because a destination like
dd_agent_host that no integration registry lists still pairs with the
Datadog credentials beside it.

A proxy admin already holds every credential the proxy has, so the rule
would buy nothing there and would break configs that predate it. A team
admin who does want to move a family deletes the entry holding it first,
which reveals nothing.

* fix(team): let one integration cover both callback events

The family rule compared variable names only, so a team admin who registered
an integration for the success event could not register the same integration,
with the same values, for the failure event.

Compare the values as well: repeating what the owning entry already stores
flattens to the same dict, so there is nothing to redirect. The stored side is
decrypted first, because the credentials are encrypted at rest and ciphertext
never equals the plaintext coming in.

* fix(team): compare the family's values, not its variable names

Comparing per variable rejected a credential written under its other spelling:
langfuse_secret and langfuse_secret_key are one key, so repeating the stored
secret under the other name read as a new value.

Ask instead whether the value is one the owning entries already carry. A
destination the caller controls is by definition not, so the redirect stays
closed, and no alias table has to stay complete for that to hold.

* fix(team): pin the family's configured variables as well as its values

Asking only whether a value is one the family holds let a held variable be
given another of the family's values, so the exporter would address or
authenticate with it.

Keep the value membership rule for a variable the family does not configure
yet, which is what lets one credential go in under its other spelling, and
require a variable it does configure to keep the value it has. Between them no
value the caller chose can enter the family.

* fix(auth): keep a newline in a :path value visible to the route gate

"." stops at a newline and the router's path converter does not, so a %0A
anywhere in a :path segment left the route unmatched here while still reaching
the handler. Every list built on this matcher inherited that: on a proxy with
DISABLE_ADMIN_ENDPOINTS set, DELETE /v1/mcp/server/abc%0Adef reached the MCP
handler instead of the 403 the same request gets without the %0A.

Match with a class that spans newlines.
---
 litellm/proxy/_types.py                       |   8 +
 litellm/proxy/auth/route_checks.py            |  20 +-
 .../callback_config_validation.py             |  85 ++++++++
 .../team_callback_endpoints.py                |  56 +++++-
 .../proxy/auth/test_route_checks.py           | 188 ++++++++++++++++++
 .../test_team_callback_endpoints.py           | 116 +++++++++++
 6 files changed, 459 insertions(+), 14 deletions(-)

diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py
index 50b87264f3f..1c8608305a2 100644
--- a/litellm/proxy/_types.py
+++ b/litellm/proxy/_types.py
@@ -835,6 +835,14 @@ class LiteLLMRoutes(enum.Enum):
         "/team/daily/activity/aggregated",
         "/team/spend/by_user",
         "/team/{team_id}/members/me",
+        # POST/GET the team's logging callbacks, and DELETE one of them. Every
+        # handler calls _verify_team_access, which admits only a proxy admin, an
+        # org admin for the team, or an admin of this team.
+        #
+        # team_id is a free-form string, so it spells these with the same path
+        # converter the router uses; the gate matches that converter.
+        "/team/{team_id:path}/callback",
+        "/team/{team_id:path}/callback/{callback_name}",
         "/model/new",
         "/model/update",
         "/model/delete",
diff --git a/litellm/proxy/auth/route_checks.py b/litellm/proxy/auth/route_checks.py
index 4dba2497bb9..953e3cf3e88 100644
--- a/litellm/proxy/auth/route_checks.py
+++ b/litellm/proxy/auth/route_checks.py
@@ -497,10 +497,22 @@ class RouteChecks:
 
         def _placeholder_to_regex(match: re.Match) -> str:
             placeholder: Final = match.group(0).strip("{}")
-            if placeholder.endswith(":path"):
-                # allow "/" in the placeholder value, but don't eat the route suffix after ":"
-                return r"[^:]+"
-            return r"[^/]+"
+            if not placeholder.endswith(":path"):
+                return r"[^/]+"
+            # A ":path" placeholder takes whatever the router's own path
+            # converter takes, slashes and colons alike, so an id spelled with
+            # either (or both) still matches the template it was mounted under.
+            #
+            # Unless the template puts a ":" literal of its own after the
+            # placeholder: the Google routes end in ":generateContent" and
+            # friends, and there the value has to stop before that suffix
+            # rather than swallow it and match a different verb.
+            #
+            # "[\s\S]" rather than ".", because "." stops at a newline and the
+            # path converter does not: a %0A anywhere in the value would leave
+            # the route unmatched here while still reaching the handler, which
+            # turns this gate into a bypass for the lists built on it.
+            return r"[^:]+" if ":" in match.string[match.end() :] else r"[\s\S]+"
 
         pattern = re.sub(r"\{[^}]+\}", _placeholder_to_regex, pattern)
         # Anchor the pattern to match the entire string
diff --git a/litellm/proxy/common_utils/callback_config_validation.py b/litellm/proxy/common_utils/callback_config_validation.py
index 7ee3bd8d829..c9d97068313 100644
--- a/litellm/proxy/common_utils/callback_config_validation.py
+++ b/litellm/proxy/common_utils/callback_config_validation.py
@@ -44,6 +44,91 @@ def _langfuse_environment_error(callback_vars: Mapping[str, str]) -> str | None:
     return None
 
 
+# Which credential family a dynamic variable belongs to. The families are the
+# integrations that share one account: every langfuse_* variable configures the
+# same Langfuse project whether it rides the classic callback or the OTel one,
+# and every dd_* variable configures the same Datadog account.
+_VAR_FAMILIES: Final[Mapping[str, str]] = MappingProxyType(
+    {
+        "arize_": "Arize",
+        "dd_": "Datadog",
+        "gcs_": "GCS",
+        "humanloop_": "Humanloop",
+        "langfuse_": "Langfuse",
+        "langsmith_": "LangSmith",
+        "newrelic_": "New Relic",
+        "posthog_": "PostHog",
+        "wandb_": "Weights & Biases",
+        "weave_": "Weights & Biases",
+    }
+)
+
+
+def _family_of(var: str) -> str | None:
+    """The credential family ``var`` configures, or ``None`` if it configures none.
+
+    ``turn_off_message_logging`` and friends belong to no backend, so they carry
+    no credentials anyone could redirect.
+    """
+    return next((family for prefix, family in _VAR_FAMILIES.items() if var.startswith(prefix)), None)
+
+
+def cross_entry_family_error(
+    callback_vars: Mapping[str, str] | None,
+    stored_vars_by_entry: Sequence[Mapping[str, str]],
+) -> str | None:
+    """Reject an entry that changes what a family another entry holds resolves to.
+
+    Every stored entry's variables are flattened into one dict before a request
+    reads them, and the flattened dict is what the exporter authenticates and
+    addresses with. So an entry naming only a destination is enough to redirect
+    credentials that were written somewhere else: a host on a second entry pairs
+    with the key from the first, and the request carries that key to the new
+    host.
+
+    Two rules together keep the flattened dict out of the caller's hands. A
+    variable the family already configures has to keep the value it has, so
+    nothing already in use can be moved. A variable the family does not yet
+    configure may only carry a value the family already holds, which is what lets
+    the same credential go in under its other spelling (``langfuse_secret`` and
+    ``langfuse_secret_key`` are one key) without anything here having to list the
+    spellings. Between them, no value the caller chose can enter the family, and
+    repeating the family as it stands is still allowed -- that is how one
+    integration gets registered for both the success and the failure event.
+
+    A team admin who does want to move a family deletes the entry holding it
+    first, which reveals nothing.
+
+    Only the writers this endpoint newly admits are held to this, because a proxy
+    admin already holds every credential the proxy has.
+
+    ``stored_vars_by_entry`` has to arrive decrypted; the credential values are
+    encrypted at rest and ciphertext never equals the plaintext coming in.
+    """
+    if not callback_vars:
+        return None
+    stored_by_var: Final = {
+        var: value for entry in stored_vars_by_entry for var, value in entry.items() if _family_of(var) is not None
+    }
+    family_values: Final = frozenset(
+        (family, value)
+        for entry in stored_vars_by_entry
+        for var, value in entry.items()
+        if (family := _family_of(var)) is not None
+    )
+    held_families: Final = frozenset(family for family, _ in family_values)
+    return next(
+        (
+            f"{family} is already configured by another callback entry on this team. "
+            f"Remove that entry before setting {var} here."
+            for var, value, family in ((v, callback_vars[v], _family_of(v)) for v in callback_vars)
+            if family in held_families
+            and (stored_by_var[var] != value if var in stored_by_var else (family, value) not in family_values)
+        ),
+        None,
+    )
+
+
 def logging_metadata_config_error(metadata: Mapping[str, object] | None) -> str | None:
     """Validate every ``logging`` entry of a team/key metadata payload."""
     if not metadata:
diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py
index fe658a13c24..1932e89717b 100644
--- a/litellm/proxy/management_endpoints/team_callback_endpoints.py
+++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py
@@ -20,6 +20,7 @@ from litellm.proxy._types import (
     LiteLLM_AuditLogs,
     LiteLLM_TeamTable,
     LitellmTableNames,
+    LitellmUserRoles,
     ProxyErrorTypes,
     ProxyException,
     TeamCallbackDeleteResponse,
@@ -28,7 +29,10 @@ from litellm.proxy._types import (
     UserAPIKeyAuth,
 )
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
-from litellm.proxy.common_utils.callback_config_validation import callback_config_error
+from litellm.proxy.common_utils.callback_config_validation import (
+    callback_config_error,
+    cross_entry_family_error,
+)
 from litellm.proxy.common_utils.callback_utils import (
     _CALLBACK_VAR_ENCRYPTED_PREFIX,
     decrypt_callback_vars,
@@ -230,6 +234,22 @@ def _callback_error(status_code: int, message: str) -> HTTPException:
     )
 
 
+def _unknown_team_error(team_id: str, user_api_key_dict: UserAPIKeyAuth, status_code: int) -> HTTPException:
+    """Report an unknown team without telling an unauthorized caller that it is unknown.
+
+    These routes are reachable by any authenticated caller so that a team admin can
+    get as far as _verify_team_access. A distinct "does not exist" would therefore let
+    any valid key probe which team ids exist, so a caller who could not have managed
+    the team either way gets the same 403 body _verify_team_access raises.
+    """
+    if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN:
+        return _callback_error(status_code, f"Team id = {team_id} does not exist.")
+    return HTTPException(
+        status_code=status.HTTP_403_FORBIDDEN,
+        detail="You do not have access to this team",
+    )
+
+
 @router.post(
     "/team/{team_id:path}/callback",
     tags=["team management"],
@@ -304,10 +324,7 @@ async def add_team_callbacks(
         # Check if team_id exists already
         _existing_team = await prisma_client.get_data(team_id=team_id, table_name="team", query_type="find_unique")
         if _existing_team is None:
-            raise HTTPException(
-                status_code=400,
-                detail={"error": f"Team id = {team_id} does not exist. Please use a different team id."},
-            )
+            raise _unknown_team_error(team_id, user_api_key_dict, status.HTTP_400_BAD_REQUEST)
 
         # IDOR guard: only proxy admins / org admins / team admins of THIS
         # team may write callback credentials. Without this, any
@@ -326,6 +343,28 @@ async def add_team_callbacks(
         if team_callback_settings is None or not isinstance(team_callback_settings, list):
             team_callback_settings = []
 
+        # One entry has to own a credential family end to end. The entries are
+        # flattened into one dict before a request reads them, so an entry
+        # naming only a destination would pair with a key written on another
+        # entry and carry it to that destination -- a key a team admin can read
+        # back nowhere. Repeating a value the owning entry already stores is
+        # fine, which is how one integration covers both events. Proxy admins
+        # are exempt: they already hold every credential the proxy has.
+        if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
+            # Decrypted, because the check compares the incoming values against
+            # the stored ones and the credentials are encrypted at rest.
+            decrypted_logging: Final = decrypt_callback_vars(team_metadata).get("logging")
+            stored_entries: Final = decrypted_logging if isinstance(decrypted_logging, list) else ()
+            stored_entry_vars: Final = [  # mutable-ok: read-only input to the check, never stored
+                entry.get("callback_vars") or {} for entry in stored_entries
+            ]
+            family_error: Final = cross_entry_family_error(data.callback_vars, stored_entry_vars)
+            if family_error is not None:
+                raise HTTPException(
+                    status_code=status.HTTP_400_BAD_REQUEST,
+                    detail=family_error,
+                )
+
         ## check if it already exists, for the same callback event
         for callback in team_callback_settings:
             if (
@@ -452,7 +491,7 @@ async def delete_team_callback(
             team_id=team_id, table_name="team", query_type="find_unique"
         )
         if _existing_team is None:
-            raise _callback_error(404, f"Team id = {team_id} does not exist.")
+            raise _unknown_team_error(team_id, user_api_key_dict, status.HTTP_404_NOT_FOUND)
 
         # IDOR guard: only proxy admins / org admins / team admins of THIS team may
         # deregister its callbacks, otherwise any authenticated key holder could
@@ -726,10 +765,7 @@ async def get_team_callbacks(
         # Check if team_id exists
         _existing_team = await prisma_client.get_data(team_id=team_id, table_name="team", query_type="find_unique")
         if _existing_team is None:
-            raise HTTPException(
-                status_code=404,
-                detail={"error": f"Team id = {team_id} does not exist."},
-            )
+            raise _unknown_team_error(team_id, user_api_key_dict, status.HTTP_404_NOT_FOUND)
 
         # IDOR guard: callback metadata holds third-party API credentials
         # (Langfuse / Langsmith / GCS). Only proxy admins / org admins /
diff --git a/tests/test_litellm/proxy/auth/test_route_checks.py b/tests/test_litellm/proxy/auth/test_route_checks.py
index 5b15d4a7d5e..0fd19f518d9 100644
--- a/tests/test_litellm/proxy/auth/test_route_checks.py
+++ b/tests/test_litellm/proxy/auth/test_route_checks.py
@@ -3638,3 +3638,191 @@ def test_agent_registry_route_gate_open_to_non_admin_roles(user_role, method, ro
         valid_token=valid_token,
         request_data={},
     )
+TEAM_CALLBACK_ROUTES = (
+    "/team/06bda574-5ca9-43d3-beb8-3b23c2f17112/callback",
+    "/team/06bda574-5ca9-43d3-beb8-3b23c2f17112/callback/langfuse",
+    # the routes register team_id with the :path converter, so a team id may
+    # contain a slash
+    "/team/tenant/06bda574/callback",
+    "/team/tenant/06bda574/callback/langfuse",
+    # team_id is a free-form string, so it may also contain a colon
+    "/team/tenant:06bda574/callback",
+    "/team/tenant:06bda574/callback/langfuse",
+    # or both, which is the shape neither a "[^:]+" nor a "[^/]+" expansion
+    # of the placeholder reaches on its own
+    "/team/tenant:acme/prod/callback",
+    "/team/tenant:acme/prod/callback/langfuse",
+)
+
+
+def _gate(route, role) -> str:
+    """Drive the real route gate for a non-proxy-admin caller.
+
+    Reports "allowed" when the gate lets the request through to its handler, and
+    the denial message otherwise, so a caller asserts the verdict as a value
+    instead of on whether an exception escaped.
+    """
+    user_obj = LiteLLM_UserTable(
+        user_id="team_admin_user",
+        user_email="team-admin@example.com",
+        user_role=role,
+    )
+    request = MagicMock(spec=Request)
+    request.query_params = {}
+    try:
+        RouteChecks.non_proxy_admin_allowed_routes_check(
+            user_obj=user_obj,
+            _user_role=role,
+            route=route,
+            request=request,
+            valid_token=UserAPIKeyAuth(user_id="team_admin_user", user_role=role),
+            request_data={},
+        )
+    except Exception as exc:
+        return f"denied: {exc}"
+    return "allowed"
+
+
+def test_team_callback_routes_are_self_managed():
+    """The grant has to come from self_managed_routes specifically.
+
+    That list is the one whose entries carry no role predicate, so the handler
+    decides. Granting the same paths through internal_user_routes instead would
+    look identical for an internal_user while silently denying the org admins and
+    view-only roles that list does not cover.
+    """
+    for template in (
+        "/team/{team_id:path}/callback",
+        "/team/{team_id:path}/callback/{callback_name}",
+    ):
+        assert template in LiteLLMRoutes.self_managed_routes.value
+
+
+@pytest.mark.parametrize("route", TEAM_CALLBACK_ROUTES)
+@pytest.mark.parametrize(
+    "role",
+    [
+        LitellmUserRoles.INTERNAL_USER.value,
+        LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value,
+        LitellmUserRoles.ORG_ADMIN.value,
+    ],
+)
+def test_team_callback_routes_reach_their_handler_for_non_admins(route, role):
+    """A team admin manages their own team's logging callbacks, so the route gate
+    must let a non-proxy-admin through to the handler.
+
+    The handler is what authorizes: every team callback endpoint calls
+    _verify_team_access, which admits only a proxy admin, an org admin for the
+    team, or an admin of that team, and 403s everyone else. Before this, the gate
+    rejected the team admin with a 401 naming proxy admin, so the handler's own
+    check was unreachable for them.
+    """
+    assert _gate(route, role) == "allowed"
+
+
+@pytest.mark.parametrize(
+    "pattern, route, matches",
+    [
+        # a :path placeholder takes what the router's path converter takes
+        ("/team/{team_id:path}/callback", "/team/plain/callback", True),
+        ("/team/{team_id:path}/callback", "/team/tenant/acme/callback", True),
+        ("/team/{team_id:path}/callback", "/team/tenant:acme/callback", True),
+        ("/team/{team_id:path}/callback", "/team/tenant:acme/prod/callback", True),
+        # and still has to reach the template's own suffix
+        ("/team/{team_id:path}/callback", "/team/tenant:acme/disable_logging", False),
+        # a template with a ":" literal after the placeholder keeps the suffix
+        (
+            "/v1beta/models/{model_name:path}:generateContent",
+            "/v1beta/models/gemini-2.5-flash:generateContent",
+            True,
+        ),
+        (
+            "/v1beta/models/{model_name:path}:generateContent",
+            "/v1beta/models/publishers/google/gemini-2.5-flash:generateContent",
+            True,
+        ),
+        # the value must not swallow that suffix and match a different verb
+        (
+            "/v1beta/models/{model_name:path}:generateContent",
+            "/v1beta/models/gemini-2.5-flash:countTokens",
+            False,
+        ),
+        # a %0A in the value reaches the handler through the path converter, so
+        # the gate has to see it too or DISABLE_ADMIN_ENDPOINTS is bypassable
+        ("/v1/mcp/server/{path:path}", "/v1/mcp/server/abc\ndef", True),
+        ("/team/{team_id:path}/callback", "/team/ten\nant/callback", True),
+        ("/v1beta/models/{model_name:path}:generateContent", "/v1beta/models/gem\nini:generateContent", True),
+        # an ordinary placeholder stays one segment
+        ("/team/{team_id}/members/me", "/team/abc/members/me", True),
+        ("/team/{team_id}/members/me", "/team/tenant/abc/members/me", False),
+        ("/team/{team_id}/members/me", "/team/ab\nc/members/me", True),
+    ],
+)
+def test_path_placeholder_matches_what_the_router_accepts(pattern, route, matches):
+    """The gate's placeholder expansion has to agree with the router's.
+
+    A team id may carry a slash, a colon, or both, and the router mounted these
+    paths with the same :path converter, so an id the router routes must not be
+    an id the gate fails to recognize. The one narrowing that stays is a template
+    whose own suffix begins with a colon: there the value stops before it, or
+    ":generateContent" would also match a ":countTokens" request.
+    """
+    assert RouteChecks._route_matches_pattern(route=route, pattern=pattern) is matches
+
+
+# Every other route the proxy mounts under /team/{team_id}, spelled the way it
+# is registered. None of them takes a path converter, so none can be reached by
+# a URL that ends in the callback suffix.
+PROTECTED_TEAM_ROUTES = (
+    "/team/06bda574-5ca9-43d3-beb8-3b23c2f17112",
+    "/team/06bda574-5ca9-43d3-beb8-3b23c2f17112/disable_logging",
+    "/team/06bda574-5ca9-43d3-beb8-3b23c2f17112/members/me",
+    "/team/06bda574-5ca9-43d3-beb8-3b23c2f17112/member/u-1/reset_spend",
+    # the same routes with the callback suffix spliced in, which is the shape a
+    # caller would craft to make a protected route look self-managed
+    "/team/06bda574/callback/disable_logging/x",
+    "/team/06bda574/callback/member/u-1/reset_spend",
+    "/team/06bda574/callback/members/me",
+)
+
+
+@pytest.mark.parametrize("route", PROTECTED_TEAM_ROUTES)
+def test_the_callback_grant_does_not_reach_another_team_route(route):
+    """Widening the callback templates must not hand out any neighbouring route.
+
+    The grant is two templates ending in the callback suffix. Every other team
+    route registers an ordinary single-segment placeholder, so no URL the router
+    sends to one of them can end in "/callback" or "/callback/<name>" -- and the
+    gate must agree, or a crafted team id would carry a caller into a handler
+    the grant never covered.
+    """
+    for template in (
+        "/team/{team_id:path}/callback",
+        "/team/{team_id:path}/callback/{callback_name}",
+    ):
+        assert RouteChecks._route_matches_pattern(route=route, pattern=template) is False
+
+
+def test_team_disable_logging_stays_proxy_admin_only():
+    """disable_logging was left out of the grant, so it must still be rejected at
+    the gate. It is the one team callback route a team admin cannot reach."""
+    verdict = _gate(
+        "/team/06bda574-5ca9-43d3-beb8-3b23c2f17112/disable_logging",
+        LitellmUserRoles.INTERNAL_USER.value,
+    )
+
+    assert "Only proxy admin" in verdict
+    assert "disable_logging" in verdict
+
+
+@pytest.mark.parametrize(
+    "route",
+    [
+        "/team/06bda574-5ca9-43d3-beb8-3b23c2f17112",
+        "/team/update",
+        "/team/06bda574-5ca9-43d3-beb8-3b23c2f17112/model/add",
+    ],
+)
+def test_neighbouring_team_routes_stay_closed(route):
+    """The grant is the callback paths and nothing else on the team namespace."""
+    assert "Only proxy admin" in _gate(route, LitellmUserRoles.INTERNAL_USER.value)
diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py
index 08e931e6405..bdc12dad4bc 100644
--- a/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py
+++ b/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py
@@ -19,6 +19,7 @@ from litellm.proxy._types import (
     LitellmUserRoles,
     UserAPIKeyAuth,
 )
+from litellm.proxy.common_utils.callback_config_validation import cross_entry_family_error
 from litellm.proxy.management_endpoints.team_callback_endpoints import (
     add_team_callbacks,
     delete_team_callback,
@@ -1443,3 +1444,118 @@ async def test_delete_team_callback_route_accepts_team_ids_containing_slashes():
     assert response.json()["data"]["success_callbacks"] == ["langsmith"]
     written = json.loads(mock_prisma.db.litellm_teamtable.update.await_args.kwargs["data"]["metadata"])
     assert [entry["callback_name"] for entry in written["logging"]] == ["langsmith"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "call_handler",
+    [
+        lambda caller: add_team_callbacks(
+            data=AddTeamCallback(
+                callback_name="langfuse",
+                callback_type="success",
+                callback_vars={"langfuse_public_key": "pk", "langfuse_secret_key": "sk"},
+            ),
+            http_request=Mock(spec=Request),
+            team_id="team-does-not-exist",
+            user_api_key_dict=caller,
+        ),
+        lambda caller: get_team_callbacks(
+            http_request=Mock(spec=Request),
+            team_id="team-does-not-exist",
+            user_api_key_dict=caller,
+        ),
+        lambda caller: delete_team_callback(
+            http_request=Mock(spec=Request),
+            team_id="team-does-not-exist",
+            callback_name="langfuse",
+            user_api_key_dict=caller,
+        ),
+    ],
+    ids=["add", "get", "delete"],
+)
+async def test_unknown_team_is_indistinguishable_from_no_access(call_handler, unauthorized_caller):
+    """An unauthorized caller must not learn whether a team id exists.
+
+    These routes are reachable by any authenticated caller so a team admin can get
+    as far as the access check, so a distinct "does not exist" would turn them into
+    a probe for valid team ids. The unknown-team response has to match the
+    no-access one exactly, status and body.
+    """
+    with patch("litellm.proxy.proxy_server.prisma_client") as mock_client:  # test-quality-ok: the handler imports prisma_client from proxy_server at call time, so there is no seam to inject through
+        mock_client.get_data = AsyncMock(return_value=None)
+        with pytest.raises(HTTPException) as unknown_team:
+            await call_handler(unauthorized_caller)
+
+    with patch("litellm.proxy.proxy_server.prisma_client") as mock_client:  # test-quality-ok: the handler imports prisma_client from proxy_server at call time, so there is no seam to inject through
+        mock_client.get_data = AsyncMock(return_value=_team_row())
+        mock_client.db.litellm_teamtable.update = AsyncMock()
+        with patch(  # test-quality-ok: _verify_team_access calls this module-level helper directly, so there is no seam to inject through
+            "litellm.proxy.management_endpoints.team_endpoints._is_user_org_admin_for_team",
+            new_callable=AsyncMock,
+            return_value=False,
+        ):
+            with pytest.raises(HTTPException) as no_access:
+                await call_handler(unauthorized_caller)
+
+    assert unknown_team.value.status_code == no_access.value.status_code == 403
+    assert unknown_team.value.detail == no_access.value.detail
+    assert "does not exist" not in str(unknown_team.value.detail)
+
+
+@pytest.mark.asyncio
+async def test_proxy_admin_still_told_the_team_is_unknown():
+    """The masking is only for callers who could not have managed the team; a proxy
+    admin keeps the diagnosable error."""
+    admin = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin", api_key="sk-admin")
+    with patch("litellm.proxy.proxy_server.prisma_client") as mock_client:  # test-quality-ok: the handler imports prisma_client from proxy_server at call time, so there is no seam to inject through
+        mock_client.get_data = AsyncMock(return_value=None)
+        with pytest.raises(HTTPException) as exc:
+            await get_team_callbacks(
+                http_request=Mock(spec=Request),
+                team_id="team-does-not-exist",
+                user_api_key_dict=admin,
+            )
+
+    assert exc.value.status_code == 404
+    assert "does not exist" in str(exc.value.detail)
+
+
+@pytest.mark.parametrize(
+    "new_vars, stored, rejected",
+    [
+        # the redirect, in every carrier a caller could pick: an entry naming
+        # only a host, pairing with a key pair written on another entry
+        ({"langfuse_host": "http://attacker.invalid"}, [{"langfuse_public_key": "pk", "langfuse_secret_key": "sk"}], True),
+        # the sibling carrier -- langfuse and langfuse_otel are one account
+        ({"langfuse_host": "http://attacker.invalid"}, [{"langfuse_host": "https://us.cloud.langfuse.com", "langfuse_secret_key": "sk"}], True),
+        # a destination variable no integration registry lists
+        ({"dd_agent_host": "attacker.invalid"}, [{"dd_api_key": "k", "dd_site": "us5.datadoghq.com"}], True),
+        # one entry owning its family end to end is the feature
+        ({"langfuse_host": "https://eu.cloud.langfuse.com", "langfuse_public_key": "pk", "langfuse_secret_key": "sk"}, [], False),
+        # a different family alongside an existing one stays fine
+        ({"gcs_bucket_name": "bucket"}, [{"langfuse_public_key": "pk", "langfuse_secret_key": "sk"}], False),
+        ({"langsmith_api_key": "k"}, [{"dd_api_key": "k"}], False),
+        # variables that configure no backend carry nothing to redirect
+        ({"turn_off_message_logging": "true"}, [{"langfuse_secret_key": "sk"}], False),
+        # the same integration registered for a second event: identical values
+        # flatten to the identical dict, so there is nothing to redirect
+        ({"langfuse_host": "https://us.cloud.langfuse.com", "langfuse_public_key": "pk", "langfuse_secret_key": "sk"}, [{"langfuse_host": "https://us.cloud.langfuse.com", "langfuse_public_key": "pk", "langfuse_secret_key": "sk"}], False),
+        # the same credential under its other spelling is the same credential
+        ({"langfuse_secret": "sk"}, [{"langfuse_public_key": "pk", "langfuse_secret_key": "sk"}], False),
+        # a value the family already holds cannot be moved into another of its
+        # variables either; the exporter would address or authenticate with it
+        ({"langfuse_host": "pk"}, [{"langfuse_host": "https://us.cloud.langfuse.com", "langfuse_public_key": "pk"}], True),
+        # the same shape with one value moved is the redirect again
+        ({"langfuse_host": "http://attacker.invalid", "langfuse_public_key": "pk", "langfuse_secret_key": "sk"}, [{"langfuse_host": "https://us.cloud.langfuse.com", "langfuse_public_key": "pk", "langfuse_secret_key": "sk"}], True),
+    ],
+)
+def test_one_entry_owns_a_credential_family(new_vars, stored, rejected):
+    """A team admin must not be able to redirect a credential they cannot read.
+
+    The stored entries are flattened into one dict before a request reads them,
+    so an entry naming only a destination pairs with a key written elsewhere and
+    carries it to that destination.
+    """
+    error = cross_entry_family_error(new_vars, stored)
+    assert (error is not None) is rejected