Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_lite_up_down

# Conflicts:
#	uv.lock

.github/CODEOWNERS

@@ -1,2 +1,3 @@
 /ui/ @yuneng-jiang @ryan-crabbe-berri
 /litellm/proxy/_experimental/out/ @yuneng-jiang @ryan-crabbe-berri
+/ui/litellm-dashboard/src/lib/http/schema.d.ts

.github/actions/setup-uv-with-retries/action.yml

@@ -0,0 +1,47 @@
+name: "Set up uv with retries"
+description: >-
+  Install uv via astral-sh/setup-uv, retrying on transient failures. Even with
+  an exact pinned version, the action resolves the artifact URL by fetching
+  https://raw.githubusercontent.com/astral-sh/versions/main/v1/uv.ndjson in a
+  single request with no retry, timeout, or fallback, so one connection-level
+  network error ("fetch failed") fails the whole job before any test runs.
+  Retrying the full step covers the manifest fetch and the binary download.
+
+inputs:
+  version:
+    description: "uv version to install"
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Set up uv (attempt 1)
+      id: attempt-1
+      continue-on-error: true
+      uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+      with:
+        version: ${{ inputs.version }}
+
+    - name: Wait before attempt 2
+      if: steps.attempt-1.outcome == 'failure'
+      shell: bash
+      run: sleep 15
+
+    - name: Set up uv (attempt 2)
+      id: attempt-2
+      if: steps.attempt-1.outcome == 'failure'
+      continue-on-error: true
+      uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+      with:
+        version: ${{ inputs.version }}
+
+    - name: Wait before attempt 3
+      if: steps.attempt-2.outcome == 'failure'
+      shell: bash
+      run: sleep 30
+
+    - name: Set up uv (attempt 3)
+      if: steps.attempt-2.outcome == 'failure'
+      uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+      with:
+        version: ${{ inputs.version }}

.github/workflows/_test-unit-base.yml

@@ -63,7 +63,7 @@ jobs:
           python-version: "3.12"
 
       - name: Set up uv
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        uses: ./.github/actions/setup-uv-with-retries
         with:
           version: "0.10.9"
 

.github/workflows/auto_update_price_and_context_window.yml

@@ -18,7 +18,7 @@ jobs:
         with:
           persist-credentials: false
       - name: Set up uv
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        uses: ./.github/actions/setup-uv-with-retries
         with:
           version: "0.10.9"
       - name: Update JSON Data

.github/workflows/check-ui-api-types.yml

@@ -31,7 +31,7 @@ jobs:
           python-version: "3.12"
 
       - name: Set up uv
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        uses: ./.github/actions/setup-uv-with-retries
         with:
           version: "0.10.9"
 

.github/workflows/codspeed.yml

@@ -37,7 +37,7 @@ jobs:
           python-version: "3.12"
 
       - name: Set up uv
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        uses: ./.github/actions/setup-uv-with-retries
         with:
           version: "0.10.9"
 

.github/workflows/mutation-test.yml

@@ -39,7 +39,7 @@ jobs:
           python-version: "3.12"
 
       - name: Set up uv
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        uses: ./.github/actions/setup-uv-with-retries
         with:
           version: "0.10.9"
 

.github/workflows/oss_daily_guardrails.yml

@@ -35,7 +35,7 @@ jobs:
           python-version: "3.12"
 
       - name: Set up uv
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        uses: ./.github/actions/setup-uv-with-retries
         with:
           version: "0.10.9"
 

.github/workflows/test-code-quality.yml

@@ -38,7 +38,7 @@ jobs:
           python-version: "3.12"
 
       - name: Set up uv
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        uses: ./.github/actions/setup-uv-with-retries
         with:
           version: "0.10.9"
 

.github/workflows/test-linting.yml

@@ -33,7 +33,7 @@ jobs:
           python-version: "3.12"
 
       - name: Set up uv
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        uses: ./.github/actions/setup-uv-with-retries
         with:
           version: "0.10.9"
 
@@ -172,7 +172,7 @@ jobs:
           python-version: "3.12"
 
       - name: Set up uv
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        uses: ./.github/actions/setup-uv-with-retries
         with:
           version: "0.10.9"
 

.github/workflows/test-mcp.yml

@@ -32,7 +32,7 @@ jobs:
           python-version: "3.12"
 
       - name: Set up uv
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        uses: ./.github/actions/setup-uv-with-retries
         with:
           version: "0.10.9"
 

.github/workflows/test-semgrep.yml

@@ -31,7 +31,7 @@ jobs:
           python-version: "3.12"
 
       - name: Set up uv
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        uses: ./.github/actions/setup-uv-with-retries
         with:
           version: "0.10.9"
 

.github/workflows/test-terraform-provider.yml

@@ -74,7 +74,7 @@ jobs:
           python-version: "3.12"
 
       - name: Set up uv
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        uses: ./.github/actions/setup-uv-with-retries
         with:
           version: "0.10.9"
 

.github/workflows/test-unit-documentation.yml

@@ -42,7 +42,7 @@ jobs:
           python-version: "3.12"
 
       - name: Set up uv
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        uses: ./.github/actions/setup-uv-with-retries
         with:
           version: "0.10.9"
 

.github/workflows/test-unit-proxy-legacy.yml

@@ -59,7 +59,7 @@ jobs:
           python-version: "3.12"
 
       - name: Set up uv
-        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        uses: ./.github/actions/setup-uv-with-retries
         with:
           version: "0.10.9"
 

.gitignore

@@ -106,6 +106,13 @@ STABILIZATION_TODO.md
 **/coverage
 test-config
 
+# Claude Code compatibility-matrix pytest artifact (CI-only output).
+compat-results.json
+compat-results.json.shards/
+compat-rate-limit-summary.json
+# Matrix JSON produced by the daily-cron publisher (pushed to litellm-docs).
+compatibility-matrix.json
+
 # ---------- Terraform ----------
 # Provider binaries + module cache — regenerated by `terraform init`.
 **/.terraform/

docker/build_from_pip/Dockerfile.build_from_pip

@@ -36,7 +36,7 @@ RUN uv venv --python python && \
       "opentelemetry-api==1.28.0" \
       "opentelemetry-sdk==1.28.0" \
       "opentelemetry-exporter-otlp==1.28.0" \
-      "ddtrace==2.19.0" \
+      "ddtrace==4.11.0" \
       "sentry-sdk==2.21.0" \
       "mangum==0.17.0" \
       "azure-ai-contentsafety==1.0.0" \

enterprise/litellm_enterprise/enterprise_callbacks/llm_guard.py

@@ -7,7 +7,8 @@
 #  Thank you users! We ❤️ you! - Krrish & Ishaan
 ## This provides an LLM Guard Integration for content moderation on the proxy
 
-from typing import Literal, Optional
+import asyncio
+from typing import Optional
 
 import aiohttp
 from fastapi import HTTPException
@@ -18,7 +19,6 @@ from litellm.integrations.custom_logger import CustomLogger
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.utils import CallTypesLiteral
-from litellm.utils import get_formatted_prompt
 
 
 class _ENTERPRISE_LLMGuard(CustomLogger):
@@ -46,45 +46,44 @@ class _ENTERPRISE_LLMGuard(CustomLogger):
         except Exception:
             pass
 
-    async def moderation_check(self, text: str):
+    async def moderation_check(self, text: str) -> str:
         """
+        Runs the LLM Guard moderation check on ``text``.
+
+        Raises an HTTPException when the content violates the safety policy;
+        otherwise returns the sanitized prompt from LLM Guard, falling back to
+        the original text when the API does not provide one.
+
         [TODO] make this more performant for high-throughput scenario
         """
         try:
-            async with aiohttp.ClientSession() as session:
-                if self.mock_redacted_text is not None:
-                    redacted_text = self.mock_redacted_text
-                else:
-                    # Make the first request to /analyze
-                    analyze_url = f"{self.llm_guard_api_base}analyze/prompt"
-                    verbose_proxy_logger.debug("Making request to: %s", analyze_url)
-                    analyze_payload = {"prompt": text}
-                    redacted_text = None
+            if self.mock_redacted_text is not None:
+                redacted_text = self.mock_redacted_text
+            else:
+                analyze_url = f"{self.llm_guard_api_base}analyze/prompt"
+                verbose_proxy_logger.debug("Making request to: %s", analyze_url)
+                async with aiohttp.ClientSession() as session:
                     async with session.post(
-                        analyze_url, json=analyze_payload
+                        analyze_url, json={"prompt": text}
                     ) as response:
                         redacted_text = await response.json()
-                verbose_proxy_logger.debug(
-                    f"LLM Guard: Received response - {redacted_text}"
+            verbose_proxy_logger.debug(
+                f"LLM Guard: Received response - {redacted_text}"
+            )
+            if redacted_text is None:
+                raise HTTPException(
+                    status_code=500,
+                    detail={
+                        "error": f"Invalid content moderation response: {redacted_text}"
+                    },
                 )
-                if redacted_text is not None:
-                    if (
-                        redacted_text.get("is_valid", None) is not None
-                        and redacted_text["is_valid"] is False
-                    ):
-                        raise HTTPException(
-                            status_code=400,
-                            detail={"error": "Violated content safety policy"},
-                        )
-                    else:
-                        pass
-                else:
-                    raise HTTPException(
-                        status_code=500,
-                        detail={
-                            "error": f"Invalid content moderation response: {redacted_text}"
-                        },
-                    )
+            if redacted_text.get("is_valid", None) is False:
+                raise HTTPException(
+                    status_code=400,
+                    detail={"error": "Violated content safety policy"},
+                )
+            sanitized_prompt = redacted_text.get("sanitized_prompt")
+            return sanitized_prompt if isinstance(sanitized_prompt, str) else text
         except Exception as e:
             verbose_proxy_logger.exception(
                 "litellm.enterprise.enterprise_hooks.llm_guard::moderation_check - Exception occurred - {}".format(
@@ -138,23 +137,75 @@ class _ENTERPRISE_LLMGuard(CustomLogger):
             return
 
         self.print_verbose("Makes LLM Guard Check")
-        try:
-            assert call_type in [
-                "completion",
-                "embeddings",
-                "image_generation",
-                "moderation",
-                "audio_transcription",
-            ]
-        except Exception:
+        if call_type not in [
+            "completion",
+            "embeddings",
+            "image_generation",
+            "moderation",
+            "audio_transcription",
+        ]:
             self.print_verbose(
                 f"Call Type - {call_type}, not in accepted list - ['completion','embeddings','image_generation','moderation','audio_transcription']"
             )
             return data
 
-        formatted_prompt = get_formatted_prompt(data=data, call_type=call_type)  # type: ignore
-        self.print_verbose(f"LLM Guard, formatted_prompt: {formatted_prompt}")
-        return await self.moderation_check(text=formatted_prompt)
+        return await self._moderate_request(data=data)
+
+    async def _moderate_request(self, data: dict) -> dict:
+        """
+        Sanitizes the request in place using the prompt returned by LLM Guard so
+        the provider-bound request carries the redacted content, then returns it.
+        """
+        messages = data.get("messages")
+        if messages is not None:
+            data["messages"] = list(
+                await asyncio.gather(
+                    *(self._moderate_message(message) for message in messages)
+                )
+            )
+            return data
+
+        input_ = data.get("input")
+        if input_ is not None:
+            data["input"] = await self._moderate_input(input_)
+            return data
+
+        prompt = data.get("prompt")
+        if isinstance(prompt, str):
+            data["prompt"] = await self.moderation_check(text=prompt)
+        return data
+
+    async def _moderate_message(self, message: dict) -> dict:
+        content = message.get("content")
+        if isinstance(content, str):
+            return {**message, "content": await self.moderation_check(text=content)}
+        if isinstance(content, list):
+            return {
+                **message,
+                "content": list(
+                    await asyncio.gather(
+                        *(self._moderate_content_part(part) for part in content)
+                    )
+                ),
+            }
+        return message
+
+    async def _moderate_content_part(self, part: dict) -> dict:
+        if part.get("type") == "text" and isinstance(part.get("text"), str):
+            return {**part, "text": await self.moderation_check(text=part["text"])}
+        return part
+
+    async def _moderate_input(self, input_: object) -> object:
+        if isinstance(input_, str):
+            return await self.moderation_check(text=input_)
+        if isinstance(input_, list):
+            return [
+                await self.moderation_check(text=item)
+                if isinstance(item, str)
+                else item
+                for item in input_
+            ]
+        return input_
 
     async def async_post_call_streaming_hook(
         self, user_api_key_dict: UserAPIKeyAuth, response: str

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.49"
+version = "0.1.50"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.49"
+version = "0.1.50"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

helm/litellm-helm/README.md

@@ -54,6 +54,12 @@ If `db.useStackgresOperator` is used (not yet implemented):
 | `pdb.annotations`           | Extra metadata annotations to add to the PDB                                                                                                                                                                                                  | `{}`                      |
 | `pdb.labels`                | Extra metadata labels to add to the PDB                                                                                                                                                                                                       | `{}`                      |
 
+| `billingMetrics.enabled`    | Enable enterprise billable-request metering. Requires an enterprise license.                                                                                                                                                                  | `false`                   |
+| `billingMetrics.endpoint`   | Collector that the billable-request counter is pushed to.                                                                                                                                                                                    | `https://telemetry.litellm.ai` |
+| `billingMetrics.secretName` | Name of an existing Secret holding the mTLS client certificate, under the keys `tls.crt` and `tls.key`.                                                                                                                                       | `litellm-billing-metrics-mtls` |
+| `billingMetrics.caSecretName` | Name of an existing Secret holding a CA bundle under the key `ca.crt`. Only needed for a private or test collector whose server certificate is not on the public web PKI.                                                                    | `""`                      |
+| `billingMetrics.exportIntervalMs` | How often the counter is pushed, in milliseconds. The proxy defaults to `60000` when unset.                                                                                                                                             | `""`                      |
+
 #### Example `proxy_config` ConfigMap from values (default):
 
 ```
@@ -94,6 +100,21 @@ data:
 type: Opaque
 ```
 
+#### Enterprise billable-request metering
+
+Enterprise licenses meter billable requests by pushing a counter to LiteLLM's collector over mutual TLS. The chart does not create the client certificate; it mounts one you already hold, read-only, so the private key is never exposed through the environment. Create the Secret under the name the chart expects, then turn the block on:
+
+```
+kubectl create secret tls litellm-billing-metrics-mtls --cert=client.crt --key=client.key
+```
+
+```
+billingMetrics:
+  enabled: true
+```
+
+Set `billingMetrics.caSecretName` only when the collector is a private or test one whose server certificate is not on the public web PKI; the production collector needs no CA override. The chart fails the render rather than deploying a proxy that silently never exports, so a missing `secretName` or an emptied `endpoint` surfaces at `helm install` time.
+
 ### Database Settings
 
 | Name                      | Description                                                                                                                                                                                                                                                                               | Value                                                                                      |

helm/litellm-helm/templates/_helpers.tpl

@@ -50,6 +50,53 @@ app.kubernetes.io/name: {{ include "litellm.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
 
+{{/*
+Enterprise billable-request metering. The client certificate identifies the
+deployment to LiteLLM's collector, so it is mounted read-only from an existing
+Secret rather than passed through the environment.
+*/}}
+{{- define "litellm.billingMetrics.certDir" -}}/etc/litellm/billing-mtls{{- end -}}
+{{- define "litellm.billingMetrics.caDir" -}}/etc/litellm/billing-mtls-ca{{- end -}}
+
+{{- define "litellm.billingMetricsEnv" -}}
+- name: LITELLM_BILLING_METRICS_ENDPOINT
+  value: {{ required "billingMetrics.endpoint is required when billingMetrics.enabled is true" .Values.billingMetrics.endpoint | quote }}
+- name: LITELLM_BILLING_METRICS_CLIENT_CERT
+  value: {{ printf "%s/tls.crt" (include "litellm.billingMetrics.certDir" .) | quote }}
+- name: LITELLM_BILLING_METRICS_CLIENT_KEY
+  value: {{ printf "%s/tls.key" (include "litellm.billingMetrics.certDir" .) | quote }}
+{{- if .Values.billingMetrics.caSecretName }}
+- name: LITELLM_BILLING_METRICS_CA_CERT
+  value: {{ printf "%s/ca.crt" (include "litellm.billingMetrics.caDir" .) | quote }}
+{{- end }}
+{{- with .Values.billingMetrics.exportIntervalMs }}
+- name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS
+  value: {{ . | quote }}
+{{- end }}
+{{- end -}}
+
+{{- define "litellm.billingMetricsVolumes" -}}
+- name: billing-metrics-mtls
+  secret:
+    secretName: {{ required "billingMetrics.secretName is required when billingMetrics.enabled is true (an existing Secret with tls.crt and tls.key)" .Values.billingMetrics.secretName }}
+{{- if .Values.billingMetrics.caSecretName }}
+- name: billing-metrics-mtls-ca
+  secret:
+    secretName: {{ .Values.billingMetrics.caSecretName }}
+{{- end }}
+{{- end -}}
+
+{{- define "litellm.billingMetricsVolumeMounts" -}}
+- name: billing-metrics-mtls
+  mountPath: {{ include "litellm.billingMetrics.certDir" . }}
+  readOnly: true
+{{- if .Values.billingMetrics.caSecretName }}
+- name: billing-metrics-mtls-ca
+  mountPath: {{ include "litellm.billingMetrics.caDir" . }}
+  readOnly: true
+{{- end }}
+{{- end -}}
+
 {{/*
 Create the name of the service account to use
 */}}

helm/litellm-helm/templates/deployment.yaml

@@ -142,6 +142,9 @@ spec:
             {{- with .Values.extraEnvVars }}
               {{- toYaml . | nindent 12 }}
             {{- end }}
+            {{- if .Values.billingMetrics.enabled }}
+            {{- include "litellm.billingMetricsEnv" . | nindent 12 }}
+            {{- end }}
             {{- if .Values.migrationJob.enabled }}
             # Schema updates are owned by the dedicated migrations Job; skip
             # the proxy's startup `prisma db push` so N replicas don't race
@@ -220,6 +223,9 @@ spec:
             - name: npm
               mountPath: /.npm
           {{- end }}
+          {{- if .Values.billingMetrics.enabled }}
+            {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }}
+          {{- end }}
           {{- with .Values.volumeMounts }}
             {{- toYaml . | nindent 12 }}
           {{- end }}
@@ -252,6 +258,9 @@ spec:
             items:
               - key: {{ .Values.proxyConfigMap.key | default "config.yaml" }}
                 path: "config.yaml"
+        {{- if .Values.billingMetrics.enabled }}
+        {{- include "litellm.billingMetricsVolumes" . | nindent 8 }}
+        {{- end }}
       {{- with .Values.volumes }}
         {{- toYaml . | nindent 8 }}
       {{- end }}

helm/litellm-helm/tests/billing_metrics_tests.yaml

@@ -0,0 +1,297 @@
+suite: test billingMetrics wiring on the proxy deployment
+templates:
+  - deployment.yaml
+  - configmap-litellm.yaml
+  - migrations-job.yaml
+tests:
+  - it: is off by default, adding no env, volume, or mount
+    template: deployment.yaml
+    asserts:
+      - notContains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls
+            secret:
+              secretName: litellm-billing-metrics-mtls
+      - notContains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: billing-metrics-mtls
+            mountPath: /etc/litellm/billing-mtls
+            readOnly: true
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+            value: https://telemetry.litellm.ai
+
+  - it: renders the endpoint and the mounted cert paths when enabled
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+            value: https://telemetry.litellm.ai
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_CLIENT_CERT
+            value: /etc/litellm/billing-mtls/tls.crt
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_CLIENT_KEY
+            value: /etc/litellm/billing-mtls/tls.key
+
+  # The conventional Secret name is the default, so enabling the block is enough.
+  - it: mounts the default cert secret read-only alongside the config volume
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+    asserts:
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls
+            secret:
+              secretName: litellm-billing-metrics-mtls
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: billing-metrics-mtls
+            mountPath: /etc/litellm/billing-mtls
+            readOnly: true
+
+  - it: honours a secretName override
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        secretName: my-billing-mtls
+    asserts:
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls
+            secret:
+              secretName: my-billing-mtls
+      - notContains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls
+            secret:
+              secretName: litellm-billing-metrics-mtls
+
+  - it: honours an endpoint override
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        endpoint: https://collector.internal:4318
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+            value: https://collector.internal:4318
+
+  # The production collector presents a public web-PKI certificate, so the CA
+  # override must stay absent unless a private collector is configured.
+  - it: omits the CA env, volume, and mount when no caSecretName is set
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+    asserts:
+      - notContains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls-ca
+            secret:
+              secretName: billing-ca
+      - notContains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: billing-metrics-mtls-ca
+            mountPath: /etc/litellm/billing-mtls-ca
+            readOnly: true
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_CA_CERT
+            value: /etc/litellm/billing-mtls-ca/ca.crt
+
+  - it: mounts the CA secret when caSecretName is set
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        caSecretName: billing-ca
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_CA_CERT
+            value: /etc/litellm/billing-mtls-ca/ca.crt
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls-ca
+            secret:
+              secretName: billing-ca
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: billing-metrics-mtls-ca
+            mountPath: /etc/litellm/billing-mtls-ca
+            readOnly: true
+
+  - it: passes the export interval through only when set
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        exportIntervalMs: 5000
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS
+            value: "5000"
+
+  - it: omits the export interval when unset
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+    asserts:
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS
+            value: "60000"
+
+  # Kubernetes resolves duplicate env names last-wins, so the chart-owned billing
+  # entries must render after .Values.envVars or a user could silently redirect
+  # the metering export. The three billing entries are the last ones emitted here
+  # (migrationJob, which appends DISABLE_SCHEMA_UPDATE, is off for this case).
+  - it: renders the billing endpoint after envVars so it cannot be shadowed
+    template: deployment.yaml
+    set:
+      migrationJob:
+        enabled: false
+      billingMetrics:
+        enabled: true
+      envVars:
+        LITELLM_BILLING_METRICS_ENDPOINT: https://shadowed.example
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+            value: https://shadowed.example
+      - equal:
+          path: spec.template.spec.containers[0].env[-3]
+          value:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+            value: https://telemetry.litellm.ai
+      - equal:
+          path: spec.template.spec.containers[0].env[-2].name
+          value: LITELLM_BILLING_METRICS_CLIENT_CERT
+      - equal:
+          path: spec.template.spec.containers[0].env[-1].name
+          value: LITELLM_BILLING_METRICS_CLIENT_KEY
+
+  - it: keeps user-supplied volumes and mounts alongside the billing secret
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+      volumes:
+        - name: custom-callbacks
+          configMap:
+            name: my-callbacks
+      volumeMounts:
+        - name: custom-callbacks
+          mountPath: /app/callbacks
+    asserts:
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: custom-callbacks
+            configMap:
+              name: my-callbacks
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls
+            secret:
+              secretName: litellm-billing-metrics-mtls
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: custom-callbacks
+            mountPath: /app/callbacks
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: billing-metrics-mtls
+            mountPath: /etc/litellm/billing-mtls
+            readOnly: true
+
+  - it: still mounts the proxy config when enabled
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: litellm-config
+            mountPath: /etc/litellm/config.yaml
+            subPath: config.yaml
+
+  # Only the proxy serves billable traffic. The migrations Job must never mount
+  # the client certificate, and it renders its own env and volumes, so nothing
+  # stops a future edit from wiring the billing include into it by mistake.
+  - it: does not touch the migrations job when enabled
+    template: migrations-job.yaml
+    set:
+      billingMetrics:
+        enabled: true
+    asserts:
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+            value: https://telemetry.litellm.ai
+      - notExists:
+          path: spec.template.spec.containers[0].volumeMounts
+      - notExists:
+          path: spec.template.spec.volumes
+
+  - it: fails loudly when enabled with an emptied secretName
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        secretName: ""
+    asserts:
+      - failedTemplate:
+          errorMessage: billingMetrics.secretName is required when billingMetrics.enabled is true (an existing Secret with tls.crt and tls.key)
+
+  - it: fails loudly when enabled without an endpoint
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        endpoint: ""
+    asserts:
+      - failedTemplate:
+          errorMessage: billingMetrics.endpoint is required when billingMetrics.enabled is true

helm/litellm-helm/values.yaml

@@ -139,6 +139,20 @@ masterkeySecretName: ""
 # if set, use this secret key for the master key; otherwise, use the default key
 masterkeySecretKey: ""
 
+# Optional: enterprise billable-request metering. When enabled, the proxy counts
+# successful requests to inference, MCP, and A2A endpoints and pushes them to
+# LiteLLM's collector over mutual TLS. Requires an enterprise license.
+# The client certificate identifies the deployment, so it is mounted read-only
+# from an existing Secret and never passed through the environment.
+billingMetrics:
+  enabled: false
+  endpoint: https://telemetry.litellm.ai   # collector to push the counter to
+  secretName: litellm-billing-metrics-mtls # existing Secret holding tls.crt and tls.key
+  # Only for private or test collectors whose server certificate is not on the
+  # public web PKI. The production collector needs no CA override.
+  caSecretName: ""                         # existing Secret holding ca.crt
+  exportIntervalMs: ""                     # push cadence; the proxy defaults to 60000
+
 proxyConfigMap:
   # when true, creates a new configmap
   create: true

helm/litellm/templates/_helpers.tpl

@@ -34,6 +34,57 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
 helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }}
 {{- end -}}
 
+{{/*
+Enterprise billable-request metering. Wired into gateway and backend, not the
+migrations job. The gateway serves nearly all billable traffic, but the backend
+keeps the named-server MCP transport (/{mcp_server_name}/mcp), which writes a
+SpendLogs row, so metering only the gateway would silently drop that traffic.
+The client certificate identifies the deployment to LiteLLM's collector, so it is
+mounted read-only from an existing Secret rather than passed through the
+environment.
+*/}}
+{{- define "litellm.billingMetrics.certDir" -}}/etc/litellm/billing-mtls{{- end -}}
+{{- define "litellm.billingMetrics.caDir" -}}/etc/litellm/billing-mtls-ca{{- end -}}
+
+{{- define "litellm.billingMetricsEnv" -}}
+- name: LITELLM_BILLING_METRICS_ENDPOINT
+  value: {{ required "billingMetrics.endpoint is required when billingMetrics.enabled is true" .Values.billingMetrics.endpoint | quote }}
+- name: LITELLM_BILLING_METRICS_CLIENT_CERT
+  value: {{ printf "%s/tls.crt" (include "litellm.billingMetrics.certDir" .) | quote }}
+- name: LITELLM_BILLING_METRICS_CLIENT_KEY
+  value: {{ printf "%s/tls.key" (include "litellm.billingMetrics.certDir" .) | quote }}
+{{- if .Values.billingMetrics.caSecretName }}
+- name: LITELLM_BILLING_METRICS_CA_CERT
+  value: {{ printf "%s/ca.crt" (include "litellm.billingMetrics.caDir" .) | quote }}
+{{- end }}
+{{- with .Values.billingMetrics.exportIntervalMs }}
+- name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS
+  value: {{ . | quote }}
+{{- end }}
+{{- end -}}
+
+{{- define "litellm.billingMetricsVolumes" -}}
+- name: billing-metrics-mtls
+  secret:
+    secretName: {{ required "billingMetrics.secretName is required when billingMetrics.enabled is true (an existing Secret with tls.crt and tls.key)" .Values.billingMetrics.secretName }}
+{{- if .Values.billingMetrics.caSecretName }}
+- name: billing-metrics-mtls-ca
+  secret:
+    secretName: {{ .Values.billingMetrics.caSecretName }}
+{{- end }}
+{{- end -}}
+
+{{- define "litellm.billingMetricsVolumeMounts" -}}
+- name: billing-metrics-mtls
+  mountPath: {{ include "litellm.billingMetrics.certDir" . }}
+  readOnly: true
+{{- if .Values.billingMetrics.caSecretName }}
+- name: billing-metrics-mtls-ca
+  mountPath: {{ include "litellm.billingMetrics.caDir" . }}
+  readOnly: true
+{{- end }}
+{{- end -}}
+
 {{/*
 Per-component selector labels — used in both Service selectors and Deployment matchLabels.
 */}}

helm/litellm/templates/backend/deployment.yaml

@@ -44,14 +44,20 @@ spec:
             - name: CONFIG_FILE_PATH
               value: /app/config/config.yaml
             {{- end }}
+            {{- if .Values.billingMetrics.enabled }}
+            {{- include "litellm.billingMetricsEnv" . | nindent 12 }}
+            {{- end }}
           {{- include "litellm.envFrom" .Values.backend | nindent 10 }}
-          {{- if or .Values.gateway.config.create .Values.backend.volumeMounts }}
+          {{- if or .Values.gateway.config.create .Values.backend.volumeMounts .Values.billingMetrics.enabled }}
           volumeMounts:
             {{- if .Values.gateway.config.create }}
             - name: gateway-config
               mountPath: /app/config/config.yaml
               subPath: config.yaml
             {{- end }}
+            {{- if .Values.billingMetrics.enabled }}
+            {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }}
+            {{- end }}
             {{- with .Values.backend.volumeMounts }}
             {{- toYaml . | nindent 12 }}
             {{- end }}
@@ -66,13 +72,16 @@ spec:
           {{- end }}
           resources:
             {{- toYaml .Values.backend.resources | nindent 12 }}
-      {{- if or .Values.gateway.config.create .Values.backend.volumes }}
+      {{- if or .Values.gateway.config.create .Values.backend.volumes .Values.billingMetrics.enabled }}
       volumes:
         {{- if .Values.gateway.config.create }}
         - name: gateway-config
           configMap:
             name: {{ include "litellm.gateway.fullname" . }}-config
         {{- end }}
+        {{- if .Values.billingMetrics.enabled }}
+        {{- include "litellm.billingMetricsVolumes" . | nindent 8 }}
+        {{- end }}
         {{- with .Values.backend.volumes }}
         {{- toYaml . | nindent 8 }}
         {{- end }}

helm/litellm/templates/gateway/deployment.yaml

@@ -46,14 +46,20 @@ spec:
             - name: NUM_WORKERS
               value: {{ .Values.gateway.numWorkers | quote }}
             {{- end }}
+            {{- if .Values.billingMetrics.enabled }}
+            {{- include "litellm.billingMetricsEnv" . | nindent 12 }}
+            {{- end }}
           {{- include "litellm.envFrom" .Values.gateway | nindent 10 }}
-          {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts }}
+          {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.billingMetrics.enabled }}
           volumeMounts:
             {{- if .Values.gateway.config.create }}
             - name: gateway-config
               mountPath: /app/config/config.yaml
               subPath: config.yaml
             {{- end }}
+            {{- if .Values.billingMetrics.enabled }}
+            {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }}
+            {{- end }}
             {{- with .Values.gateway.volumeMounts }}
             {{- toYaml . | nindent 12 }}
             {{- end }}
@@ -68,13 +74,16 @@ spec:
           {{- end }}
           resources:
             {{- toYaml .Values.gateway.resources | nindent 12 }}
-      {{- if or .Values.gateway.config.create .Values.gateway.volumes }}
+      {{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.billingMetrics.enabled }}
       volumes:
         {{- if .Values.gateway.config.create }}
         - name: gateway-config
           configMap:
             name: {{ include "litellm.gateway.fullname" . }}-config
         {{- end }}
+        {{- if .Values.billingMetrics.enabled }}
+        {{- include "litellm.billingMetricsVolumes" . | nindent 8 }}
+        {{- end }}
         {{- with .Values.gateway.volumes }}
         {{- toYaml . | nindent 8 }}
         {{- end }}

helm/litellm/tests/billing_metrics_tests.yaml

@@ -0,0 +1,249 @@
+suite: test billingMetrics wiring on gateway and backend
+templates:
+  - gateway/deployment.yaml
+  - gateway/configmap.yaml
+  - backend/deployment.yaml
+  - migrations-job.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: is off by default, adding no env, volume, or mount
+    template: gateway/deployment.yaml
+    asserts:
+      - notContains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls
+            secret:
+              secretName: billing-mtls
+      - equal:
+          path: spec.template.spec.containers[0].volumeMounts
+          value:
+            - name: gateway-config
+              mountPath: /app/config/config.yaml
+              subPath: config.yaml
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+            value: https://telemetry.litellm.ai
+
+  - it: renders the endpoint and the mounted cert paths when enabled
+    template: gateway/deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        secretName: billing-mtls
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+            value: https://telemetry.litellm.ai
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_CLIENT_CERT
+            value: /etc/litellm/billing-mtls/tls.crt
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_CLIENT_KEY
+            value: /etc/litellm/billing-mtls/tls.key
+
+  - it: mounts the cert secret read-only alongside the config volume
+    template: gateway/deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        secretName: billing-mtls
+    asserts:
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls
+            secret:
+              secretName: billing-mtls
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: billing-metrics-mtls
+            mountPath: /etc/litellm/billing-mtls
+            readOnly: true
+
+  # The production collector presents a public web-PKI certificate, so the CA
+  # override must stay absent unless a private collector is configured.
+  - it: omits the CA env, volume, and mount when no caSecretName is set
+    template: gateway/deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        secretName: billing-mtls
+    asserts:
+      - notContains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls-ca
+            secret:
+              secretName: billing-ca
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_CA_CERT
+            value: /etc/litellm/billing-mtls-ca/ca.crt
+
+  - it: mounts the CA secret when caSecretName is set
+    template: gateway/deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        secretName: billing-mtls
+        caSecretName: billing-ca
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_CA_CERT
+            value: /etc/litellm/billing-mtls-ca/ca.crt
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls-ca
+            secret:
+              secretName: billing-ca
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: billing-metrics-mtls-ca
+            mountPath: /etc/litellm/billing-mtls-ca
+            readOnly: true
+
+  - it: passes the export interval through only when set
+    template: gateway/deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        secretName: billing-mtls
+        exportIntervalMs: 5000
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS
+            value: "5000"
+
+  - it: keeps user-supplied gateway volumes alongside the billing secret
+    template: gateway/deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        secretName: billing-mtls
+      gateway.volumes:
+        - name: custom-callbacks
+          configMap:
+            name: my-callbacks
+      gateway.volumeMounts:
+        - name: custom-callbacks
+          mountPath: /app/callbacks
+    asserts:
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: custom-callbacks
+            configMap:
+              name: my-callbacks
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls
+            secret:
+              secretName: billing-mtls
+
+  # The backend keeps the named-server MCP transport (/{mcp_server_name}/mcp),
+  # which writes a SpendLogs row, so it must meter too or that traffic is lost.
+  - it: meters the backend as well, since it serves the MCP transport
+    template: backend/deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        secretName: billing-mtls
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+            value: https://telemetry.litellm.ai
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: billing-metrics-mtls
+            mountPath: /etc/litellm/billing-mtls
+            readOnly: true
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls
+            secret:
+              secretName: billing-mtls
+
+  - it: leaves the backend alone when metering is off
+    template: backend/deployment.yaml
+    asserts:
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+            value: https://telemetry.litellm.ai
+
+  # The migrations job runs prisma and serves no traffic; it must never receive
+  # the client key.
+  - it: never mounts the billing cert on the migrations job
+    template: migrations-job.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        secretName: billing-mtls
+    asserts:
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+            value: https://telemetry.litellm.ai
+      - isNull:
+          path: spec.template.spec.volumes
+
+  # The conventional Secret name is the default, so enabling metering needs no
+  # secretName at all; the guard below only fires on an explicitly blanked one.
+  - it: uses the conventional secret name by default
+    template: gateway/deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+    asserts:
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls
+            secret:
+              secretName: litellm-billing-metrics-mtls
+
+  - it: fails loudly when the secretName is explicitly blanked
+    template: gateway/deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        secretName: ""
+    asserts:
+      - failedTemplate:
+          errorMessage: billingMetrics.secretName is required when billingMetrics.enabled is true (an existing Secret with tls.crt and tls.key)
+
+  - it: fails loudly when enabled without an endpoint
+    template: gateway/deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        endpoint: ""
+        secretName: billing-mtls
+    asserts:
+      - failedTemplate:
+          errorMessage: billingMetrics.endpoint is required when billingMetrics.enabled is true

helm/litellm/values.yaml

@@ -73,6 +73,25 @@ masterKey:
   secretName: litellm-master-key-secret              # name of a Secret containing the master key
   secretKey: master-key
 
+# Optional: enterprise billable-request metering. When enabled, the gateway and
+# backend count successful requests to inference, MCP, and A2A endpoints and push
+# them to LiteLLM's collector over mutual TLS. Both components serve billable
+# routes: the backend keeps the named-server MCP transport. Requires an
+# enterprise license. The client certificate identifies the deployment, so it is
+# mounted read-only from an existing Secret and never passed through the env.
+billingMetrics:
+  enabled: false
+  endpoint: https://telemetry.litellm.ai              # collector to push the counter to
+  # An existing Secret holding the client certificate under tls.crt and its key
+  # under tls.key, usually created from the onboarding artifact. The default is
+  # the conventional name, so the common path is to create that Secret and set
+  # enabled: true. Override only if yours is named differently.
+  secretName: litellm-billing-metrics-mtls
+  # Only for private or test collectors whose server certificate is not on the
+  # public web PKI. The production collector needs no CA override.
+  caSecretName: ""                                    # existing Secret holding ca.crt
+  exportIntervalMs: ""                                # push cadence; the proxy defaults to 60000
+
 # External Postgres connection. 
 database:
   writer:

litellm-proxy-extras/litellm_proxy_extras/migrations/20260713230852_add_key_type_to_litellm_verification_token/migration.sql

@@ -0,0 +1,6 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_DeletedVerificationToken" ADD COLUMN     "key_type" TEXT;
+
+-- AlterTable
+ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN     "key_type" TEXT;
+

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -422,6 +422,7 @@ model LiteLLM_VerificationToken {
     budget_reset_at DateTime?
     allowed_cache_controls String[] @default([])
     allowed_routes   String[] @default([])
+    key_type         String?
     policies         String[] @default([])
     access_group_ids String[] @default([])
     model_spend      Json @default("{}")
@@ -516,6 +517,7 @@ model LiteLLM_DeletedVerificationToken {
     budget_reset_at         DateTime?
     allowed_cache_controls  String[] @default([])
     allowed_routes          String[] @default([])
+    key_type                String?
     policies                String[] @default([])
     access_group_ids String[] @default([])
     model_spend             Json     @default("{}")

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.76"
+version = "0.4.77"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.76"
+version = "0.4.77"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm/caching/caching_handler.py

@@ -85,6 +85,22 @@ class CachingHandlerResponse(BaseModel):
 in_memory_cache_obj = InMemoryCache()
 
 
+def _drop_logging_obj_from_kwargs(request_kwargs: dict[str, object]) -> dict[str, object]:
+    """
+    The caching handler is stored on the Logging object
+    (``logging_obj._llm_caching_handler``), so keeping ``litellm_logging_obj``
+    inside ``request_kwargs`` closes a reference cycle
+    (Logging -> LLMCachingHandler -> kwargs -> Logging) that keeps the full
+    request payload (messages included) alive until a generational GC pass
+    instead of being freed by refcount when the request ends. Nothing in the
+    caching layer reads the logging object from these kwargs; cache-key
+    generation ignores litellm-internal params.
+    """
+    if "litellm_logging_obj" not in request_kwargs:
+        return request_kwargs
+    return {k: v for k, v in request_kwargs.items() if k != "litellm_logging_obj"}
+
+
 def _is_chat_completion_cached_dict(cached_result: dict) -> bool:
     cached_id = cached_result.get("id")
     if isinstance(cached_id, str) and cached_id.startswith("chatcmpl"):
@@ -118,7 +134,7 @@ class LLMCachingHandler:
 
         self.async_streaming_chunks: List[ModelResponse] = []
         self.sync_streaming_chunks: List[ModelResponse] = []
-        self.request_kwargs = request_kwargs
+        self.request_kwargs = _drop_logging_obj_from_kwargs(request_kwargs)
         self.preset_cache_key: Optional[str] = None
         self.original_function = original_function
         self.start_time = start_time
@@ -297,7 +313,7 @@ class LLMCachingHandler:
                 new_kwargs.pop("metadata", None)
             if new_kwargs.get("stream") is True and "cache_key" not in new_kwargs:
                 new_kwargs["cache_key"] = litellm.cache.get_cache_key(**new_kwargs)
-            self.request_kwargs = new_kwargs
+            self.request_kwargs = _drop_logging_obj_from_kwargs(new_kwargs)
             print_verbose("Checking Sync Cache")
             cached_result = litellm.cache.get_cache(**new_kwargs)
             if cached_result is not None:
@@ -693,7 +709,7 @@ class LLMCachingHandler:
             new_kwargs.pop("metadata", None)
         if new_kwargs.get("stream") is True and "cache_key" not in new_kwargs:
             new_kwargs["cache_key"] = litellm.cache.get_cache_key(**new_kwargs)
-        self.request_kwargs = new_kwargs
+        self.request_kwargs = _drop_logging_obj_from_kwargs(new_kwargs)
         cached_result: Optional[Any] = None
         if call_type == CallTypes.aembedding.value:
             if isinstance(new_kwargs["input"], str):

litellm/constants.py

@@ -1496,6 +1496,7 @@ MAX_TEAM_LIST_LIMIT = int(os.getenv("MAX_TEAM_LIST_LIMIT", 20))
 MAX_POLICY_ESTIMATE_IMPACT_ROWS = int(os.getenv("MAX_POLICY_ESTIMATE_IMPACT_ROWS", 1000))
 DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD = float(os.getenv("DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD", 0.7))
 LENGTH_OF_LITELLM_GENERATED_KEY = int(os.getenv("LENGTH_OF_LITELLM_GENERATED_KEY", 16))
+MINIMUM_CUSTOM_KEY_LENGTH = int(os.getenv("MINIMUM_CUSTOM_KEY_LENGTH", 16))
 SECRET_MANAGER_REFRESH_INTERVAL = int(os.getenv("SECRET_MANAGER_REFRESH_INTERVAL", 86400))
 LITELLM_SETTINGS_SAFE_DB_OVERRIDES = [
     "default_internal_user_params",

litellm/integrations/custom_guardrail.py

@@ -515,6 +515,22 @@ class CustomGuardrail(CustomLogger):
                     return True
         return False
 
+    def uses_apply_guardrail_interface(self) -> bool:
+        return type(self).apply_guardrail is not CustomGuardrail.apply_guardrail
+
+    def _deployment_pre_call_target(self) -> "CustomLogger":
+        if not self.uses_apply_guardrail_interface():
+            return self
+        try:
+            from litellm.proxy.utils import unified_guardrail
+        except ImportError as e:
+            raise ImportError(
+                f"Guardrail {self.guardrail_name or type(self).__name__} implements apply_guardrail, which needs "
+                "the litellm proxy dependencies to run at the deployment level. "
+                "Install them with: pip install 'litellm[proxy]'"
+            ) from e
+        return unified_guardrail
+
     async def async_pre_call_deployment_hook(
         self, kwargs: Dict[str, Any], call_type: Optional[CallTypes]
     ) -> Optional[dict]:
@@ -533,7 +549,10 @@ class CustomGuardrail(CustomLogger):
 
         # CHECK IF GUARDRAIL REJECTS THE REQUEST
         if call_type == CallTypes.completion or call_type == CallTypes.acompletion:
-            result = await self.async_pre_call_hook(
+            target = self._deployment_pre_call_target()
+            if target is not self:
+                kwargs["guardrail_to_apply"] = self
+            result = await target.async_pre_call_hook(
                 user_api_key_dict=UserAPIKeyAuth(
                     user_id=kwargs.get("user_api_key_user_id"),
                     team_id=kwargs.get("user_api_key_team_id"),
@@ -543,7 +562,7 @@ class CustomGuardrail(CustomLogger):
                 ),
                 cache=dc,
                 data=kwargs,
-                call_type=call_type.value or "acompletion",  # type: ignore
+                call_type="completion" if call_type == CallTypes.completion else "acompletion",
             )
 
             if result is not None and isinstance(result, dict):

litellm/integrations/prometheus.py

@@ -239,6 +239,18 @@ class PrometheusLogger(CustomLogger):
                 labelnames=self.get_labels_for_metric("litellm_output_audio_tokens_metric"),
             )
 
+            self.litellm_video_duration_seconds_metric = self._counter_factory(
+                "litellm_video_duration_seconds_metric",
+                "Seconds of video generated, from usage.duration_seconds on video generation calls",
+                labelnames=self.get_labels_for_metric("litellm_video_duration_seconds_metric"),
+            )
+
+            self.litellm_images_generated_metric = self._counter_factory(
+                "litellm_images_generated_metric",
+                "Number of images generated, from the image generation response",
+                labelnames=self.get_labels_for_metric("litellm_images_generated_metric"),
+            )
+
             # Remaining Budget for Team
             self.litellm_remaining_team_budget_metric = self._gauge_factory(
                 "litellm_remaining_team_budget_metric",
@@ -1336,6 +1348,12 @@ class PrometheusLogger(CustomLogger):
             label_context=label_context,
         )
 
+        self._increment_media_generation_metrics(
+            standard_logging_payload=standard_logging_payload,
+            enum_values=enum_values,
+            label_context=label_context,
+        )
+
         # MCP tool call metrics
         self._increment_mcp_tool_call_metrics(
             standard_logging_payload=standard_logging_payload,
@@ -1459,8 +1477,65 @@ class PrometheusLogger(CustomLogger):
             ),
         ]
 
-        for counter, metric_name, value in detail_metrics:
-            if not isinstance(value, (int, float)) or value <= 0:
+        PrometheusLogger._inc_sparse_usage_counters(
+            self,
+            detail_metrics,
+            enum_values=enum_values,
+            label_context=label_context,
+        )
+
+    def _increment_media_generation_metrics(
+        self,
+        standard_logging_payload: StandardLoggingPayload,
+        enum_values: UserAPIKeyLabelValues,
+        label_context: PrometheusLabelFactoryContext | None = None,
+    ) -> None:
+        """
+        Increment video-seconds and images-generated counters from
+        ``standard_logging_payload["metadata"]["usage_object"]``. Video
+        providers report ``duration_seconds`` there; image generation calls
+        report ``output_image_count``. Both are sparse: only emitted when the
+        value is present and > 0, so token-only call types are unaffected.
+        """
+        metadata = standard_logging_payload.get("metadata") or {}
+        usage_object = metadata.get("usage_object") if isinstance(metadata, dict) else None
+        if not isinstance(usage_object, dict):
+            return
+
+        media_metrics: list[tuple[Any, DEFINED_PROMETHEUS_METRICS, Any]] = [
+            (
+                self.litellm_video_duration_seconds_metric,
+                "litellm_video_duration_seconds_metric",
+                usage_object.get("duration_seconds"),
+            ),
+            (
+                self.litellm_images_generated_metric,
+                "litellm_images_generated_metric",
+                usage_object.get("output_image_count"),
+            ),
+        ]
+
+        PrometheusLogger._inc_sparse_usage_counters(
+            self,
+            media_metrics,
+            enum_values=enum_values,
+            label_context=label_context,
+        )
+
+    def _inc_sparse_usage_counters(
+        self,
+        counters_with_values: list[tuple[Any, DEFINED_PROMETHEUS_METRICS, Any]],
+        enum_values: UserAPIKeyLabelValues,
+        label_context: PrometheusLabelFactoryContext | None = None,
+    ) -> None:
+        """
+        Increment each ``(counter, metric_name, value)`` entry whose value is
+        a positive number. Non-numeric values (including booleans from
+        malformed provider usage dicts) and values <= 0 are skipped, keeping
+        scrape output sparse.
+        """
+        for counter, metric_name, value in counters_with_values:
+            if isinstance(value, bool) or not isinstance(value, (int, float)) or value <= 0:
                 continue
             PrometheusLogger._inc_labeled_counter(
                 self,

litellm/integrations/s3.py

@@ -161,8 +161,13 @@ def get_s3_object_key(
     start_time: datetime,
     s3_file_name: str,
 ) -> str:
+    sanitized_s3_file_name = s3_file_name.replace("/", "_")
     s3_object_key = (
-        (s3_path.rstrip("/") + "/" if s3_path else "") + prefix + start_time.strftime("%Y-%m-%d") + "/" + s3_file_name
+        (s3_path.rstrip("/") + "/" if s3_path else "")
+        + prefix
+        + start_time.strftime("%Y-%m-%d")
+        + "/"
+        + sanitized_s3_file_name
     )  # we need the s3 key to include the time, so we log cache hits too
     s3_object_key += ".json"
     return s3_object_key

litellm/integrations/websearch_interception/handler.py

@@ -19,9 +19,11 @@ from litellm.integrations.custom_logger import CustomLogger
 from litellm.integrations.websearch_interception.tools import (
     get_litellm_web_search_tool,
     get_litellm_web_search_tool_openai,
+    get_litellm_web_search_tool_responses,
     is_anthropic_native_web_search_tool,
     is_web_search_tool,
     is_web_search_tool_chat_completion,
+    is_web_search_tool_responses,
 )
 from litellm.integrations.websearch_interception.transformation import (
     WebSearchTransformation,
@@ -32,11 +34,12 @@ from litellm.types.integrations.websearch_interception import (
 )
 from litellm.types.integrations.custom_logger import (
     CHAT_COMPLETION_AGENTIC_SURFACE,
+    RESPONSES_AGENTIC_SURFACE,
     AgenticLoopPlan,
     AgenticLoopRequestPatch,
 )
 from litellm.types.llms.openai import AllMessageValues
-from litellm.types.utils import LlmProviders
+from litellm.types.utils import CallTypes, LlmProviders
 from litellm.utils import ProviderConfigManager
 
 # Key used to flag, on per-request kwargs, that the originating client sent
@@ -251,6 +254,9 @@ class WebSearchInterceptionLogger(CustomLogger):
         if not tools:
             return None
 
+        if call_type in (CallTypes.responses, CallTypes.aresponses):
+            return self._convert_responses_tools(kwargs=kwargs, tools=tools)
+
         # Check if any tool is a web search tool (native or already LiteLLM standard)
         has_websearch = any(is_web_search_tool(t) for t in tools)
 
@@ -291,6 +297,26 @@ class WebSearchInterceptionLogger(CustomLogger):
 
         return kwargs
 
+    def _convert_responses_tools(self, kwargs: dict[str, Any], tools: list[dict[str, Any]]) -> dict | None:
+        """Convert Responses API web search tools to the LiteLLM standard function tool."""
+        if not any(is_web_search_tool_responses(tool) for tool in tools):
+            return None
+
+        verbose_logger.debug("WebSearchInterception: Converting Responses web_search tools to LiteLLM standard")
+
+        converted_tools = [
+            get_litellm_web_search_tool_responses() if is_web_search_tool_responses(tool) else tool for tool in tools
+        ]
+
+        converted_kwargs = {**kwargs, "tools": converted_tools}
+
+        if kwargs.get("stream"):
+            verbose_logger.debug("WebSearchInterception: deployment hook converting stream=True to stream=False")
+            converted_kwargs["stream"] = False
+            converted_kwargs["_websearch_interception_converted_stream"] = True
+
+        return converted_kwargs
+
     @classmethod
     def from_config_yaml(cls, config: WebSearchInterceptionConfig) -> "WebSearchInterceptionLogger":
         """
@@ -461,6 +487,17 @@ class WebSearchInterceptionLogger(CustomLogger):
                 kwargs=kwargs,
             )
 
+        if kwargs.get("_agentic_loop_api_surface") == RESPONSES_AGENTIC_SURFACE:
+            return await self.async_should_run_responses_agentic_loop(
+                response=response,
+                model=model,
+                messages=messages,
+                tools=tools,
+                stream=stream,
+                custom_llm_provider=custom_llm_provider,
+                kwargs=kwargs,
+            )
+
         verbose_logger.debug(f"WebSearchInterception: Hook called! provider={custom_llm_provider}, stream={stream}")
         verbose_logger.debug(f"WebSearchInterception: Response type: {type(response)}")
 
@@ -597,6 +634,54 @@ class WebSearchInterceptionLogger(CustomLogger):
         }
         return True, tools_dict
 
+    async def async_should_run_responses_agentic_loop(
+        self,
+        response: Any,
+        model: str,
+        messages: list[dict],
+        tools: list[dict] | None,
+        stream: bool,
+        custom_llm_provider: str,
+        kwargs: dict,
+    ) -> tuple[bool, dict]:
+        """Check if WebSearch interception is needed for the Responses API."""
+        verbose_logger.debug(
+            f"WebSearchInterception: Responses hook called! provider={custom_llm_provider}, stream={stream}"
+        )
+
+        if self.enabled_providers is not None and custom_llm_provider not in self.enabled_providers:
+            verbose_logger.debug(
+                f"WebSearchInterception: Skipping provider {custom_llm_provider} (not in enabled list: {self.enabled_providers})"
+            )
+            return False, {}
+
+        has_websearch_tool = any(is_web_search_tool_responses(t) for t in (tools or []))
+        if not has_websearch_tool:
+            verbose_logger.debug("WebSearchInterception: No litellm_web_search tool in responses request")
+            return False, {}
+
+        should_intercept, tool_calls = WebSearchTransformation.transform_request(
+            response=response,
+            stream=stream,
+            response_format="responses",
+        )
+
+        if not should_intercept:
+            verbose_logger.debug("WebSearchInterception: No WebSearch function_call detected in responses output")
+            return False, {}
+
+        verbose_logger.debug(
+            f"WebSearchInterception: Detected {len(tool_calls)} WebSearch function_call(s), executing agentic loop"
+        )
+
+        tools_dict = {
+            "tool_calls": tool_calls,
+            "tool_type": "websearch",
+            "provider": custom_llm_provider,
+            "response_format": "responses",
+        }
+        return True, tools_dict
+
     async def async_run_agentic_loop(
         self,
         tools: Dict,
@@ -655,6 +740,18 @@ class WebSearchInterceptionLogger(CustomLogger):
                 kwargs=kwargs,
             )
 
+        if kwargs.get("_agentic_loop_api_surface") == RESPONSES_AGENTIC_SURFACE:
+            return await self.async_build_responses_agentic_loop_plan(
+                tools=tools,
+                model=model,
+                messages=messages,
+                response=response,
+                optional_params=anthropic_messages_optional_request_params,
+                logging_obj=logging_obj,
+                stream=stream,
+                kwargs=kwargs,
+            )
+
         tool_calls = tools["tool_calls"]
         thinking_blocks = tools.get("thinking_blocks", [])
         request_patch, structured_results = await self._build_anthropic_request_patch(
@@ -809,6 +906,133 @@ class WebSearchInterceptionLogger(CustomLogger):
             metadata={"tool_type": "websearch", "response_format": response_format},
         )
 
+    async def async_build_responses_agentic_loop_plan(
+        self,
+        tools: dict,
+        model: str,
+        messages: list[dict],
+        response: Any,
+        optional_params: dict,
+        logging_obj: Any,
+        stream: bool,
+        kwargs: dict,
+    ) -> AgenticLoopPlan:
+        tool_calls = tools["tool_calls"]
+        request_patch = await self._build_responses_request_patch(
+            model=model,
+            messages=messages,
+            tool_calls=tool_calls,
+            optional_params=optional_params,
+            kwargs=kwargs,
+        )
+        return AgenticLoopPlan(
+            run_agentic_loop=True,
+            request_patch=request_patch,
+            metadata={"tool_type": "websearch", "response_format": "responses"},
+        )
+
+    async def _build_responses_request_patch(
+        self,
+        model: str,
+        messages: Union[str, list[dict]],
+        tool_calls: list[dict],
+        optional_params: dict,
+        kwargs: dict,
+    ) -> AgenticLoopRequestPatch:
+        """Execute litellm.asearch() and build a Responses API rerun patch."""
+        search_tasks = [
+            (
+                self._execute_search(tool_call["input"]["query"], kwargs=kwargs)
+                if isinstance(tool_call.get("input"), dict) and tool_call["input"].get("query")
+                else self._create_empty_search_result()
+            )
+            for tool_call in tool_calls
+        ]
+
+        verbose_logger.debug(f"WebSearchInterception: Executing {len(search_tasks)} responses search(es) in parallel")
+        search_results = await asyncio.gather(*search_tasks, return_exceptions=True)
+
+        search_texts = [self._extract_search_text(result) for result in search_results]
+
+        followup_items = [
+            item
+            for tool_call, search_text in zip(tool_calls, search_texts)
+            for item in (
+                {
+                    "type": "function_call",
+                    "call_id": tool_call.get("call_id"),
+                    "name": LITELLM_WEB_SEARCH_TOOL_NAME,
+                    "arguments": tool_call.get("arguments", ""),
+                },
+                {
+                    "type": "function_call_output",
+                    "call_id": tool_call.get("call_id"),
+                    "output": search_text,
+                },
+            )
+        ]
+
+        input_list = self._normalize_responses_input(messages) + followup_items
+
+        tools_param = optional_params.get("tools")
+        optional_params_clean = {
+            k: v
+            for k, v in optional_params.items()
+            if k not in {"tools", "tool_choice", "stream", "model_alias_map", "stream_response", "custom_prompt_dict"}
+        }
+
+        kwargs_for_followup = {
+            k: v
+            for k, v in kwargs.items()
+            if not k.startswith("_websearch_interception")
+            and k
+            not in {
+                "_agentic_loop_api_surface",
+                "litellm_logging_obj",
+                "acompletion",
+                "custom_llm_provider",
+                "model_alias_map",
+            }
+        }
+
+        full_model_name = model
+        if "/" not in model and isinstance(kwargs.get("custom_llm_provider"), str):
+            full_model_name = f"{kwargs['custom_llm_provider']}/{model}"
+
+        verbose_logger.debug(
+            "WebSearchInterception: Built responses request patch model=%s input_items=%d searches=%d",
+            full_model_name,
+            len(input_list),
+            len(search_texts),
+        )
+
+        return AgenticLoopRequestPatch(
+            model=full_model_name,
+            messages=input_list,
+            tools=tools_param if isinstance(tools_param, list) else None,
+            optional_params=optional_params_clean,
+            kwargs=kwargs_for_followup,
+        )
+
+    @staticmethod
+    def _normalize_responses_input(messages: Union[str, list[dict]]) -> list[dict]:
+        if isinstance(messages, str):
+            return [{"role": "user", "content": messages}]
+        if isinstance(messages, list):
+            return list(messages)
+        return []
+
+    @staticmethod
+    def _extract_search_text(result: Any) -> str:
+        if isinstance(result, Exception):
+            verbose_logger.error(f"WebSearchInterception: Responses search failed with error: {str(result)}")
+            return f"Search failed: {str(result)}"
+        if isinstance(result, tuple) and len(result) == 2:
+            text_value, _ = result
+            return text_value if isinstance(text_value, str) else str(text_value)
+        verbose_logger.debug(f"WebSearchInterception: Unexpected search result type {type(result)}")
+        return str(result)
+
     @staticmethod
     def _resolve_max_tokens(
         optional_params: Dict,

litellm/integrations/websearch_interception/tools.py

@@ -82,6 +82,75 @@ def get_litellm_web_search_tool_openai() -> Dict[str, Any]:
     }
 
 
+def get_litellm_web_search_tool_responses() -> dict[str, Any]:
+    """
+    Get the standard LiteLLM web search tool definition in Responses API format.
+
+    Used by async_pre_call_deployment_hook on the Responses API path, where a
+    function tool is a flat object (``type: "function"`` with a top-level
+    ``name`` and ``parameters``) rather than the nested ``function`` wrapper
+    used by Chat Completions.
+
+    Returns:
+        Dict containing the Responses-style function tool definition.
+    """
+    return {
+        "type": "function",
+        "name": LITELLM_WEB_SEARCH_TOOL_NAME,
+        "description": (
+            "Search the web for information. Use this when you need current "
+            "information or answers to questions that require up-to-date data."
+        ),
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "query": {
+                    "type": "string",
+                    "description": "The search query to execute",
+                }
+            },
+            "required": ["query"],
+        },
+    }
+
+
+def is_web_search_tool_responses(tool: dict[str, Any]) -> bool:
+    """
+    Check if a tool is a web search tool for the Responses API.
+
+    Detects:
+    - OpenAI native Responses web search tools, whose ``type`` is one of
+      ``web_search``, ``web_search_2025_08_26``, ``web_search_preview``,
+      ``web_search_preview_2025_03_11`` (matched by the ``web_search`` prefix)
+    - The LiteLLM standard function tool in Responses shape:
+      ``{"type": "function", "name": "litellm_web_search"}``
+
+    Args:
+        tool: Tool dictionary to check
+
+    Returns:
+        True if tool is a Responses-API web search tool
+
+    Example:
+        >>> is_web_search_tool_responses({"type": "web_search"})
+        True
+        >>> is_web_search_tool_responses({"type": "web_search_preview"})
+        True
+        >>> is_web_search_tool_responses({"type": "function", "name": "litellm_web_search"})
+        True
+        >>> is_web_search_tool_responses({"type": "function", "name": "get_weather"})
+        False
+    """
+    tool_type = tool.get("type", "")
+    if not isinstance(tool_type, str):
+        return False
+
+    if tool_type == "function":
+        return tool.get("name") == LITELLM_WEB_SEARCH_TOOL_NAME
+
+    return tool_type == "web_search" or tool_type.startswith("web_search_")
+
+
 def is_web_search_tool_chat_completion(tool: Dict[str, Any]) -> bool:
     """
     Check if a tool is a web search tool for Chat Completions API (strict check).

litellm/integrations/websearch_interception/transformation.py

@@ -59,9 +59,73 @@ class WebSearchTransformation:
         # Parse non-streaming response based on format
         if response_format == "openai":
             return WebSearchTransformation._detect_from_openai_response(response)
+        elif response_format == "responses":
+            return WebSearchTransformation._detect_from_responses_response(response)
         else:
             return WebSearchTransformation._detect_from_non_streaming_response(response)
 
+    @staticmethod
+    def _detect_from_responses_response(
+        response: Any,
+    ) -> tuple[bool, list[dict]]:
+        """Parse a Responses API response for ``litellm_web_search`` function calls.
+
+        After pre-request conversion the native web search tool is replaced by a
+        ``litellm_web_search`` function tool, so the model emits ``function_call``
+        items in ``response.output`` instead of a native ``web_search_call``.
+        """
+        if isinstance(response, dict):
+            output = response.get("output", [])
+        else:
+            output = getattr(response, "output", None) or []
+
+        if not isinstance(output, list):
+            return False, []
+
+        tool_calls: list[dict] = []
+        for item in output:
+            if isinstance(item, dict):
+                item_type = item.get("type")
+                item_name = item.get("name")
+                call_id = item.get("call_id")
+                arguments = item.get("arguments", "")
+            else:
+                item_type = getattr(item, "type", None)
+                item_name = getattr(item, "name", None)
+                call_id = getattr(item, "call_id", None)
+                arguments = getattr(item, "arguments", "")
+
+            if item_type != "function_call" or item_name != LITELLM_WEB_SEARCH_TOOL_NAME:
+                continue
+
+            if isinstance(arguments, str):
+                try:
+                    parsed_input = json.loads(arguments) if arguments else {}
+                except json.JSONDecodeError:
+                    verbose_logger.warning(
+                        f"WebSearchInterception: Failed to parse function_call arguments: {arguments}"
+                    )
+                    parsed_input = {}
+            elif isinstance(arguments, dict):
+                parsed_input = arguments
+            else:
+                parsed_input = {}
+
+            arguments_str = arguments if isinstance(arguments, str) else json.dumps(parsed_input)
+            tool_calls.append(
+                {
+                    "id": call_id,
+                    "call_id": call_id,
+                    "type": "function_call",
+                    "name": item_name,
+                    "arguments": arguments_str,
+                    "input": parsed_input,
+                }
+            )
+            verbose_logger.debug(f"WebSearchInterception: Found {item_name} function_call with call_id={call_id}")
+
+        return len(tool_calls) > 0, tool_calls
+
     @staticmethod
     def _detect_from_non_streaming_response(
         response: Any,

litellm/litellm_core_utils/dd_tracing.py

@@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, Any, Optional, Union
 from litellm.secret_managers.main import get_secret_bool
 
 if TYPE_CHECKING:
-    from ddtrace.tracer import Tracer as DD_TRACER
+    from ddtrace.trace import Tracer as DD_TRACER
 else:
     DD_TRACER = Any
 

litellm/litellm_core_utils/litellm_logging.py

@@ -925,7 +925,6 @@ class Logging(LiteLLMLoggingBaseClass):
 
     def pre_call(self, input, api_key, model=None, additional_args={}):
         # Log the exact input to the LLM API
-        litellm.error_logs["PRE_CALL"] = locals()
         try:
             self._pre_call(
                 input=input,
@@ -1135,7 +1134,6 @@ class Logging(LiteLLMLoggingBaseClass):
 
     def post_call(self, original_response, input=None, api_key=None, additional_args={}):
         # Log the exact result from the LLM API, for streaming - log the type of response received
-        litellm.error_logs["POST_CALL"] = locals()
         if isinstance(original_response, dict):
             original_response = json.dumps(original_response, default=str)
         try:
@@ -3074,7 +3072,7 @@ class Logging(LiteLLMLoggingBaseClass):
     def get_combined_callback_list(self, dynamic_success_callbacks: Optional[List], global_callbacks: List) -> List:
         if dynamic_success_callbacks is None:
             return list(global_callbacks)
-        return list(set(dynamic_success_callbacks + global_callbacks))
+        return list(dict.fromkeys(dynamic_success_callbacks + global_callbacks))
 
     def _remove_internal_litellm_callbacks(self, callbacks: List) -> List:
         """
@@ -5212,10 +5210,15 @@ def get_standard_logging_object_payload(
         call_type = kwargs.get("call_type")
         cache_hit = kwargs.get("cache_hit", False)
         # Extract usage as a plain dict, avoiding Pydantic round-trip
-        usage_dict = StandardLoggingPayloadSetup.get_usage_as_dict(
+        raw_usage_dict = StandardLoggingPayloadSetup.get_usage_as_dict(
             response_obj=response_obj,
             combined_usage_object=cast(Optional[Usage], kwargs.get("combined_usage_object")),
         )
+        usage_dict = (
+            {**raw_usage_dict, "output_image_count": len(init_response_obj.data)}
+            if isinstance(init_response_obj, ImageResponse) and init_response_obj.data
+            else raw_usage_dict
+        )
 
         id = response_obj.get("id", kwargs.get("litellm_call_id"))
 

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -445,6 +445,7 @@ class PromptTokensDetailsResult(TypedDict):
     text_tokens: int
     audio_tokens: int
     image_tokens: int
+    video_tokens: int
     character_count: int
     image_count: int
     video_length_seconds: float
@@ -473,6 +474,7 @@ def _parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
     )
     audio_tokens = cast(Optional[int], getattr(usage.prompt_tokens_details, "audio_tokens", 0)) or 0
     image_tokens = cast(Optional[int], getattr(usage.prompt_tokens_details, "image_tokens", 0)) or 0
+    video_tokens = _coerce_token_count(getattr(usage.prompt_tokens_details, "video_tokens", 0))
     character_count = (
         cast(
             Optional[int],
@@ -503,6 +505,7 @@ def _parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
         text_tokens=text_tokens,
         audio_tokens=audio_tokens,
         image_tokens=image_tokens,
+        video_tokens=video_tokens,
         character_count=character_count,
         image_count=image_count,
         video_length_seconds=float(video_length_seconds),
@@ -515,6 +518,7 @@ class CompletionTokensDetailsResult(TypedDict):
     text_tokens: int
     reasoning_tokens: int
     image_tokens: int
+    video_tokens: int
 
 
 def _parse_completion_tokens_details(usage: Usage) -> CompletionTokensDetailsResult:
@@ -546,12 +550,14 @@ def _parse_completion_tokens_details(usage: Usage) -> CompletionTokensDetailsRes
         )
         or 0
     )
+    video_tokens = _coerce_token_count(getattr(usage.completion_tokens_details, "video_tokens", 0))
 
     return CompletionTokensDetailsResult(
         audio_tokens=audio_tokens,
         text_tokens=text_tokens,
         reasoning_tokens=reasoning_tokens,
         image_tokens=image_tokens,
+        video_tokens=video_tokens,
     )
 
 
@@ -586,6 +592,13 @@ def _calculate_input_cost(
             image_token_cost_key = "input_cost_per_token"
         prompt_cost += calculate_cost_component(model_info, image_token_cost_key, prompt_tokens_details["image_tokens"])
 
+    ### VIDEO TOKEN COST
+    if prompt_tokens_details["video_tokens"]:
+        video_token_cost_key = "input_cost_per_video_token"
+        if model_info.get(video_token_cost_key) is None:
+            video_token_cost_key = "input_cost_per_token"
+        prompt_cost += calculate_cost_component(model_info, video_token_cost_key, prompt_tokens_details["video_tokens"])
+
     ### CACHE WRITING COST - Now uses tiered pricing
     if (
         prompt_tokens_details["cache_creation_tokens"]
@@ -698,6 +711,7 @@ def generic_cost_per_token(
         text_tokens=usage.prompt_tokens,
         audio_tokens=0,
         image_tokens=0,
+        video_tokens=0,
         character_count=0,
         image_count=0,
         video_length_seconds=0.0,
@@ -716,13 +730,14 @@ def generic_cost_per_token(
     audio_tokens = prompt_tokens_details["audio_tokens"]
     cache_creation = prompt_tokens_details["cache_creation_tokens"]
     image_tokens = prompt_tokens_details["image_tokens"]
+    video_tokens = prompt_tokens_details["video_tokens"]
 
     # Check for double-counting: sum of details > prompt_tokens means overlap
-    total_details = text_tokens + cache_hit + audio_tokens + cache_creation + image_tokens
+    total_details = text_tokens + cache_hit + audio_tokens + cache_creation + image_tokens + video_tokens
     has_double_counting = cache_hit > 0 and total_details > usage.prompt_tokens
 
     if (text_tokens == 0 and prompt_tokens_details["image_count"] == 0) or has_double_counting:
-        text_tokens = usage.prompt_tokens - cache_hit - audio_tokens - cache_creation - image_tokens
+        text_tokens = usage.prompt_tokens - cache_hit - audio_tokens - cache_creation - image_tokens - video_tokens
         # Clamp to zero: inconsistent streaming usage
         if text_tokens < 0:
             text_tokens = 0
@@ -751,6 +766,7 @@ def generic_cost_per_token(
     audio_tokens = 0
     reasoning_tokens = 0
     image_tokens = 0
+    video_tokens = 0
     is_text_tokens_total = False
     if usage.completion_tokens_details is not None:
         completion_tokens_details = _parse_completion_tokens_details(usage)
@@ -758,19 +774,20 @@ def generic_cost_per_token(
         text_tokens = completion_tokens_details["text_tokens"]
         reasoning_tokens = completion_tokens_details["reasoning_tokens"]
         image_tokens = completion_tokens_details["image_tokens"]
+        video_tokens = completion_tokens_details["video_tokens"]
 
     # Handle text_tokens calculation:
     # 1. If text_tokens is explicitly provided and > 0, use it
-    # 2. If there's a breakdown (reasoning/audio/image tokens), calculate text_tokens as the remainder
+    # 2. If there's a breakdown (reasoning/audio/image/video tokens), calculate text_tokens as the remainder
     # 3. If no breakdown at all, assume all completion_tokens are text_tokens
-    has_token_breakdown = image_tokens > 0 or audio_tokens > 0 or reasoning_tokens > 0
+    has_token_breakdown = image_tokens > 0 or audio_tokens > 0 or reasoning_tokens > 0 or video_tokens > 0
     if text_tokens == 0:
         if has_token_breakdown:
             # Calculate text tokens as remainder when we have a breakdown
             # This handles cases like OpenAI's reasoning models where text_tokens isn't provided
             text_tokens = max(
                 0,
-                usage.completion_tokens - reasoning_tokens - audio_tokens - image_tokens,
+                usage.completion_tokens - reasoning_tokens - audio_tokens - image_tokens - video_tokens,
             )
         else:
             # No breakdown at all, all tokens are text tokens
@@ -803,6 +820,14 @@ def generic_cost_per_token(
         )
         completion_cost += float(image_tokens) * _output_cost_per_image_token
 
+    ## VIDEO COST
+    if not is_text_tokens_total and video_tokens and video_tokens > 0:
+        _output_cost_per_video_token = _get_cost_per_unit(model_info, "output_cost_per_video_token", None)
+        _output_cost_per_video_token = (
+            _output_cost_per_video_token if _output_cost_per_video_token is not None else completion_base_cost
+        )
+        completion_cost += float(video_tokens) * _output_cost_per_video_token
+
     ## REGIONAL DATA-RESIDENCY UPLIFT
     # Applied as a flat multiplier across all token costs for the request
     # when the upstream is a regionalized OpenAI host (eu./us.api.openai.com).

litellm/litellm_core_utils/redact_messages.py

@@ -77,6 +77,22 @@ def _redact_streaming_response(streaming_response):
             streaming_response.reasoning = None
 
 
+def _redact_tool_calls(tool_calls) -> None:
+    """Redact tool call arguments (assistant tool calls carry prompt-derived data)."""
+    if not tool_calls:
+        return
+    for tool_call in tool_calls:
+        function = getattr(tool_call, "function", None)
+        if function is not None and hasattr(function, "arguments"):
+            function.arguments = "redacted-by-litellm"
+
+
+def _redact_function_call(function_call) -> None:
+    """Redact legacy assistant function_call arguments."""
+    if function_call is not None and hasattr(function_call, "arguments"):
+        function_call.arguments = "redacted-by-litellm"
+
+
 def _redact_choice_content(choice):
     """Helper to redact content in a choice (message or delta)."""
     if isinstance(choice, litellm.Choices):
@@ -85,12 +101,16 @@ def _redact_choice_content(choice):
             choice.message.reasoning_content = "redacted-by-litellm"
         if hasattr(choice.message, "thinking_blocks"):
             choice.message.thinking_blocks = None
+        _redact_tool_calls(getattr(choice.message, "tool_calls", None))
+        _redact_function_call(getattr(choice.message, "function_call", None))
     elif isinstance(choice, litellm.utils.StreamingChoices):
         choice.delta.content = "redacted-by-litellm"
         if hasattr(choice.delta, "reasoning_content"):
             choice.delta.reasoning_content = "redacted-by-litellm"
         if hasattr(choice.delta, "thinking_blocks"):
             choice.delta.thinking_blocks = None
+        _redact_tool_calls(getattr(choice.delta, "tool_calls", None))
+        _redact_function_call(getattr(choice.delta, "function_call", None))
 
 
 def _redact_responses_api_output(output_items):
@@ -111,6 +131,9 @@ def _redact_responses_api_output(output_items):
                     if hasattr(summary_item, "text"):
                         summary_item.text = "redacted-by-litellm"
 
+        if hasattr(output_item, "type") and output_item.type == "function_call" and hasattr(output_item, "arguments"):
+            output_item.arguments = "redacted-by-litellm"
+
 
 def _redact_responses_api_output_dict(output_items, redacted_str: str):
     """Helper to redact ResponsesAPIResponse output items in dict form."""
@@ -131,6 +154,9 @@ def _redact_responses_api_output_dict(output_items, redacted_str: str):
                 if isinstance(summary_item, dict) and "text" in summary_item:
                     summary_item["text"] = redacted_str
 
+        if output_item.get("type") == "function_call" and "arguments" in output_item:
+            output_item["arguments"] = redacted_str
+
 
 def _redact_standard_logging_object(model_call_details: dict):
     """Redact messages and response inside standard_logging_object if present."""
@@ -162,6 +188,19 @@ def _redact_standard_logging_object(model_call_details: dict):
             standard_logging_object["response"] = {"text": redacted_str}
 
 
+def _redact_tool_calls_dict(message: dict, redacted_str: str) -> None:
+    """Redact tool call / function_call arguments in a dict-form message or delta."""
+    tool_calls = message.get("tool_calls")
+    if isinstance(tool_calls, list):
+        for tool_call in tool_calls:
+            if isinstance(tool_call, dict) and isinstance(tool_call.get("function"), dict):
+                tool_call["function"]["arguments"] = redacted_str
+
+    function_call = message.get("function_call")
+    if isinstance(function_call, dict) and "arguments" in function_call:
+        function_call["arguments"] = redacted_str
+
+
 def _redact_model_response_dict_choices(choices, redacted_str: str):
     for choice in choices:
         if isinstance(choice, dict):
@@ -173,6 +212,7 @@ def _redact_model_response_dict_choices(choices, redacted_str: str):
                     choice["message"]["thinking_blocks"] = None
                 if "audio" in choice["message"]:
                     choice["message"]["audio"] = None
+                _redact_tool_calls_dict(choice["message"], redacted_str)
             elif "delta" in choice and isinstance(choice["delta"], dict):
                 choice["delta"]["content"] = redacted_str
                 if "reasoning_content" in choice["delta"]:
@@ -181,6 +221,7 @@ def _redact_model_response_dict_choices(choices, redacted_str: str):
                     choice["delta"]["thinking_blocks"] = None
                 if "audio" in choice["delta"]:
                     choice["delta"]["audio"] = None
+                _redact_tool_calls_dict(choice["delta"], redacted_str)
         else:
             _redact_choice_content(choice)
 

litellm/litellm_core_utils/secret_redaction.py

@@ -9,6 +9,8 @@ secrets from strings without depending on the logging-configuration module.
 import re
 from typing import List
 
+from litellm.constants import MINIMUM_CUSTOM_KEY_LENGTH
+
 _REDACTED = "REDACTED"
 
 
@@ -30,7 +32,7 @@ def _build_secret_patterns() -> "re.Pattern[str]":
         # Basic auth headers
         r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}",
         # OpenAI / Anthropic sk- prefixed keys
-        r"sk-[A-Za-z0-9\-_]{20,}",
+        rf"sk-[A-Za-z0-9\-_]{{{MINIMUM_CUSTOM_KEY_LENGTH - len('sk-')},}}",
         # Generic api_key / api-key / apikey (handles 'key': 'value' dict repr)
         r"(?:api[_-]?key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]{8,}",
         # x-api-key / api-key header values (handles 'key': 'value' dict repr)

litellm/llms/anthropic/chat/transformation.py

@@ -1441,24 +1441,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                     output_key=param,
                 )
             elif param == "response_format" and isinstance(value, dict):
-                if any(
-                    substring in model
-                    for substring in {
-                        "sonnet-4.5",
-                        "sonnet-4-5",
-                        "opus-4.1",
-                        "opus-4-1",
-                        "opus-4.5",
-                        "opus-4-5",
-                        "opus-4.6",
-                        "opus-4-6",
-                        "opus-4.7",
-                        "opus-4-7",
-                        "sonnet-4.6",
-                        "sonnet-4-6",
-                        "sonnet_4.6",
-                        "sonnet_4_6",
-                    }
+                if AnthropicConfig._supports_model_capability(
+                    model,
+                    "supports_native_structured_output",
+                    self._resolved_provider,
                 ):
                     _output_format = self.map_response_format_to_anthropic_output_format(value)
                     if _output_format is not None:

litellm/llms/anthropic/common_utils.py

@@ -340,11 +340,15 @@ class AnthropicModelInfo(BaseLLMModelInfo):
     def _get_model_capability(model: str, key: str) -> Optional[bool]:
         """Read boolean capability ``key`` from the model map, or None when
         no entry declares it."""
+        from litellm.utils import _get_bundled_model_cost_map
+
         try:
-            for cand in AnthropicModelInfo._model_map_lookup_candidates(model):
-                value = litellm.model_cost.get(cand, {}).get(key)
-                if isinstance(value, bool):
-                    return value
+            candidates = AnthropicModelInfo._model_map_lookup_candidates(model)
+            for model_cost in (litellm.model_cost, _get_bundled_model_cost_map()):
+                for cand in candidates:
+                    value = model_cost.get(cand, {}).get(key)
+                    if isinstance(value, bool):
+                        return value
         except Exception:
             pass
         return None

litellm/llms/anthropic/experimental_pass_through/adapters/streaming_iterator.py

@@ -13,14 +13,18 @@ from typing import (
     List,
     Literal,
     Optional,
+    get_args,
 )
 
+from typing_extensions import assert_never
+
 from litellm._logging import verbose_logger
 from litellm._uuid import uuid
 from litellm.types.llms.anthropic import (
     AppliedEdit,
     CompactionBlock,
     ContextManagementResponse,
+    StreamingContentBlockDeltaType,
     UsageDelta,
     UsageIteration,
 )
@@ -30,6 +34,23 @@ if TYPE_CHECKING:
     from litellm.types.utils import ModelResponseStream
 
 
+_STREAMING_DELTA_TYPES = frozenset(get_args(StreamingContentBlockDeltaType))
+
+
+def _delta_payload_field(delta_type: StreamingContentBlockDeltaType) -> str:
+    match delta_type:
+        case "text_delta":
+            return "text"
+        case "input_json_delta":
+            return "partial_json"
+        case "thinking_delta":
+            return "thinking"
+        case "signature_delta":
+            return "signature"
+        case _:
+            assert_never(delta_type)
+
+
 class _CombinedChunkSplitter:
     """
     Splits a streaming chunk that carries BOTH response content and a
@@ -458,12 +479,15 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
 
                     # 3. If the trigger chunk carries delta content, queue it
                     # so the first delta of the new block is not silently dropped.
-                    if self._trigger_delta_has_content(processed_chunk):
+                    if self._delta_has_content(processed_chunk):
                         self.chunk_queue.append(processed_chunk)
 
                     self.sent_content_block_finish = False
                     return self.chunk_queue.popleft()
 
+                if processed_chunk["type"] == "content_block_delta" and not self._delta_has_content(processed_chunk):
+                    continue
+
                 if processed_chunk["type"] == "message_delta" and self.sent_content_block_finish is False:
                     # Queue both the content_block_stop and the message_delta
                     self.chunk_queue.append(
@@ -670,13 +694,18 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
 
                         # 3. If the trigger chunk carries delta content, queue it
                         # so the first delta of the new block is not silently dropped.
-                        if self._trigger_delta_has_content(processed_chunk):
+                        if self._delta_has_content(processed_chunk):
                             self.chunk_queue.append(processed_chunk)
 
                         # Reset state for new block
                         self.sent_content_block_finish = False
                         return self.chunk_queue.popleft()
 
+                    if processed_chunk["type"] == "content_block_delta" and not self._delta_has_content(
+                        processed_chunk
+                    ):
+                        continue
+
                     if processed_chunk["type"] == "message_delta" and self.sent_content_block_finish is False:
                         # Queue both the content_block_stop and the holding chunk
                         self.chunk_queue.append(
@@ -808,20 +837,33 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
         self.current_content_block_index += 1
 
     @staticmethod
-    def _trigger_delta_has_content(processed_chunk: Dict[str, Any]) -> bool:
-        """Return True if a translated trigger chunk carries a non-empty
-        ``content_block_delta`` payload that must be re-emitted after a
-        block transition.
+    def _delta_has_content(processed_chunk: Dict[str, Any]) -> bool:
+        """Return True if a translated chunk carries a non-empty
+        ``content_block_delta`` payload.
 
-        When an upstream chunk both *triggers* a new content block (its type
-        differs from the active block) and *carries* delta content, that
-        content belongs to the new block. The synthesized
-        ``content_block_start`` only ever carries an empty body — see
+        Gates every ``content_block_delta`` emission. An empty delta carries
+        no information, and the translate fallback types empty deltas as
+        ``text_delta`` regardless of the active block's type — emitting one
+        into an open ``thinking`` block (e.g. Bedrock Converse sends an empty
+        reasoning delta mid-block) crashes strict Anthropic SDK clients with
+        "Content block is not a text block".
+
+        Also gates re-emission after a block transition: when an upstream
+        chunk both *triggers* a new content block (its type differs from the
+        active block) and *carries* delta content, that content belongs to
+        the new block. The synthesized ``content_block_start`` only ever
+        carries an empty body — see
         ``_translate_streaming_openai_chunk_to_anthropic_content_block``,
         which returns an empty ``TextBlock``/``ToolUseBlock``/thinking block —
         so the trigger chunk's delta must be re-queued or the first token of
         the new block (the first non-empty text/thinking delta, or bundled
         tool arguments) is silently dropped.
+
+        Delta types outside ``StreamingContentBlockDeltaType`` — the closed
+        set the translate layer can produce — are treated as empty. The
+        per-type payload lookup is exhaustively matched against that set in
+        ``_delta_payload_field``, so extending the translate layer with a new
+        delta type fails type-checking here until it is handled.
         """
         if processed_chunk.get("type") != "content_block_delta":
             return False
@@ -829,15 +871,9 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
         if not isinstance(delta, dict):
             return False
         delta_type = delta.get("type")
-        if delta_type == "text_delta":
-            return bool(delta.get("text"))
-        if delta_type == "input_json_delta":
-            return bool(delta.get("partial_json"))
-        if delta_type == "thinking_delta":
-            return bool(delta.get("thinking"))
-        if delta_type == "signature_delta":
-            return bool(delta.get("signature"))
-        return False
+        if delta_type not in _STREAMING_DELTA_TYPES:
+            return False
+        return bool(delta.get(_delta_payload_field(delta_type)))
 
     def _should_start_new_content_block(self, chunk: "ModelResponseStream") -> bool:
         """

litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py

@@ -104,6 +104,7 @@ from litellm.types.llms.anthropic import (
     ContextManagementResponse,
     MessageBlockDelta,
     MessageDelta,
+    StreamingContentBlockDeltaType,
     UsageDelta,
     UsageIteration,
 )
@@ -1423,7 +1424,7 @@ class LiteLLMAnthropicMessagesAdapter:
     def _translate_streaming_openai_chunk_to_anthropic(
         self, choices: List[Union[OpenAIStreamingChoice, StreamingChoices]]
     ) -> Tuple[
-        Literal["text_delta", "input_json_delta", "thinking_delta", "signature_delta"],
+        StreamingContentBlockDeltaType,
         Union[
             ContentTextBlockDelta,
             ContentJsonBlockDelta,

litellm/llms/anthropic/experimental_pass_through/messages/transformation.py

@@ -375,6 +375,36 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
             else:
                 optional_params.pop("output_config", None)
 
+    @staticmethod
+    def _drop_incompatible_temperature_for_thinking(
+        model: str, optional_params: dict, custom_llm_provider: str
+    ) -> None:
+        """Anthropic rejects any ``temperature`` other than 1 while extended thinking
+        is enabled ("temperature may only be set to 1 when thinking is enabled").
+
+        Clients like Claude Code send ``thinking``/``output_config.effort`` together
+        with a pinned ``temperature`` (e.g. the safety classifier uses ``temperature=0``
+        for determinism). When the request lands on a non-adaptive model, the effort
+        interface is reshaped above into legacy ``thinking={type: enabled}`` (or kept
+        as ``output_config.effort`` on Opus 4.5), and the leftover ``temperature`` would
+        400. Preserving the thinking the caller asked for wins over an unhonorable
+        sampling value (Anthropic forces ``temperature=1`` under thinking regardless),
+        so drop it and let the API default apply.
+
+        Adaptive models (4.6+) own this natively and are left untouched.
+        """
+        if AnthropicModelInfo._is_adaptive_thinking_model(model, custom_llm_provider):
+            return
+        temperature = optional_params.get("temperature")
+        if temperature is None or temperature == 1:
+            return
+        thinking = optional_params.get("thinking")
+        output_config = optional_params.get("output_config")
+        thinking_enabled = isinstance(thinking, dict) and thinking.get("type") == "enabled"
+        effort_enabled = isinstance(output_config, dict) and output_config.get("effort") is not None
+        if thinking_enabled or effort_enabled:
+            optional_params.pop("temperature", None)
+
     def transform_anthropic_messages_request(
         self,
         model: str,
@@ -415,6 +445,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
             custom_llm_provider=self._resolved_provider,
         )
 
+        self._drop_incompatible_temperature_for_thinking(
+            model=model,
+            optional_params=anthropic_messages_optional_request_params,
+            custom_llm_provider=self._resolved_provider,
+        )
+
         system_param = anthropic_messages_optional_request_params.get("system")
         if self.should_strip_billing_metadata() and system_param is not None:
             filtered_system = self._filter_billing_headers_from_system(system_param)

litellm/llms/base_llm/guardrail_translation/base_translation.py

@@ -1,4 +1,5 @@
 from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Any, Dict, List, Optional
 
 if TYPE_CHECKING:
@@ -11,10 +12,30 @@ if TYPE_CHECKING:
     from litellm.types.llms.openai import AllMessageValues
 
 
+@dataclass(slots=True)
+class StreamTransformSink:
+    """Out-parameter used by ``process_output_streaming_response`` to hand the
+    guardrailed streaming state back to the caller.
+
+    The streaming text-transform path must not mutate ``responses_so_far`` (it is
+    the raw accumulator the guardrail re-reads every round), so the guardrailed
+    accumulated text per choice (``mutated_text_per_choice``, keyed by
+    ``StreamingChoices.index``) and the per-choice trailing holdback the guardrail
+    requested (``holdback_per_choice``, from ``stream_holdback_chars``) are
+    reported here instead of in place. Only the OpenAI chat handler populates this
+    today; the hook passes a fresh sink per round and reads it afterwards. A
+    mutable dataclass is deliberate: it is a write-once output parameter for a
+    single call, not shared state.
+    """
+
+    mutated_text_per_choice: dict[int, str] = field(default_factory=dict)
+    holdback_per_choice: dict[int, int] = field(default_factory=dict)
+
+
 class BaseTranslation(ABC):
     @staticmethod
     def transform_user_api_key_dict_to_metadata(
-        user_api_key_dict: Optional[Any],
+        user_api_key_dict: Any | None,
     ) -> Dict[str, Any]:
         """
         Transform user_api_key_dict to a metadata dict with prefixed keys.
@@ -73,7 +94,7 @@ class BaseTranslation(ABC):
         guardrail_to_apply: "CustomGuardrail",
         litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None,
         user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
-        request_data: Optional[dict] = None,
+        request_data: dict | None = None,
     ) -> Any:
         """
         Process output response with guardrails.
@@ -92,12 +113,15 @@ class BaseTranslation(ABC):
         guardrail_to_apply: "CustomGuardrail",
         litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None,
         user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
-        request_data: Optional[dict] = None,
+        request_data: dict | None = None,
+        stream_transform_sink: StreamTransformSink | None = None,
     ) -> Any:
         """
         Process output streaming response with guardrails.
 
-        Optional to override in subclasses.
+        Optional to override in subclasses. ``stream_transform_sink`` is the
+        out-parameter used by handlers that support streaming text
+        transformations (see ``StreamTransformSink``); base handlers ignore it.
         """
         return responses_so_far
 
@@ -105,8 +129,8 @@ class BaseTranslation(ABC):
         self,
         exc: "ModifyResponseException",
         stream_started: bool = False,
-        responses_so_far: Optional[list[Any]] = None,
-    ) -> Optional[list[bytes]]:
+        responses_so_far: list[Any] | None = None,
+    ) -> list[bytes] | None:
         """
         Build the streaming chunks that deliver a guardrail block message and
         cleanly terminate the stream in this provider's wire format.
@@ -125,7 +149,7 @@ class BaseTranslation(ABC):
         """
         return None
 
-    def get_structured_messages(self, data: dict) -> Optional[List["AllMessageValues"]]:
+    def get_structured_messages(self, data: dict) -> List["AllMessageValues"] | None:
         """
         Convert request data to OpenAI-spec structured messages.
 

litellm/llms/custom_httpx/llm_http_handler.py

@@ -1879,6 +1879,7 @@ class BaseLLMHTTPHandler:
         litellm_params: GenericLiteLLMParams,
         api_key: Optional[str],
         model: str,
+        timeout: Optional[Union[float, httpx.Timeout]] = None,
     ) -> httpx.Response:
         max_attempts = max(provider_config.max_retry_on_anthropic_messages_http_error, 1)
         litellm_params_dict = dict(litellm_params)
@@ -1891,6 +1892,7 @@ class BaseLLMHTTPHandler:
                     data=signed_json_body or json.dumps(request_body),
                     stream=stream or False,
                     logging_obj=logging_obj,
+                    timeout=timeout,
                 )
                 response.raise_for_status()
                 return response
@@ -1925,6 +1927,32 @@ class BaseLLMHTTPHandler:
 
         raise RuntimeError("unreachable: anthropic messages HTTP retry loop exited without return")
 
+    @staticmethod
+    def _resolve_anthropic_messages_timeout(
+        litellm_params: GenericLiteLLMParams,
+        stream: bool,
+        custom_llm_provider: str,
+    ) -> Optional[Union[float, httpx.Timeout]]:
+        from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
+        from litellm.litellm_core_utils.request_timeout_resolver import (
+            get_configured_request_timeout,
+        )
+        from litellm.utils import supports_httpx_timeout
+
+        stream_timeout = litellm_params.get("stream_timeout") if stream else None
+        model_timeout = stream_timeout if stream_timeout is not None else litellm_params.get("timeout")
+        request_timeout = litellm_params.get("request_timeout")
+        global_timeout = get_configured_request_timeout()
+        if model_timeout is None and request_timeout is None and global_timeout is None:
+            return None
+        return CompletionTimeout.resolve(
+            model_timeout,
+            {"request_timeout": request_timeout},
+            custom_llm_provider,
+            global_timeout=global_timeout,
+            supports_httpx_timeout=supports_httpx_timeout,
+        )
+
     async def async_anthropic_messages_handler(
         self,
         model: str,
@@ -2075,6 +2103,11 @@ class BaseLLMHTTPHandler:
             litellm_params=litellm_params,
             api_key=api_key,
             model=model,
+            timeout=self._resolve_anthropic_messages_timeout(
+                litellm_params=litellm_params,
+                stream=stream or False,
+                custom_llm_provider=custom_llm_provider,
+            ),
         )
 
         # used for logging + cost tracking
@@ -2657,9 +2690,10 @@ class BaseLLMHTTPHandler:
         )
 
         result = final_response if final_response is not None else initial_response
-        if litellm_params.get("_code_interpreter_interception_converted_stream") and not litellm_params.get(
-            "_agentic_loop_depth"
-        ):
+        interception_converted_stream = litellm_params.get(
+            "_code_interpreter_interception_converted_stream"
+        ) or litellm_params.get("_websearch_interception_converted_stream")
+        if interception_converted_stream and not litellm_params.get("_agentic_loop_depth"):
             return self._wrap_responses_response_as_fake_stream(
                 result=result,
                 model=model,
@@ -5224,6 +5258,8 @@ class BaseLLMHTTPHandler:
         tools = anthropic_messages_optional_request_params.get("tools", [])
         depth, max_loops, fingerprints = self._get_agentic_loop_settings(kwargs=kwargs)
 
+        hook_kwargs = {**kwargs, "_agentic_loop_api_surface": api_surface}
+
         for callback in callbacks:
             if not isinstance(callback, CustomLogger):
                 continue
@@ -5244,7 +5280,7 @@ class BaseLLMHTTPHandler:
                     tools=tools,
                     stream=stream,
                     custom_llm_provider=custom_llm_provider,
-                    kwargs=kwargs,
+                    kwargs=hook_kwargs,
                 )
             except Exception as e:
                 _call_id = getattr(logging_obj, "litellm_call_id", "unknown")
@@ -5270,7 +5306,7 @@ class BaseLLMHTTPHandler:
             )
 
             try:
-                kwargs_with_provider = kwargs.copy() if kwargs else {}
+                kwargs_with_provider = hook_kwargs.copy()
                 kwargs_with_provider["custom_llm_provider"] = custom_llm_provider
                 build_plan_overridden = (
                     callback.__class__.async_build_agentic_loop_plan is not CustomLogger.async_build_agentic_loop_plan

litellm/llms/openai/chat/guardrail_translation/handler.py

@@ -14,11 +14,14 @@ Pattern Overview:
 This pattern can be replicated for other message formats (e.g., Anthropic).
 """
 
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
+from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Union, cast
 
 import litellm
 from litellm._logging import verbose_proxy_logger
-from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
+from litellm.llms.base_llm.guardrail_translation.base_translation import (
+    BaseTranslation,
+    StreamTransformSink,
+)
 from litellm.llms.base_llm.guardrail_translation.utils import (
     effective_skip_system_message_for_guardrail,
     effective_skip_tool_message_for_guardrail,
@@ -27,6 +30,9 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
 )
 from litellm.main import stream_chunk_builder
 from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
+from litellm.types.proxy.guardrails.guardrail_hooks.generic_guardrail_api import (
+    coerce_stream_holdback_value,
+)
 from litellm.types.utils import (
     Choices,
     GenericGuardrailAPIInputs,
@@ -50,7 +56,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
     Methods can be overridden to customize behavior for different message formats.
     """
 
-    def get_structured_messages(self, data: dict) -> Optional[List[AllMessageValues]]:
+    def get_structured_messages(self, data: dict) -> List[AllMessageValues] | None:
         """
         Convert chat completions request data to OpenAI-spec structured messages.
 
@@ -65,7 +71,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         self,
         data: dict,
         guardrail_to_apply: "CustomGuardrail",
-        litellm_logging_obj: Optional[Any] = None,
+        litellm_logging_obj: Any | None = None,
     ) -> Any:
         """
         Process input messages by applying guardrails to text content.
@@ -80,7 +86,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         texts_to_check: List[str] = []
         images_to_check: List[str] = []
         tool_calls_to_check: List[ChatCompletionToolParam] = []
-        text_task_mappings: List[Tuple[int, Optional[int]]] = []
+        text_task_mappings: List[Tuple[int, int | None]] = []
         tool_call_task_mappings: List[Tuple[int, int]] = []
 
         # Step 1: Extract all text content, images, and tool calls
@@ -184,7 +190,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         texts_to_check: List[str],
         images_to_check: List[str],
         tool_calls_to_check: List[ChatCompletionToolParam],
-        text_task_mappings: List[Tuple[int, Optional[int]]],
+        text_task_mappings: List[Tuple[int, int | None]],
         tool_call_task_mappings: List[Tuple[int, int]],
         skip_system_message: bool = False,
         skip_tool_message: bool = False,
@@ -239,7 +245,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         self,
         messages: List[Dict[str, Any]],
         responses: List[str],
-        task_mappings: List[Tuple[int, Optional[int]]],
+        task_mappings: List[Tuple[int, int | None]],
     ) -> None:
         """
         Apply guardrail responses back to input message text content.
@@ -249,7 +255,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         for task_idx, guardrail_response in enumerate(responses):
             mapping = task_mappings[task_idx]
             msg_idx = cast(int, mapping[0])
-            content_idx_optional = cast(Optional[int], mapping[1])
+            content_idx_optional = cast(int | None, mapping[1])
 
             # Handle content
             content = messages[msg_idx].get("content", None)
@@ -291,9 +297,9 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         self,
         response: "ModelResponse",
         guardrail_to_apply: "CustomGuardrail",
-        litellm_logging_obj: Optional[Any] = None,
-        user_api_key_dict: Optional[Any] = None,
-        request_data: Optional[dict] = None,
+        litellm_logging_obj: Any | None = None,
+        user_api_key_dict: Any | None = None,
+        request_data: dict | None = None,
     ) -> Any:
         """
         Process output response by applying guardrails to text content.
@@ -320,7 +326,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         texts_to_check: List[str] = []
         images_to_check: List[str] = []
         tool_calls_to_check: List[Dict[str, Any]] = []
-        text_task_mappings: List[Tuple[int, Optional[int]]] = []
+        text_task_mappings: List[Tuple[int, int | None]] = []
         tool_call_task_mappings: List[Tuple[int, int]] = []
         # text_task_mappings: Track (choice_index, content_index) for each text
         # content_index is None for string content, int for list content
@@ -402,9 +408,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         self,
         responses_so_far: List["ModelResponseStream"],
         guardrail_to_apply: "CustomGuardrail",
-        litellm_logging_obj: Optional[Any] = None,
-        user_api_key_dict: Optional[Any] = None,
-        request_data: Optional[dict] = None,
+        litellm_logging_obj: Any | None = None,
+        user_api_key_dict: Any | None = None,
+        request_data: dict | None = None,
+        stream_transform_sink: StreamTransformSink | None = None,
     ) -> List["ModelResponseStream"]:
         """
         Process output streaming responses by applying guardrails to text content.
@@ -414,14 +421,50 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
             guardrail_to_apply: The guardrail instance to apply
             litellm_logging_obj: Optional logging object
             user_api_key_dict: User API key metadata to pass to guardrails
+            stream_transform_sink: Optional out-parameter for the streaming text
+                transformation path. When provided, the guardrail runs over the raw
+                accumulated text (``responses_so_far`` is left untouched so it stays
+                a correct raw accumulator across rounds) and the guardrailed text
+                plus requested holdback are reported per choice on the sink.
 
         Returns:
-            Modified list of responses with guardrail applied to content
+            The (unmodified) list of responses.
 
         Response Format Support:
             - String content: choice.message.content = "text here"
             - List content: choice.message.content = [{"type": "text", "text": "text here"}, ...]
         """
+        if stream_transform_sink is not None:
+            await self._process_streaming_transform(
+                responses_so_far=responses_so_far,
+                guardrail_to_apply=guardrail_to_apply,
+                litellm_logging_obj=litellm_logging_obj,
+                user_api_key_dict=user_api_key_dict,
+                request_data=request_data,
+                sink=stream_transform_sink,
+            )
+            return responses_so_far
+
+        return await self._process_streaming_block_only(
+            responses_so_far=responses_so_far,
+            guardrail_to_apply=guardrail_to_apply,
+            litellm_logging_obj=litellm_logging_obj,
+            user_api_key_dict=user_api_key_dict,
+            request_data=request_data,
+        )
+
+    async def _process_streaming_block_only(
+        self,
+        *,
+        responses_so_far: list["ModelResponseStream"],
+        guardrail_to_apply: "CustomGuardrail",
+        litellm_logging_obj: Any | None,
+        user_api_key_dict: Any | None,
+        request_data: dict | None,
+    ) -> list["ModelResponseStream"]:
+        """Block-only streaming path: run the guardrail so an in-flight BLOCK can
+        terminate the stream. Text rewrites are not propagated to the client here
+        (see ``_process_streaming_transform`` for the incremental_diff path)."""
         # check if the stream has ended
         has_stream_ended = False
         for chunk in responses_so_far:
@@ -467,7 +510,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         # Step 2: Create lists for guardrail processing
         texts_to_check: List[str] = []
         images_to_check: List[str] = []
-        task_mappings: List[Tuple[int, Optional[int]]] = []
+        task_mappings: List[Tuple[int, int | None]] = []
         # Track (choice_index, content_index) for each combined text
 
         for (map_choice_idx, map_content_idx), combined_text in combined_texts.items():
@@ -520,9 +563,109 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
 
         return responses_so_far
 
+    @staticmethod
+    def _accumulate_string_content_by_choice_index(
+        responses_so_far: list["ModelResponseStream"],
+    ) -> dict[int, str]:
+        """Accumulate raw string ``delta.content`` per choice, keyed by
+        ``StreamingChoices.index`` (not enumerate position, which collapses to 0
+        when each chunk carries a single non-zero-indexed choice for ``n > 1``).
+
+        Only string content participates; list-of-blocks content is out of scope
+        for the incremental transform path. Reads ``responses_so_far`` without
+        mutating it so it stays a correct raw accumulator across rounds.
+        """
+        accumulated: dict[int, str] = {}
+        for response in responses_so_far:
+            for choice in response.choices:
+                if isinstance(choice, litellm.StreamingChoices):
+                    content = choice.delta.content
+                elif isinstance(choice, litellm.Choices):
+                    content = choice.message.content
+                else:
+                    continue
+                if isinstance(content, str) and content:
+                    idx = getattr(choice, "index", 0) or 0
+                    accumulated[idx] = accumulated.get(idx, "") + content
+        return accumulated
+
+    async def _process_streaming_transform(
+        self,
+        *,
+        responses_so_far: list["ModelResponseStream"],
+        guardrail_to_apply: "CustomGuardrail",
+        litellm_logging_obj: Any | None,
+        user_api_key_dict: Any | None,
+        request_data: dict | None,
+        sink: StreamTransformSink,
+    ) -> None:
+        """Run the guardrail over the raw accumulated text and report the
+        guardrailed text plus requested holdback per choice on ``sink``.
+
+        Unlike the block-only path this never mutates ``responses_so_far``: it
+        re-derives the raw accumulated text every round (so a rewrite guardrail
+        always sees consistent input) and hands the result back out of band.
+        """
+        raw_by_index = self._accumulate_string_content_by_choice_index(responses_so_far)
+        if not raw_by_index:
+            sink.mutated_text_per_choice = {}
+            sink.holdback_per_choice = {}
+            return
+
+        # Fix #2 — sort by StreamingChoices.index so an n>1 stream that emits
+        # choice 1 before choice 0 still hands the guardrail texts in a
+        # deterministic index order. Without this, the guardrail's returned
+        # texts (aligned to the input order it received) would map back to the
+        # wrong choice indices when we rebuild the sink dicts by
+        # ``enumerate(indices)``.
+        indices = sorted(raw_by_index.keys())
+        texts_to_check = [raw_by_index[i] for i in indices]
+
+        if request_data is None:
+            request_data = {"responses": responses_so_far}
+        elif "responses" not in request_data:
+            request_data["responses"] = responses_so_far
+        if "litellm_metadata" not in request_data:
+            user_metadata = self.transform_user_api_key_dict_to_metadata(user_api_key_dict)
+            if user_metadata:
+                request_data["litellm_metadata"] = user_metadata
+
+        inputs = GenericGuardrailAPIInputs(texts=texts_to_check)
+        if responses_so_far and getattr(responses_so_far[0], "model", None):
+            inputs["model"] = responses_so_far[0].model
+        guardrailed_inputs = await guardrail_to_apply.apply_guardrail(
+            inputs=inputs,
+            request_data=request_data,
+            input_type="response",
+            logging_obj=litellm_logging_obj,
+        )
+
+        returned_texts = guardrailed_inputs.get("texts")
+        # No "texts" key means the guardrail made no change (action NONE): the raw
+        # accumulated text is the guardrailed text. A present-but-shorter list is a
+        # guardrail contract violation; those choices are omitted below (withheld,
+        # not emitted raw) so a malformed response fails closed instead of leaking.
+        if returned_texts is None:
+            returned_texts = texts_to_check
+        elif len(returned_texts) < len(texts_to_check):
+            verbose_proxy_logger.warning(
+                "OpenAI Chat Completions: guardrail returned %s transformed texts for %s inputs on the "
+                "streaming transform path; withholding the unmatched choices to fail closed.",
+                len(returned_texts),
+                len(texts_to_check),
+            )
+
+        holdback = guardrailed_inputs.get("stream_holdback_chars") or []
+        sink.mutated_text_per_choice = {
+            idx: returned_texts[i] for i, idx in enumerate(indices) if i < len(returned_texts)
+        }
+        sink.holdback_per_choice = {
+            indices[i]: coerce_stream_holdback_value(holdback[i]) for i in range(len(indices)) if i < len(holdback)
+        }
+
     def _combine_streaming_texts(
         self, responses_so_far: List["ModelResponseStream"]
-    ) -> Dict[Tuple[int, Optional[int]], str]:
+    ) -> Dict[Tuple[int, int | None], str]:
         """
         Combine all streaming chunks into complete text per choice.
 
@@ -534,7 +677,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         Returns:
             Dict mapping (choice_idx, content_idx) to combined text string
         """
-        combined_texts: Dict[Tuple[int, Optional[int]], str] = {}
+        combined_texts: Dict[Tuple[int, int | None], str] = {}
 
         for response_idx, response in enumerate(responses_so_far):
             for choice_idx, choice in enumerate(response.choices):
@@ -550,7 +693,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
 
                 if isinstance(content, str):
                     # String content - accumulate for this choice
-                    str_key: Tuple[int, Optional[int]] = (choice_idx, None)
+                    str_key: Tuple[int, int | None] = (choice_idx, None)
                     if str_key not in combined_texts:
                         combined_texts[str_key] = ""
                     combined_texts[str_key] += content
@@ -560,7 +703,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
                     for content_idx, content_item in enumerate(content):
                         text_str = content_item.get("text")
                         if text_str:
-                            list_key: Tuple[int, Optional[int]] = (
+                            list_key: Tuple[int, int | None] = (
                                 choice_idx,
                                 content_idx,
                             )
@@ -607,7 +750,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         texts_to_check: List[str],
         images_to_check: List[str],
         tool_calls_to_check: List[Dict[str, Any]],
-        text_task_mappings: List[Tuple[int, Optional[int]]],
+        text_task_mappings: List[Tuple[int, int | None]],
         tool_call_task_mappings: List[Tuple[int, int]],
     ) -> None:
         """
@@ -619,7 +762,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
 
         # Determine content source and tool calls based on choice type
         content = None
-        tool_calls: Optional[List[Any]] = None
+        tool_calls: List[Any] | None = None
         if isinstance(choice, litellm.Choices):
             content = choice.message.content
             tool_calls = choice.message.tool_calls
@@ -662,7 +805,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
                     tool_calls_to_check.append(tool_call_dict)
                     tool_call_task_mappings.append((choice_idx, int(tool_call_idx)))
 
-    def _convert_tool_call_to_dict(self, tool_call: Union[Dict[str, Any], Any]) -> Optional[Dict[str, Any]]:
+    def _convert_tool_call_to_dict(self, tool_call: Union[Dict[str, Any], Any]) -> Dict[str, Any] | None:
         """
         Convert a tool call object to dictionary format.
 
@@ -691,7 +834,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         self,
         response: "ModelResponse",
         responses: List[str],
-        task_mappings: List[Tuple[int, Optional[int]]],
+        task_mappings: List[Tuple[int, int | None]],
     ) -> None:
         """
         Apply guardrail text responses back to output response.
@@ -701,7 +844,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         for task_idx, guardrail_response in enumerate(responses):
             mapping = task_mappings[task_idx]
             choice_idx = cast(int, mapping[0])
-            content_idx_optional = cast(Optional[int], mapping[1])
+            content_idx_optional = cast(int | None, mapping[1])
 
             choice = cast(Choices, response.choices[choice_idx])
 
@@ -755,7 +898,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         self,
         responses: List["ModelResponseStream"],
         guardrailed_texts: List[str],
-        task_mappings: List[Tuple[int, Optional[int]]],
+        task_mappings: List[Tuple[int, int | None]],
     ) -> None:
         """
         Apply guardrail responses back to output streaming responses.
@@ -771,16 +914,16 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         Override this method to customize how responses are applied to streaming responses.
         """
         # Build a mapping of what guardrailed text to use for each (choice_idx, content_idx)
-        guardrail_map: Dict[Tuple[int, Optional[int]], str] = {}
+        guardrail_map: Dict[Tuple[int, int | None], str] = {}
         for task_idx, guardrail_response in enumerate(guardrailed_texts):
             mapping = task_mappings[task_idx]
             choice_idx = cast(int, mapping[0])
-            content_idx_optional = cast(Optional[int], mapping[1])
+            content_idx_optional = cast(int | None, mapping[1])
             guardrail_map[(choice_idx, content_idx_optional)] = guardrail_response
 
         # Track which choices we've already set the guardrailed text for
         # Key: (choice_idx, content_idx), Value: boolean (True if already set)
-        already_set: Dict[Tuple[int, Optional[int]], bool] = {}
+        already_set: Dict[Tuple[int, int | None], bool] = {}
 
         # Iterate through all responses and update content
         for response_idx, response in enumerate(responses):
@@ -797,7 +940,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
 
                 if isinstance(content, str):
                     # String content
-                    str_key: Tuple[int, Optional[int]] = (choice_idx_in_response, None)
+                    str_key: Tuple[int, int | None] = (choice_idx_in_response, None)
                     if str_key in guardrail_map:
                         if str_key not in already_set:
                             # First chunk - set the complete guardrailed text
@@ -817,7 +960,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
                     # List content - handle each content item
                     for content_idx, content_item in enumerate(content):
                         if "text" in content_item:
-                            list_key: Tuple[int, Optional[int]] = (
+                            list_key: Tuple[int, int | None] = (
                                 choice_idx_in_response,
                                 content_idx,
                             )

litellm/llms/openai/responses/guardrail_translation/handler.py

@@ -247,6 +247,8 @@ class OpenAIResponsesHandler(BaseTranslation):
         """
         Merge remapped guardrailed tools with original tools that were not sent
         to the guardrail (e.g. web_search, web_search_preview), preserving order.
+        Tools a guardrail appended (``remapped`` longer than ``original_tools``)
+        have no original slot and are kept so an injected tool is not dropped.
         """
         if not original_tools:
             return remapped
@@ -262,6 +264,8 @@ class OpenAIResponsesHandler(BaseTranslation):
                 if j < len(remapped):
                     result.append(remapped[j])
                     j += 1
+        # Keep guardrail-appended tools that matched no original slot above.
+        result.extend(remapped[j:])
         return result
 
     def _apply_guardrailed_tools_to_data(

litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py

@@ -998,6 +998,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
                 response_modalities.append("IMAGE")
             elif modality == "audio":
                 response_modalities.append("AUDIO")
+            elif modality == "video":
+                response_modalities.append("VIDEO")
             else:
                 response_modalities.append("MODALITY_UNSPECIFIED")
         return response_modalities

litellm/model_prices_and_context_window_backup.json

@@ -11331,6 +11331,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_native_structured_output": true,
         "supports_tool_choice": true,
         "supports_vision": true
     },
@@ -11362,6 +11363,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_native_structured_output": true,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_web_search": true
@@ -11424,6 +11426,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_native_structured_output": true,
         "supports_max_reasoning_effort": true,
         "supports_tool_choice": true,
         "supports_vision": true,
@@ -11479,6 +11482,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_native_structured_output": true,
         "supports_tool_choice": true,
         "supports_vision": true
     },
@@ -11506,6 +11510,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_native_structured_output": true,
         "supports_tool_choice": true,
         "supports_vision": true
     },
@@ -11559,6 +11564,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_native_structured_output": true,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_output_config": true
@@ -11586,6 +11592,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_native_structured_output": true,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_output_config": true
@@ -11614,6 +11621,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_native_structured_output": true,
         "supports_tool_choice": true,
         "supports_vision": true,
         "provider_specific_entry": {
@@ -11648,6 +11656,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_native_structured_output": true,
         "supports_tool_choice": true,
         "supports_vision": true,
         "provider_specific_entry": {
@@ -11682,6 +11691,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_native_structured_output": true,
         "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
@@ -11718,6 +11728,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_native_structured_output": true,
         "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
@@ -11788,6 +11799,7 @@
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
+        "supports_native_structured_output": true,
         "supports_sampling_params": false,
         "supports_tool_choice": true,
         "supports_vision": true,
@@ -19600,6 +19612,39 @@
         },
         "web_search_billing_unit": "per_query"
     },
+    "gemini/gemini-omni-flash-preview": {
+        "input_cost_per_audio_token": 1.5e-06,
+        "input_cost_per_token": 1.5e-06,
+        "litellm_provider": "gemini",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65535,
+        "max_tokens": 65535,
+        "mode": "chat",
+        "output_cost_per_reasoning_token": 9e-06,
+        "output_cost_per_token": 9e-06,
+        "output_cost_per_video_token": 1.75e-05,
+        "rpm": 2000,
+        "source": "https://ai.google.dev/gemini-api/docs/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "audio",
+            "video"
+        ],
+        "supported_output_modalities": [
+            "text",
+            "video"
+        ],
+        "supports_audio_input": true,
+        "supports_reasoning": true,
+        "supports_system_messages": true,
+        "supports_video_input": true,
+        "supports_vision": true,
+        "tpm": 800000
+    },
     "gemini/gemini-3.1-pro-preview": {
         "cache_read_input_token_cost": 2e-07,
         "cache_read_input_token_cost_above_200k_tokens": 4e-07,
@@ -19764,6 +19809,37 @@
         },
         "web_search_billing_unit": "per_query"
     },
+    "gemini-omni-flash-preview": {
+        "input_cost_per_audio_token": 1.5e-06,
+        "input_cost_per_token": 1.5e-06,
+        "litellm_provider": "vertex_ai-language-models",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65535,
+        "max_tokens": 65535,
+        "mode": "chat",
+        "output_cost_per_reasoning_token": 9e-06,
+        "output_cost_per_token": 9e-06,
+        "output_cost_per_video_token": 1.75e-05,
+        "source": "https://docs.cloud.google.com/gemini-enterprise-agent-platform/models/gemini/omni-flash-preview",
+        "supported_endpoints": [
+            "/v1/chat/completions"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "audio",
+            "video"
+        ],
+        "supported_output_modalities": [
+            "text",
+            "video"
+        ],
+        "supports_audio_input": true,
+        "supports_reasoning": true,
+        "supports_system_messages": true,
+        "supports_video_input": true,
+        "supports_vision": true
+    },
     "gemini-3.5-flash": {
         "cache_read_input_token_cost": 1.5e-07,
         "input_cost_per_audio_token": 1e-06,
@@ -44185,6 +44261,90 @@
         "supports_response_schema": true,
         "supports_tool_choice": true
     },
+    "bedrock_mantle/openai.gpt-5.6-sol": {
+        "input_cost_per_token": 5.5e-06,
+        "cache_creation_input_token_cost": 6.875e-06,
+        "cache_read_input_token_cost": 5.5e-07,
+        "output_cost_per_token": 3.3e-05,
+        "litellm_provider": "bedrock_mantle",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "responses",
+        "use_openai_responses_path": true,
+        "supported_endpoints": [
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "bedrock_mantle/openai.gpt-5.6-terra": {
+        "input_cost_per_token": 2.75e-06,
+        "cache_creation_input_token_cost": 3.4375e-06,
+        "cache_read_input_token_cost": 2.75e-07,
+        "output_cost_per_token": 1.65e-05,
+        "litellm_provider": "bedrock_mantle",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "responses",
+        "use_openai_responses_path": true,
+        "supported_endpoints": [
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "bedrock_mantle/openai.gpt-5.6-luna": {
+        "input_cost_per_token": 1.1e-06,
+        "cache_creation_input_token_cost": 1.375e-06,
+        "cache_read_input_token_cost": 1.1e-07,
+        "output_cost_per_token": 6.6e-06,
+        "litellm_provider": "bedrock_mantle",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "responses",
+        "use_openai_responses_path": true,
+        "supported_endpoints": [
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
     "bedrock_mantle/openai.gpt-5.5": {
         "input_cost_per_token": 5.5e-06,
         "cache_read_input_token_cost": 5.5e-07,
@@ -44297,6 +44457,7 @@
         "supports_vision": true
     },
     "bedrock_mantle/xai.grok-4.3": {
+        "use_openai_responses_path": true,
         "input_cost_per_token": 1.25e-06,
         "output_cost_per_token": 2.5e-06,
         "cache_read_input_token_cost": 2e-07,

litellm/models/verification_token.py

@@ -36,6 +36,7 @@ class LiteLLM_VerificationToken(LiteLLMPydanticObjectBase):
     budget_reset_at: Optional[datetime] = None
     allowed_cache_controls: Optional[list] = []
     allowed_routes: Optional[list] = []
+    key_type: str | None = None
     permissions: Dict = {}
     model_spend: Dict = {}
     model_max_budget: Dict = {}

litellm/proxy/_experimental/mcp_server/bridge_token_flow.py

@@ -0,0 +1,694 @@
+"""Bridge token flow: litellm identity resolution and the DCR-bridge oauth_delegate mint/refresh pipeline."""
+
+import math
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING, Literal, Optional
+
+from fastapi import HTTPException, Request
+from fastapi.responses import JSONResponse
+from pydantic import SecretStr
+from typing_extensions import assert_never
+
+from litellm._logging import verbose_logger
+from litellm.proxy._experimental.mcp_server.oauth_utils import TOKEN_NO_CACHE_HEADERS
+from litellm.types.mcp_server.mcp_server_manager import MCPServer
+
+if TYPE_CHECKING:
+    from litellm.proxy._experimental.mcp_server.discoverable_endpoints import _BridgeAuthorizationCode
+    from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import (
+        EnvelopeIdentity,
+        EnvelopeKeys,
+        RefreshCredential,
+        UpstreamTokenGrant,
+    )
+    from litellm.proxy._types import UserAPIKeyAuth
+
+
+def _litellm_key_from_request(request: Request) -> Optional[str]:
+    """Return the LiteLLM API key presented on the request, or ``None``.
+
+    Accepts the key from ``x-litellm-api-key`` (what MCP clients such as Claude Desktop/Code
+    send) as well as ``Authorization``; either may carry a bare token or ``Bearer <token>``.
+    ``x-litellm-api-key`` wins when both are present, since ``Authorization`` may instead carry
+    an OAuth/upstream bearer.
+    """
+    for header_value in (
+        request.headers.get("x-litellm-api-key"),
+        request.headers.get("Authorization") or request.headers.get("authorization"),
+    ):
+        if not header_value:
+            continue
+        value = header_value.strip()
+        if value.lower().startswith("bearer "):
+            value = value[7:].strip()
+        if value:
+            return value
+    return None
+
+
+def _key_is_active(key_obj: "UserAPIKeyAuth") -> bool:
+    """``True`` when the presented key is neither blocked nor past its expiry.
+
+    The OAuth token endpoint is unauthenticated, so the presented key is validated here before it is
+    trusted; a revoked or expired key must not mint a bridge envelope or write a stored credential.
+    ``get_key_object`` resolves a row without these checks (the main ``user_api_key_auth`` pipeline
+    enforces them downstream, which this endpoint bypasses), so they are applied here. Deleted keys
+    are already rejected upstream, where ``get_key_object`` raises on a row that no longer exists.
+
+    This is an active-state gate only; it deliberately does not require a ``user_id``. A valid
+    team-scoped or service-account key has no ``user_id`` yet is a legitimate credential, so gating
+    on ``user_id`` presence would wrongly reject it. Callers that need the user (the per-user token
+    store) derive it separately via :func:`_active_key_user_id`.
+
+    Total by design: ``expires`` is typed ``str | datetime``, and an unparseable string would make
+    ``datetime.fromisoformat`` raise. Since the callers run this outside their key-resolution
+    ``try``, an uncaught parse error would surface as a 500 instead of the endpoint's fail-closed
+    behavior, so a malformed expiry is treated as inactive (return ``False``) rather than raising.
+    """
+    if key_obj.blocked is True:
+        return False
+    expires = key_obj.expires
+    if expires is not None:
+        if isinstance(expires, datetime):
+            expiry = expires
+        else:
+            try:
+                expiry = datetime.fromisoformat(expires)
+            except (ValueError, TypeError):
+                return False
+        if expiry.tzinfo is None or expiry.tzinfo.utcoffset(expiry) is None:
+            expiry = expiry.replace(tzinfo=timezone.utc)
+        if expiry < datetime.now(timezone.utc):
+            return False
+    return True
+
+
+def _active_key_user_id(key_obj: "UserAPIKeyAuth") -> str | None:
+    """The active key's ``user_id``, or ``None`` when the key is blocked/expired or simply has no
+    ``user_id`` (a team-scoped or service-account key). Used only by the per-user token store, which
+    needs a user to key the stored credential; the bridge mint uses the key hash and does not."""
+    return key_obj.user_id if _key_is_active(key_obj) else None
+
+
+@dataclass(frozen=True, slots=True)
+class _ResolvedKey:
+    """An active litellm key resolved from the token request: its hash (the value ``get_key_object``
+    and the cache/DB layer key the record by) and the live record."""
+
+    key_hash: str
+    key: "UserAPIKeyAuth"
+
+
+_KeyResolutionFailure = Literal["no_active_key", "unavailable", "unresolvable"]
+"""Why a token request yielded no active litellm key, kept distinct so a caller statuses each truthfully
+instead of blaming the client for a gateway problem:
+- ``no_active_key``: none was presented, or the presented key is unknown / blocked / expired (the
+  caller's request is at fault)
+- ``unavailable``: the auth database was transiently unreachable while resolving (retryable)
+- ``unresolvable``: the gateway cannot resolve identity right now (no DB connection, or an unexpected
+  error) -- a gateway fault, not the caller's
+The classification mirrors admission's ``_reload_admitted_key`` so the mint (ingress) and admission
+(egress) never disagree on the status of the same outage."""
+
+
+async def _resolve_active_litellm_key(request: Request) -> "_ResolvedKey | _KeyResolutionFailure":
+    """Resolve the presented litellm key to an active key record, or say precisely why not.
+
+    Single resolution path the OAuth token endpoint reuses, resolving authoritatively via
+    ``get_key_object`` (cache first, then DB). The failure is a value, not a bare ``None``, so a caller
+    can tell "the client sent no usable credential" (a request error) apart from "the gateway could not
+    check" (an infrastructure error) and status each truthfully; collapsing both to ``None`` is what let
+    a DB outage read as a 400. A resolved key is still gated by ``_key_is_active``, so a blocked or
+    expired key is ``no_active_key`` while a valid team-scoped or service-account key (no ``user_id``)
+    resolves. Classification mirrors admission's ``_reload_admitted_key``: no DB connection is a gateway
+    fault, a ``ProxyException`` / ``HTTPException`` from ``get_key_object`` is an unknown or invalid key,
+    a database-service-unavailable error is a retryable outage, and anything else is an unexpected
+    gateway fault."""
+    token = _litellm_key_from_request(request)
+    if not token:
+        return "no_active_key"
+    from litellm.proxy._types import hash_token  # noqa: PLC0415  # inline import avoids a module-load circular import
+
+    return await _reload_active_key_by_hash(hash_token(token))
+
+
+async def _reload_active_key_by_hash(key_hash: str) -> "_ResolvedKey | _KeyResolutionFailure":
+    """Reload the live key record for ``key_hash`` (cache first, then DB) and gate it on active state,
+    returning the resolved key or a precise failure. Shared by the token request's presented-key
+    resolution (:func:`_resolve_active_litellm_key`, which hashes the presented key) and the refresh
+    path (which already holds the hash sealed in the refresh envelope), so both re-validate identity
+    through one active-key gate and one failure classification. Classification mirrors admission's
+    ``_reload_admitted_key``: no DB connection is a gateway fault, a ``ProxyException`` / ``HTTPException``
+    from ``get_key_object`` is an unknown or invalid key, a database-service-unavailable error is a
+    retryable outage, and anything else is an unexpected gateway fault. A blocked or expired key is
+    ``no_active_key``, so a revoked key can neither mint nor refresh a bridge envelope."""
+    from litellm.proxy._types import (
+        ProxyException,  # noqa: PLC0415  # inline import avoids a module-load circular import
+    )
+    from litellm.proxy.auth.auth_checks import (  # noqa: PLC0415  # inline import avoids a module-load circular import
+        get_key_object,
+    )
+    from litellm.proxy.db.exception_handler import (  # noqa: PLC0415  # inline import avoids a module-load circular import
+        PrismaDBExceptionHandler,
+    )
+    from litellm.proxy.proxy_server import (  # noqa: PLC0415  # inline import avoids a module-load circular import
+        prisma_client,
+        user_api_key_cache,
+    )
+
+    if prisma_client is None:
+        return "unresolvable"
+    try:
+        key_obj = await get_key_object(
+            hashed_token=key_hash,
+            prisma_client=prisma_client,
+            user_api_key_cache=user_api_key_cache,
+        )
+    except (ProxyException, HTTPException):
+        return "no_active_key"
+    except Exception as exc:  # noqa: BLE001  # classify: a DB outage is retryable, anything else is an opaque gateway fault
+        if PrismaDBExceptionHandler.is_database_service_unavailable_error(exc):
+            return "unavailable"
+        verbose_logger.debug(
+            "_reload_active_key_by_hash: unexpected key-resolution error (%s)",
+            type(exc).__name__,
+        )
+        return "unresolvable"
+    if not _key_is_active(key_obj):
+        return "no_active_key"
+    return _ResolvedKey(key_hash=key_hash, key=key_obj)
+
+
+async def _reload_active_user_by_id(user_id: str) -> "_KeyResolutionFailure | None":
+    """Re-validate a live litellm user by id, returning ``None`` when the user is active or a precise
+    failure otherwise. The interactive DCR client authenticates via SSO, so its refresh envelope seals a
+    user subject; renewing it must re-check the user is still live (present and not SCIM-deactivated) so a
+    deactivated user cannot keep refreshing, mirroring how admission re-validates the same user subject on
+    the egress side. No DB connection is a gateway fault (``unresolvable``) and a
+    database-service-unavailable error is a retryable outage (``unavailable``). Everything else fails
+    closed as ``no_active_key`` (the caller maps it to invalid_grant): a ``ProxyException`` /
+    ``HTTPException``, a SCIM-deactivated user, and, unlike the key path, a missing user. ``get_user_object``
+    catches every DB failure and re-raises a bare ``ValueError`` (a deleted user and a real outage look
+    identical, the original error surviving only as ``__context__``), so the outage check walks the cause
+    chain, and a missing user falls through to ``no_active_key`` rather than an opaque gateway fault."""
+    from litellm.proxy._types import (
+        ProxyException,  # noqa: PLC0415  # inline import avoids a module-load circular import
+    )
+    from litellm.proxy.auth.auth_checks import (  # noqa: PLC0415  # inline import avoids a module-load circular import
+        get_user_object,
+    )
+    from litellm.proxy.db.exception_handler import (  # noqa: PLC0415  # inline import avoids a module-load circular import
+        PrismaDBExceptionHandler,
+    )
+    from litellm.proxy.proxy_server import (  # noqa: PLC0415  # inline import avoids a module-load circular import
+        prisma_client,
+        user_api_key_cache,
+    )
+
+    if prisma_client is None:
+        return "unresolvable"
+    try:
+        user_object = await get_user_object(
+            user_id=user_id,
+            prisma_client=prisma_client,
+            user_api_key_cache=user_api_key_cache,
+            user_id_upsert=False,
+        )
+    except (ProxyException, HTTPException):
+        return "no_active_key"
+    except Exception as exc:  # noqa: BLE001  # a DB outage is retryable; a missing user (get_user_object's wrapped ValueError) or any other resolution failure fails closed as no_active_key, never a 500
+        if PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(exc):
+            return "unavailable"
+        verbose_logger.debug("_reload_active_user_by_id: user-resolution error (%s)", type(exc).__name__)
+        return "no_active_key"
+    if user_object is None:
+        return "no_active_key"
+    if isinstance(user_object.metadata, dict) and user_object.metadata.get("scim_active") is False:
+        return "no_active_key"
+    return None
+
+
+async def _key_owner_scim_deactivated(key: "UserAPIKeyAuth") -> bool:
+    """True only when the key's owning user was explicitly SCIM-deactivated, so a refresh revokes an
+    offboarded owner's key exactly as admission does via ``_reject_if_admitted_owner_scim_deactivated``.
+    A key with no owner, a missing owner record, or a failed lookup fails OPEN (returns ``False``),
+    matching admission and the standard builder: a key may outlive its owner record, and a transient DB
+    blip must not revoke a live key. Only an explicit ``scim_active`` of ``False`` gates renewal."""
+    if key.user_id is None:
+        return False
+    from litellm.proxy.auth.auth_checks import (  # noqa: PLC0415  # inline import avoids a module-load circular import
+        get_user_object,
+    )
+    from litellm.proxy.proxy_server import (  # noqa: PLC0415  # inline import avoids a module-load circular import
+        prisma_client,
+        user_api_key_cache,
+    )
+
+    if prisma_client is None:
+        return False
+    try:
+        owner = await get_user_object(
+            user_id=key.user_id,
+            prisma_client=prisma_client,
+            user_api_key_cache=user_api_key_cache,
+            user_id_upsert=False,
+        )
+    except Exception as exc:  # noqa: BLE001  # fail open: a missing owner (get_user_object's wrapped ValueError) or a DB blip must not revoke a live key
+        verbose_logger.debug("refresh: key-owner SCIM lookup failed, not revoking (%s)", type(exc).__name__)
+        return False
+    return owner is not None and isinstance(owner.metadata, dict) and owner.metadata.get("scim_active") is False
+
+
+async def _revalidate_active_subject(identity: "EnvelopeIdentity") -> "_KeyResolutionFailure | None":
+    """Re-validate that the subject sealed in a refresh envelope is still live, dispatching on its type:
+    a key_hash reloads the virtual key, a user_id reloads the user. Returns ``None`` when the subject is
+    active or a precise failure otherwise, so revocation gates renewal for either identity source the same
+    way admission gates the egress: a blocked or expired key, a SCIM-deactivated key owner (mirroring
+    admission's owner check, so an offboarded user cannot keep renewing a still-active key), and a
+    deactivated or deleted user all fail closed to ``no_active_key``."""
+    match identity.subject_type:
+        case "key_hash":
+            reloaded = await _reload_active_key_by_hash(identity.subject)
+            if not isinstance(reloaded, _ResolvedKey):
+                return reloaded
+            if await _key_owner_scim_deactivated(reloaded.key):
+                return "no_active_key"
+            return None
+        case "user_id":
+            return await _reload_active_user_by_id(identity.subject)
+        case _:
+            assert_never(identity.subject_type)
+
+
+async def _extract_user_id_from_request(request: Request) -> str | None:
+    """The litellm ``user_id`` for the token request, so a per-user token is stored under the same
+    identity the egress later reads it by. Storage is best-effort, so every non-resolved outcome
+    (including a transient DB outage) collapses to ``None`` here and the caller simply skips the store;
+    the bridge mint, which must status those outcomes differently, consumes
+    :func:`_resolve_active_litellm_key` directly."""
+    resolved = await _resolve_active_litellm_key(request)
+    if not isinstance(resolved, _ResolvedKey):
+        return None
+    return _active_key_user_id(resolved.key)
+
+
+_UpstreamGrantRejection = Literal["no_access_token", "expired_lifetime"]
+"""Why an upstream token response cannot back a bridge envelope:
+- ``no_access_token``: the response carries no usable ``access_token``
+- ``expired_lifetime``: the response reports a parseable, non-positive ``expires_in``, i.e. an upstream
+  token that is already dead, so sealing it would forward a bearer the edge cannot use
+An absent or unparseable ``expires_in`` is NOT a rejection; the lifetime is merely unknown and the
+envelope caps it, the by-design behaviour for an upstream that omits the field."""
+
+
+def _classify_upstream_lifetime(raw_expires_in: object) -> "int | Literal['unspecified', 'expired']":
+    """Classify an upstream ``expires_in`` into a positive number of seconds, ``"unspecified"`` (absent
+    or unparseable, so the envelope caps it), or ``"expired"`` (a non-positive value the upstream reports
+    as already elapsed). Telling "we do not know the lifetime" apart from "the upstream says it is
+    already dead" is what stops an explicitly-expired token from silently receiving the envelope's 1h
+    cap. The expired decision is made on the parsed numeric value, not on ``int(...)`` of it, so a
+    positive sub-second lifetime in ``(0, 1)`` is not truncated to ``0`` and misread as elapsed; the
+    envelope works in whole seconds, so such a lifetime clamps up to its 1s floor. ``bool`` is excluded
+    (an ``int`` subclass but never a real lifetime), and the conversions can raise on ``NaN`` /
+    ``Infinity`` / oversized input, which reads as unparseable rather than surfacing as a 500."""
+    if raw_expires_in is None or isinstance(raw_expires_in, bool) or not isinstance(raw_expires_in, (int, float, str)):
+        return "unspecified"
+    try:
+        numeric = float(raw_expires_in)
+        seconds = int(numeric)
+    except (ValueError, TypeError, OverflowError):
+        return "unspecified"
+    if numeric <= 0:
+        return "expired"
+    return max(1, seconds)
+
+
+def _bridge_grant_from_token_response(token_response: object) -> "UpstreamTokenGrant | _UpstreamGrantRejection":
+    """Validate an upstream OAuth token response into a typed grant, or say why it cannot back an
+    envelope. Each field is isinstance-checked so nothing untyped from ``response.json()`` reaches the
+    grant. ``expires_in`` is read three ways (see :func:`_classify_upstream_lifetime`): an unknown
+    lifetime leaves the grant ``expires_in`` ``None`` for the envelope to cap, a positive value is
+    honoured, and an explicit already-elapsed value is a rejection rather than a silent fall-through to
+    the cap."""
+    from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import (  # noqa: PLC0415  # inline import avoids a module-load circular import
+        UpstreamTokenGrant,
+    )
+
+    if not isinstance(token_response, dict):
+        return "no_access_token"
+    access = token_response.get("access_token")
+    if not isinstance(access, str) or not access:
+        return "no_access_token"
+    lifetime = _classify_upstream_lifetime(token_response.get("expires_in"))
+    if lifetime == "expired":
+        return "expired_lifetime"
+    token_type = token_response.get("token_type")
+    scope = token_response.get("scope")
+    return UpstreamTokenGrant(
+        access_token=SecretStr(access),
+        token_type=token_type if isinstance(token_type, str) and token_type else "Bearer",
+        # The upstream refresh_token is deliberately NOT sealed: the edge never consumes it (it forwards
+        # only token_type + access_token), so it would be dead weight embedding a long-lived upstream
+        # credential in the client-held bearer, and it enlarges the envelope. Refresh support is a
+        # follow-up (a dedicated refresh-envelope); the client re-runs authorization_code at the cap.
+        refresh_token=None,
+        scope=scope if isinstance(scope, str) and scope else None,
+        expires_in=lifetime if isinstance(lifetime, int) else None,
+    )
+
+
+# ---------------------------------------------------------------------------
+# DCR-bridge oauth_delegate mint: a three-phase pipeline whose failures are values.
+#
+#   prepare  (before the upstream exchange) -> validate every precondition and resolve identity+keys
+#   exchange (the single-use upstream code is consumed here, in exchange_token_with_server)
+#   finish   (after the exchange)           -> seal the upstream grant into the client-held envelope
+#
+# Every precondition lives in ``prepare``, which runs BEFORE the exchange, so no failure can burn the
+# single-use code or rotate a refresh token, for either grant type -- that whole class of bug is gone
+# by construction rather than guarded case by case. Failures are values mapped to an OAuth-shaped
+# response in one place (``_bridge_mint_error_response``), so status codes and the RFC 6749 §5.2 body
+# shape are uniform. Adding a failure mode is a new literal plus a match arm the type checker forces.
+# ---------------------------------------------------------------------------
+
+_BridgeMintError = Literal[
+    "no_identity",
+    "invalid_refresh",
+    "identity_unavailable",
+    "identity_unresolvable",
+    "not_configured",
+    "no_upstream_token",
+    "upstream_token_expired",
+    "too_large",
+]
+
+
+@dataclass(frozen=True, slots=True)
+class _BridgeMintReady:
+    """Everything the seal needs, resolved once before the exchange: the identity to bind the envelope
+    to and the master-key-derived envelope keys. The identity is a key_hash subject for the scripted
+    two-header client (resolved from the litellm key it presents) or a user_id subject for the
+    interactive SSO client (the user recovered from the gateway authorization code), so one phase-3 seal
+    serves both. Resolving identity here means ``_finish_bridge_mint`` has no preconditions left to
+    fail."""
+
+    identity: "EnvelopeIdentity"
+    keys: "EnvelopeKeys"
+
+
+def _bridge_mint_error_response(error: _BridgeMintError) -> JSONResponse:
+    """Map a bridge-mint failure value to its token-endpoint response: one place, RFC 6749 §5.2 shape
+    (top-level ``error``, no-store headers) for every case, with a status truthful about where the
+    failure is. The caller's request is 400, a transient gateway outage is 503, a gateway
+    misconfiguration is 500, and an upstream problem is 502. The identity-resolution statuses match how
+    admission statuses the same conditions on the egress side, so mint and admit never disagree under
+    one outage."""
+    match error:
+        case "no_identity":
+            status, code, desc = (
+                400,
+                "invalid_request",
+                "this server issues a gateway-bound credential; complete the interactive sign-in, or "
+                "send a litellm credential (x-litellm-api-key or Authorization) on the token request",
+            )
+        case "invalid_refresh":
+            status, code, desc = (
+                400,
+                "invalid_grant",
+                "the refresh credential is not a valid, live refresh envelope for this server; "
+                "re-run authorization_code to obtain a new one",
+            )
+        case "identity_unavailable":
+            status, code, desc = (
+                503,
+                "temporarily_unavailable",
+                "the authentication database is temporarily unreachable; retry shortly",
+            )
+        case "identity_unresolvable":
+            status, code, desc = (
+                500,
+                "server_error",
+                "the gateway could not resolve the litellm identity for this request",
+            )
+        case "not_configured":
+            status, code, desc = (
+                500,
+                "server_error",
+                "the gateway is not configured to mint a gateway-bound credential (master_key is not set)",
+            )
+        case "no_upstream_token":
+            status, code, desc = (
+                502,
+                "server_error",
+                "the upstream token response has no usable access_token",
+            )
+        case "upstream_token_expired":
+            status, code, desc = (
+                502,
+                "server_error",
+                "the upstream token response reports an already-expired lifetime",
+            )
+        case "too_large":
+            status, code, desc = (
+                502,
+                "server_error",
+                "the upstream token is too large to seal into a gateway-bound credential",
+            )
+        case _:
+            assert_never(error)
+    return JSONResponse(
+        status_code=status, content={"error": code, "error_description": desc}, headers=TOKEN_NO_CACHE_HEADERS
+    )
+
+
+def _key_resolution_failure_to_mint_error(failure: _KeyResolutionFailure) -> _BridgeMintError:
+    """Lift an identity-resolution failure into the mint taxonomy, preserving origin so the status stays
+    truthful: the caller's missing credential is 400, a transient DB outage is 503, and a gateway that
+    cannot resolve identity is 500."""
+    match failure:
+        case "no_active_key":
+            return "no_identity"
+        case "unavailable":
+            return "identity_unavailable"
+        case "unresolvable":
+            return "identity_unresolvable"
+        case _:
+            assert_never(failure)
+
+
+def _upstream_rejection_to_mint_error(rejection: _UpstreamGrantRejection) -> _BridgeMintError:
+    """Lift an upstream-response rejection into the mint taxonomy; both are upstream faults (502)."""
+    match rejection:
+        case "no_access_token":
+            return "no_upstream_token"
+        case "expired_lifetime":
+            return "upstream_token_expired"
+        case _:
+            assert_never(rejection)
+
+
+async def _prepare_bridge_mint(
+    request: Request,
+    mcp_server: MCPServer,
+    bridge_identity: "_BridgeAuthorizationCode | None" = None,
+) -> "_BridgeMintReady | _BridgeMintError":
+    """Phase 1 for the authorization_code grant, BEFORE the upstream exchange: confirm the gateway can
+    mint (master_key set), resolve the litellm identity, and derive the envelope keys. Returns a ready
+    context or a precise failure value. Running before the exchange is what makes every failure here fail
+    closed without consuming the single-use code.
+
+    Two identity sources, one envelope. The interactive DCR client authenticates via SSO at the bridged
+    authorize, so its identity arrives as ``bridge_identity`` (the user recovered from the gateway
+    authorization code) and mints a user subject. The scripted two-header client presents a litellm key
+    on the token request instead, so its identity is the active key's hash and mints a key_hash subject.
+    A missing or invalid presented key keeps its resolution origin so the mapper statuses it truthfully;
+    neither source present is ``no_identity``. The refresh_token grant has its own phase-1
+    (:func:`_prepare_bridge_refresh`), which recovers identity from the presented refresh envelope."""
+    from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import (  # noqa: PLC0415  # inline import avoids a module-load circular import
+        envelope_keys_from_master_key,
+    )
+    from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import (  # noqa: PLC0415  # inline import avoids a module-load circular import
+        key_hash_identity,
+        user_identity,
+    )
+    from litellm.proxy.proxy_server import (  # noqa: PLC0415  # inline import avoids a module-load circular import
+        master_key,
+    )
+
+    if not master_key:
+        return "not_configured"
+    keys = envelope_keys_from_master_key(master_key)
+    if bridge_identity is not None:
+        identity = user_identity(server_id=mcp_server.server_id, user_id=bridge_identity.litellm_user_id)
+        return _BridgeMintReady(identity=identity, keys=keys)
+    resolved = await _resolve_active_litellm_key(request)
+    if not isinstance(resolved, _ResolvedKey):
+        return _key_resolution_failure_to_mint_error(resolved)
+    identity = key_hash_identity(server_id=mcp_server.server_id, key_hash=resolved.key_hash)
+    return _BridgeMintReady(identity=identity, keys=keys)
+
+
+@dataclass(frozen=True, slots=True)
+class _BridgeRefreshReady:
+    """A validated refresh request: the identity+keys to mint the renewed pair under, the upstream refresh
+    token (unwrapped from the client's refresh envelope) to exchange with the upstream IdP, and the scope
+    sealed alongside it at mint. The upstream refresh token is a ``SecretStr`` like every other credential
+    in this layer, so a repr or a traceback that captures this value never exposes the raw upstream refresh
+    token in plaintext. ``upstream_scope`` carries the originally-granted scope so the renewal re-requests
+    it when the client (a DCR/MCP client that typically omits scope on refresh) sends none, keeping the
+    renewed token's scope stable against an upstream that would otherwise narrow or drop it."""
+
+    ready: "_BridgeMintReady"
+    upstream_refresh_token: SecretStr
+    upstream_scope: str | None = None
+
+
+def _refresh_key_failure_to_mint_error(failure: _KeyResolutionFailure) -> _BridgeMintError:
+    """Lift an identity-resolution failure on the refresh path into the mint taxonomy. Unlike the mint
+    path, a resolved-but-inactive (or unknown) key is ``invalid_grant`` rather than ``invalid_request``:
+    the client did present an identity (sealed in the refresh envelope), but it is no longer live, so the
+    refresh is invalid and the client must re-authenticate. A transient outage is still 503 and a gateway
+    fault still 500, matching the mint path and admission."""
+    match failure:
+        case "no_active_key":
+            return "invalid_refresh"
+        case "unavailable":
+            return "identity_unavailable"
+        case "unresolvable":
+            return "identity_unresolvable"
+        case _:
+            assert_never(failure)
+
+
+async def _prepare_bridge_refresh(
+    mcp_server: MCPServer, refresh_value: str | None
+) -> "_BridgeRefreshReady | _BridgeMintError":
+    """Phase 1 for the refresh_token grant, BEFORE the upstream exchange: open the client's refresh
+    envelope, re-validate the sealed litellm identity so a revoked key cannot keep refreshing, and
+    recover the upstream refresh token to exchange. Identity comes entirely from the sealed envelope, not
+    the HTTP request, so the request object is not needed here. The client presents a refresh envelope,
+    never a raw upstream refresh token, so a missing value, a non-envelope, an unopenable envelope, or one
+    minted for another server is ``invalid_grant``. Running before the exchange means a rejected refresh
+    never consumes or rotates the upstream refresh token."""
+    from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import (  # noqa: PLC0415  # inline import avoids a module-load circular import
+        BridgeRefreshOpened,
+        envelope_keys_from_master_key,
+        open_bridge_refresh_envelope,
+    )
+    from litellm.proxy.proxy_server import (  # noqa: PLC0415  # inline import avoids a module-load circular import
+        master_key,
+    )
+
+    if not master_key:
+        return "not_configured"
+    if not refresh_value:
+        return "invalid_refresh"
+    keys = envelope_keys_from_master_key(master_key)
+    opened = open_bridge_refresh_envelope(refresh_value, keys, datetime.now(timezone.utc), mcp_server.server_id)
+    if not isinstance(opened, BridgeRefreshOpened):
+        return "invalid_refresh"
+    failure = await _revalidate_active_subject(opened.identity)
+    if failure is not None:
+        return _refresh_key_failure_to_mint_error(failure)
+    return _BridgeRefreshReady(
+        ready=_BridgeMintReady(identity=opened.identity, keys=keys),
+        upstream_refresh_token=opened.refresh.refresh_token,
+        upstream_scope=opened.refresh.scope,
+    )
+
+
+def _finish_bridge_mint(
+    ready: "_BridgeMintReady", mcp_server: MCPServer, token_response: object, now: datetime
+) -> "JSONResponse | _BridgeMintError":
+    """Phase 3, AFTER the upstream exchange: seal the upstream grant into the client-held access envelope
+    using the pre-resolved identity and keys, and, when the upstream returned a refresh token, seal a
+    long-lived refresh envelope alongside it so the client can renew without re-authenticating. Shared by
+    the authorization_code and refresh_token paths, so a renewal that the upstream rotates re-issues a
+    fresh refresh envelope. The only hard failures here are properties of the upstream access token (no
+    usable token, an already-expired lifetime, or a token too large to seal); a refresh token that cannot
+    be sealed degrades to an access-only response rather than failing the whole exchange."""
+    from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import (  # noqa: PLC0415  # inline import avoids a module-load circular import
+        build_bridge_token_response,
+    )
+    from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import (  # noqa: PLC0415  # inline import avoids a module-load circular import
+        SealedEnvelope,
+        UpstreamTokenGrant,
+    )
+
+    grant = _bridge_grant_from_token_response(token_response)
+    if not isinstance(grant, UpstreamTokenGrant):
+        return _upstream_rejection_to_mint_error(grant)
+    sealed = build_bridge_token_response(ready.identity, grant, ready.keys, now)
+    if not isinstance(sealed, SealedEnvelope):
+        return "too_large"
+    # Report expires_in from the JWT's own second-truncated exp, rounding the elapsed portion up, so the
+    # client is never told the bearer lives past the point admission (which uses that exp) rejects it.
+    expires_in = max(0, int(sealed.expires_at.timestamp()) - math.ceil(now.timestamp()))
+    refresh_envelope = _mint_refresh_envelope_value(ready.identity, token_response, ready.keys, now, mcp_server)
+    body = {
+        "access_token": sealed.token.get_secret_value(),
+        "token_type": "Bearer",
+        "expires_in": expires_in,
+        # A refresh envelope rides along only when the upstream returned a refresh token to seal; when it
+        # rotates on renewal, the client receives the new one and the old envelope's upstream token dies.
+        **({"refresh_token": refresh_envelope} if refresh_envelope is not None else {}),
+    }
+    return JSONResponse(body, headers=TOKEN_NO_CACHE_HEADERS)
+
+
+def _upstream_refresh_credential(token_response: object) -> "RefreshCredential | None":
+    """Extract the upstream refresh grant from a token response, or ``None`` when there is none to seal.
+    Each field is isinstance-checked so nothing untyped reaches the refresh envelope; ``refresh_expires_in``
+    (the refresh token's own lifetime, when the upstream reports it) is classified like ``expires_in`` and
+    bounds the refresh envelope's TTL. An upstream that reports the refresh token itself as already elapsed
+    (``refresh_expires_in`` non-positive) yields ``None`` rather than a refresh envelope: sealing a dead
+    token would hand the client a full-TTL-capped envelope the IdP will reject, so the exchange degrades to
+    an access-only response (the client re-authenticates at access expiry), mirroring how
+    :func:`_bridge_grant_from_token_response` refuses an already-elapsed access token instead of capping it."""
+    from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import (  # noqa: PLC0415  # inline import avoids a module-load circular import
+        RefreshCredential,
+    )
+
+    if not isinstance(token_response, dict):
+        return None
+    refresh = token_response.get("refresh_token")
+    if not isinstance(refresh, str) or not refresh:
+        return None
+    lifetime = _classify_upstream_lifetime(token_response.get("refresh_expires_in"))
+    if lifetime == "expired":
+        return None
+    scope = token_response.get("scope")
+    return RefreshCredential(
+        refresh_token=SecretStr(refresh),
+        scope=scope if isinstance(scope, str) and scope else None,
+        expires_in=lifetime if isinstance(lifetime, int) else None,
+    )
+
+
+def _mint_refresh_envelope_value(
+    identity: "EnvelopeIdentity", token_response: object, keys: "EnvelopeKeys", now: datetime, mcp_server: MCPServer
+) -> str | None:
+    """Seal the upstream refresh grant (if any) into a refresh envelope and return its bearer string, or
+    ``None`` when the upstream returned no refresh token or the refresh token is too large to seal. A
+    too-large refresh token degrades to an access-only response (logged) rather than failing an exchange
+    that already succeeded upstream: the client simply re-authenticates when the access envelope expires."""
+    from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import (  # noqa: PLC0415  # inline import avoids a module-load circular import
+        build_bridge_refresh_token_response,
+    )
+    from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import (  # noqa: PLC0415  # inline import avoids a module-load circular import
+        SealedEnvelope,
+    )
+
+    refresh_credential = _upstream_refresh_credential(token_response)
+    if refresh_credential is None:
+        return None
+    sealed = build_bridge_refresh_token_response(identity, refresh_credential, keys, now)
+    if isinstance(sealed, SealedEnvelope):
+        return sealed.token.get_secret_value()
+    verbose_logger.warning(
+        "bridge mint: the upstream refresh token is too large to seal into a refresh envelope for "
+        "server=%s; issuing an access-only response, so the client re-authenticates at access expiry",
+        mcp_server.server_id,
+    )
+    return None

litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py

@@ -1,10 +1,8 @@
 import asyncio
 import html as _html
 import json
-import math
 import secrets
 import time
-from dataclasses import dataclass
 from datetime import datetime, timezone
 from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, Tuple
 from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
@@ -13,7 +11,6 @@ import httpx
 from fastapi import APIRouter, Form, HTTPException, Request
 from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse, Response
 from pydantic import BaseModel, ConfigDict, Field, SecretStr, ValidationError
-from typing_extensions import assert_never
 
 from litellm._logging import verbose_logger
 from litellm.llms.custom_httpx.http_handler import (
@@ -24,6 +21,24 @@ from litellm.proxy._experimental.mcp_server.auth.token_endpoint_auth import (
     TokenEndpointAuthConfigError,
     build_token_endpoint_client_auth,
 )
+from litellm.proxy._experimental.mcp_server.bridge_token_flow import (
+    _bridge_mint_error_response,
+    _BridgeMintReady,
+    _BridgeRefreshReady,
+    _extract_user_id_from_request,
+    _finish_bridge_mint,
+    _prepare_bridge_mint,
+    _prepare_bridge_refresh,
+)
+from litellm.proxy._experimental.mcp_server.faults import (
+    CallerRejected,
+    CredentialSource,
+    UpstreamProtocolFault,
+    classify_upstream_dcr_rejection,
+    classify_upstream_token_rejection,
+    dcr_fault_detail,
+    render_token_fault,
+)
 from litellm.proxy._experimental.mcp_server.oauth_utils import (
     TOKEN_NO_CACHE_HEADERS,
     get_request_base_url,
@@ -40,13 +55,7 @@ from litellm.types.mcp import MCPAuth, MCPCredentials
 from litellm.types.mcp_server.mcp_server_manager import MCPServer
 
 if TYPE_CHECKING:
-    from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import (
-        EnvelopeIdentity,
-        EnvelopeKeys,
-        RefreshCredential,
-        UpstreamTokenGrant,
-    )
-    from litellm.proxy._types import LiteLLM_MCPServerTable, UserAPIKeyAuth
+    from litellm.proxy._types import LiteLLM_MCPServerTable
 
 # TTL cache for upstream OAuth metadata fetched from pass-through MCP servers.
 # Keeps us from hammering the upstream IdP on each discovery request.
@@ -384,274 +393,6 @@ def _validate_token_response(
             )
 
 
-def _litellm_key_from_request(request: Request) -> Optional[str]:
-    """Return the LiteLLM API key presented on the request, or ``None``.
-
-    Accepts the key from ``x-litellm-api-key`` (what MCP clients such as Claude Desktop/Code
-    send) as well as ``Authorization``; either may carry a bare token or ``Bearer <token>``.
-    ``x-litellm-api-key`` wins when both are present, since ``Authorization`` may instead carry
-    an OAuth/upstream bearer.
-    """
-    for header_value in (
-        request.headers.get("x-litellm-api-key"),
-        request.headers.get("Authorization") or request.headers.get("authorization"),
-    ):
-        if not header_value:
-            continue
-        value = header_value.strip()
-        if value.lower().startswith("bearer "):
-            value = value[7:].strip()
-        if value:
-            return value
-    return None
-
-
-def _key_is_active(key_obj: "UserAPIKeyAuth") -> bool:
-    """``True`` when the presented key is neither blocked nor past its expiry.
-
-    The OAuth token endpoint is unauthenticated, so the presented key is validated here before it is
-    trusted; a revoked or expired key must not mint a bridge envelope or write a stored credential.
-    ``get_key_object`` resolves a row without these checks (the main ``user_api_key_auth`` pipeline
-    enforces them downstream, which this endpoint bypasses), so they are applied here. Deleted keys
-    are already rejected upstream, where ``get_key_object`` raises on a row that no longer exists.
-
-    This is an active-state gate only; it deliberately does not require a ``user_id``. A valid
-    team-scoped or service-account key has no ``user_id`` yet is a legitimate credential, so gating
-    on ``user_id`` presence would wrongly reject it. Callers that need the user (the per-user token
-    store) derive it separately via :func:`_active_key_user_id`.
-
-    Total by design: ``expires`` is typed ``str | datetime``, and an unparseable string would make
-    ``datetime.fromisoformat`` raise. Since the callers run this outside their key-resolution
-    ``try``, an uncaught parse error would surface as a 500 instead of the endpoint's fail-closed
-    behavior, so a malformed expiry is treated as inactive (return ``False``) rather than raising.
-    """
-    if key_obj.blocked is True:
-        return False
-    expires = key_obj.expires
-    if expires is not None:
-        if isinstance(expires, datetime):
-            expiry = expires
-        else:
-            try:
-                expiry = datetime.fromisoformat(expires)
-            except (ValueError, TypeError):
-                return False
-        if expiry.tzinfo is None or expiry.tzinfo.utcoffset(expiry) is None:
-            expiry = expiry.replace(tzinfo=timezone.utc)
-        if expiry < datetime.now(timezone.utc):
-            return False
-    return True
-
-
-def _active_key_user_id(key_obj: "UserAPIKeyAuth") -> str | None:
-    """The active key's ``user_id``, or ``None`` when the key is blocked/expired or simply has no
-    ``user_id`` (a team-scoped or service-account key). Used only by the per-user token store, which
-    needs a user to key the stored credential; the bridge mint uses the key hash and does not."""
-    return key_obj.user_id if _key_is_active(key_obj) else None
-
-
-@dataclass(frozen=True, slots=True)
-class _ResolvedKey:
-    """An active litellm key resolved from the token request: its hash (the value ``get_key_object``
-    and the cache/DB layer key the record by) and the live record."""
-
-    key_hash: str
-    key: "UserAPIKeyAuth"
-
-
-_KeyResolutionFailure = Literal["no_active_key", "unavailable", "unresolvable"]
-"""Why a token request yielded no active litellm key, kept distinct so a caller statuses each truthfully
-instead of blaming the client for a gateway problem:
-- ``no_active_key``: none was presented, or the presented key is unknown / blocked / expired (the
-  caller's request is at fault)
-- ``unavailable``: the auth database was transiently unreachable while resolving (retryable)
-- ``unresolvable``: the gateway cannot resolve identity right now (no DB connection, or an unexpected
-  error) -- a gateway fault, not the caller's
-The classification mirrors admission's ``_reload_admitted_key`` so the mint (ingress) and admission
-(egress) never disagree on the status of the same outage."""
-
-
-async def _resolve_active_litellm_key(request: Request) -> "_ResolvedKey | _KeyResolutionFailure":
-    """Resolve the presented litellm key to an active key record, or say precisely why not.
-
-    Single resolution path the OAuth token endpoint reuses, resolving authoritatively via
-    ``get_key_object`` (cache first, then DB). The failure is a value, not a bare ``None``, so a caller
-    can tell "the client sent no usable credential" (a request error) apart from "the gateway could not
-    check" (an infrastructure error) and status each truthfully; collapsing both to ``None`` is what let
-    a DB outage read as a 400. A resolved key is still gated by ``_key_is_active``, so a blocked or
-    expired key is ``no_active_key`` while a valid team-scoped or service-account key (no ``user_id``)
-    resolves. Classification mirrors admission's ``_reload_admitted_key``: no DB connection is a gateway
-    fault, a ``ProxyException`` / ``HTTPException`` from ``get_key_object`` is an unknown or invalid key,
-    a database-service-unavailable error is a retryable outage, and anything else is an unexpected
-    gateway fault."""
-    token = _litellm_key_from_request(request)
-    if not token:
-        return "no_active_key"
-    from litellm.proxy._types import hash_token  # noqa: PLC0415  # inline import avoids a module-load circular import
-
-    return await _reload_active_key_by_hash(hash_token(token))
-
-
-async def _reload_active_key_by_hash(key_hash: str) -> "_ResolvedKey | _KeyResolutionFailure":
-    """Reload the live key record for ``key_hash`` (cache first, then DB) and gate it on active state,
-    returning the resolved key or a precise failure. Shared by the token request's presented-key
-    resolution (:func:`_resolve_active_litellm_key`, which hashes the presented key) and the refresh
-    path (which already holds the hash sealed in the refresh envelope), so both re-validate identity
-    through one active-key gate and one failure classification. Classification mirrors admission's
-    ``_reload_admitted_key``: no DB connection is a gateway fault, a ``ProxyException`` / ``HTTPException``
-    from ``get_key_object`` is an unknown or invalid key, a database-service-unavailable error is a
-    retryable outage, and anything else is an unexpected gateway fault. A blocked or expired key is
-    ``no_active_key``, so a revoked key can neither mint nor refresh a bridge envelope."""
-    from litellm.proxy._types import (
-        ProxyException,  # noqa: PLC0415  # inline import avoids a module-load circular import
-    )
-    from litellm.proxy.auth.auth_checks import (  # noqa: PLC0415  # inline import avoids a module-load circular import
-        get_key_object,
-    )
-    from litellm.proxy.db.exception_handler import (  # noqa: PLC0415  # inline import avoids a module-load circular import
-        PrismaDBExceptionHandler,
-    )
-    from litellm.proxy.proxy_server import (  # noqa: PLC0415  # inline import avoids a module-load circular import
-        prisma_client,
-        user_api_key_cache,
-    )
-
-    if prisma_client is None:
-        return "unresolvable"
-    try:
-        key_obj = await get_key_object(
-            hashed_token=key_hash,
-            prisma_client=prisma_client,
-            user_api_key_cache=user_api_key_cache,
-        )
-    except (ProxyException, HTTPException):
-        return "no_active_key"
-    except Exception as exc:  # noqa: BLE001  # classify: a DB outage is retryable, anything else is an opaque gateway fault
-        if PrismaDBExceptionHandler.is_database_service_unavailable_error(exc):
-            return "unavailable"
-        verbose_logger.debug(
-            "_reload_active_key_by_hash: unexpected key-resolution error (%s)",
-            type(exc).__name__,
-        )
-        return "unresolvable"
-    if not _key_is_active(key_obj):
-        return "no_active_key"
-    return _ResolvedKey(key_hash=key_hash, key=key_obj)
-
-
-async def _reload_active_user_by_id(user_id: str) -> "_KeyResolutionFailure | None":
-    """Re-validate a live litellm user by id, returning ``None`` when the user is active or a precise
-    failure otherwise. The interactive DCR client authenticates via SSO, so its refresh envelope seals a
-    user subject; renewing it must re-check the user is still live (present and not SCIM-deactivated) so a
-    deactivated user cannot keep refreshing, mirroring how admission re-validates the same user subject on
-    the egress side. No DB connection is a gateway fault (``unresolvable``) and a
-    database-service-unavailable error is a retryable outage (``unavailable``). Everything else fails
-    closed as ``no_active_key`` (the caller maps it to invalid_grant): a ``ProxyException`` /
-    ``HTTPException``, a SCIM-deactivated user, and, unlike the key path, a missing user. ``get_user_object``
-    catches every DB failure and re-raises a bare ``ValueError`` (a deleted user and a real outage look
-    identical, the original error surviving only as ``__context__``), so the outage check walks the cause
-    chain, and a missing user falls through to ``no_active_key`` rather than an opaque gateway fault."""
-    from litellm.proxy._types import (
-        ProxyException,  # noqa: PLC0415  # inline import avoids a module-load circular import
-    )
-    from litellm.proxy.auth.auth_checks import (  # noqa: PLC0415  # inline import avoids a module-load circular import
-        get_user_object,
-    )
-    from litellm.proxy.db.exception_handler import (  # noqa: PLC0415  # inline import avoids a module-load circular import
-        PrismaDBExceptionHandler,
-    )
-    from litellm.proxy.proxy_server import (  # noqa: PLC0415  # inline import avoids a module-load circular import
-        prisma_client,
-        user_api_key_cache,
-    )
-
-    if prisma_client is None:
-        return "unresolvable"
-    try:
-        user_object = await get_user_object(
-            user_id=user_id,
-            prisma_client=prisma_client,
-            user_api_key_cache=user_api_key_cache,
-            user_id_upsert=False,
-        )
-    except (ProxyException, HTTPException):
-        return "no_active_key"
-    except Exception as exc:  # noqa: BLE001  # a DB outage is retryable; a missing user (get_user_object's wrapped ValueError) or any other resolution failure fails closed as no_active_key, never a 500
-        if PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(exc):
-            return "unavailable"
-        verbose_logger.debug("_reload_active_user_by_id: user-resolution error (%s)", type(exc).__name__)
-        return "no_active_key"
-    if user_object is None:
-        return "no_active_key"
-    if isinstance(user_object.metadata, dict) and user_object.metadata.get("scim_active") is False:
-        return "no_active_key"
-    return None
-
-
-async def _key_owner_scim_deactivated(key: "UserAPIKeyAuth") -> bool:
-    """True only when the key's owning user was explicitly SCIM-deactivated, so a refresh revokes an
-    offboarded owner's key exactly as admission does via ``_reject_if_admitted_owner_scim_deactivated``.
-    A key with no owner, a missing owner record, or a failed lookup fails OPEN (returns ``False``),
-    matching admission and the standard builder: a key may outlive its owner record, and a transient DB
-    blip must not revoke a live key. Only an explicit ``scim_active`` of ``False`` gates renewal."""
-    if key.user_id is None:
-        return False
-    from litellm.proxy.auth.auth_checks import (  # noqa: PLC0415  # inline import avoids a module-load circular import
-        get_user_object,
-    )
-    from litellm.proxy.proxy_server import (  # noqa: PLC0415  # inline import avoids a module-load circular import
-        prisma_client,
-        user_api_key_cache,
-    )
-
-    if prisma_client is None:
-        return False
-    try:
-        owner = await get_user_object(
-            user_id=key.user_id,
-            prisma_client=prisma_client,
-            user_api_key_cache=user_api_key_cache,
-            user_id_upsert=False,
-        )
-    except Exception as exc:  # noqa: BLE001  # fail open: a missing owner (get_user_object's wrapped ValueError) or a DB blip must not revoke a live key
-        verbose_logger.debug("refresh: key-owner SCIM lookup failed, not revoking (%s)", type(exc).__name__)
-        return False
-    return owner is not None and isinstance(owner.metadata, dict) and owner.metadata.get("scim_active") is False
-
-
-async def _revalidate_active_subject(identity: "EnvelopeIdentity") -> "_KeyResolutionFailure | None":
-    """Re-validate that the subject sealed in a refresh envelope is still live, dispatching on its type:
-    a key_hash reloads the virtual key, a user_id reloads the user. Returns ``None`` when the subject is
-    active or a precise failure otherwise, so revocation gates renewal for either identity source the same
-    way admission gates the egress: a blocked or expired key, a SCIM-deactivated key owner (mirroring
-    admission's owner check, so an offboarded user cannot keep renewing a still-active key), and a
-    deactivated or deleted user all fail closed to ``no_active_key``."""
-    match identity.subject_type:
-        case "key_hash":
-            reloaded = await _reload_active_key_by_hash(identity.subject)
-            if not isinstance(reloaded, _ResolvedKey):
-                return reloaded
-            if await _key_owner_scim_deactivated(reloaded.key):
-                return "no_active_key"
-            return None
-        case "user_id":
-            return await _reload_active_user_by_id(identity.subject)
-        case _:
-            assert_never(identity.subject_type)
-
-
-async def _extract_user_id_from_request(request: Request) -> str | None:
-    """The litellm ``user_id`` for the token request, so a per-user token is stored under the same
-    identity the egress later reads it by. Storage is best-effort, so every non-resolved outcome
-    (including a transient DB outage) collapses to ``None`` here and the caller simply skips the store;
-    the bridge mint, which must status those outcomes differently, consumes
-    :func:`_resolve_active_litellm_key` directly."""
-    resolved = await _resolve_active_litellm_key(request)
-    if not isinstance(resolved, _ResolvedKey):
-        return None
-    return _active_key_user_id(resolved.key)
-
-
 async def _store_per_user_token_server_side(
     server: MCPServer,
     user_id: str,
@@ -937,420 +678,11 @@ async def authorize_with_server(
     return response
 
 
-_UpstreamGrantRejection = Literal["no_access_token", "expired_lifetime"]
-"""Why an upstream token response cannot back a bridge envelope:
-- ``no_access_token``: the response carries no usable ``access_token``
-- ``expired_lifetime``: the response reports a parseable, non-positive ``expires_in``, i.e. an upstream
-  token that is already dead, so sealing it would forward a bearer the edge cannot use
-An absent or unparseable ``expires_in`` is NOT a rejection; the lifetime is merely unknown and the
-envelope caps it, the by-design behaviour for an upstream that omits the field."""
-
-
-def _classify_upstream_lifetime(raw_expires_in: object) -> "int | Literal['unspecified', 'expired']":
-    """Classify an upstream ``expires_in`` into a positive number of seconds, ``"unspecified"`` (absent
-    or unparseable, so the envelope caps it), or ``"expired"`` (a non-positive value the upstream reports
-    as already elapsed). Telling "we do not know the lifetime" apart from "the upstream says it is
-    already dead" is what stops an explicitly-expired token from silently receiving the envelope's 1h
-    cap. The expired decision is made on the parsed numeric value, not on ``int(...)`` of it, so a
-    positive sub-second lifetime in ``(0, 1)`` is not truncated to ``0`` and misread as elapsed; the
-    envelope works in whole seconds, so such a lifetime clamps up to its 1s floor. ``bool`` is excluded
-    (an ``int`` subclass but never a real lifetime), and the conversions can raise on ``NaN`` /
-    ``Infinity`` / oversized input, which reads as unparseable rather than surfacing as a 500."""
-    if raw_expires_in is None or isinstance(raw_expires_in, bool) or not isinstance(raw_expires_in, (int, float, str)):
-        return "unspecified"
-    try:
-        numeric = float(raw_expires_in)
-        seconds = int(numeric)
-    except (ValueError, TypeError, OverflowError):
-        return "unspecified"
-    if numeric <= 0:
-        return "expired"
-    return max(1, seconds)
-
-
-def _bridge_grant_from_token_response(token_response: object) -> "UpstreamTokenGrant | _UpstreamGrantRejection":
-    """Validate an upstream OAuth token response into a typed grant, or say why it cannot back an
-    envelope. Each field is isinstance-checked so nothing untyped from ``response.json()`` reaches the
-    grant. ``expires_in`` is read three ways (see :func:`_classify_upstream_lifetime`): an unknown
-    lifetime leaves the grant ``expires_in`` ``None`` for the envelope to cap, a positive value is
-    honoured, and an explicit already-elapsed value is a rejection rather than a silent fall-through to
-    the cap."""
-    from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import (  # noqa: PLC0415  # inline import avoids a module-load circular import
-        UpstreamTokenGrant,
-    )
-
-    if not isinstance(token_response, dict):
-        return "no_access_token"
-    access = token_response.get("access_token")
-    if not isinstance(access, str) or not access:
-        return "no_access_token"
-    lifetime = _classify_upstream_lifetime(token_response.get("expires_in"))
-    if lifetime == "expired":
-        return "expired_lifetime"
-    token_type = token_response.get("token_type")
-    scope = token_response.get("scope")
-    return UpstreamTokenGrant(
-        access_token=SecretStr(access),
-        token_type=token_type if isinstance(token_type, str) and token_type else "Bearer",
-        # The upstream refresh_token is deliberately NOT sealed: the edge never consumes it (it forwards
-        # only token_type + access_token), so it would be dead weight embedding a long-lived upstream
-        # credential in the client-held bearer, and it enlarges the envelope. Refresh support is a
-        # follow-up (a dedicated refresh-envelope); the client re-runs authorization_code at the cap.
-        refresh_token=None,
-        scope=scope if isinstance(scope, str) and scope else None,
-        expires_in=lifetime if isinstance(lifetime, int) else None,
-    )
-
-
-# ---------------------------------------------------------------------------
-# DCR-bridge oauth_delegate mint: a three-phase pipeline whose failures are values.
-#
-#   prepare  (before the upstream exchange) -> validate every precondition and resolve identity+keys
-#   exchange (the single-use upstream code is consumed here, in exchange_token_with_server)
-#   finish   (after the exchange)           -> seal the upstream grant into the client-held envelope
-#
-# Every precondition lives in ``prepare``, which runs BEFORE the exchange, so no failure can burn the
-# single-use code or rotate a refresh token, for either grant type -- that whole class of bug is gone
-# by construction rather than guarded case by case. Failures are values mapped to an OAuth-shaped
-# response in one place (``_bridge_mint_error_response``), so status codes and the RFC 6749 §5.2 body
-# shape are uniform. Adding a failure mode is a new literal plus a match arm the type checker forces.
-# ---------------------------------------------------------------------------
-
-_BridgeMintError = Literal[
-    "no_identity",
-    "invalid_refresh",
-    "identity_unavailable",
-    "identity_unresolvable",
-    "not_configured",
-    "no_upstream_token",
-    "upstream_token_expired",
-    "too_large",
-]
-
-
-@dataclass(frozen=True, slots=True)
-class _BridgeMintReady:
-    """Everything the seal needs, resolved once before the exchange: the identity to bind the envelope
-    to and the master-key-derived envelope keys. The identity is a key_hash subject for the scripted
-    two-header client (resolved from the litellm key it presents) or a user_id subject for the
-    interactive SSO client (the user recovered from the gateway authorization code), so one phase-3 seal
-    serves both. Resolving identity here means ``_finish_bridge_mint`` has no preconditions left to
-    fail."""
-
-    identity: "EnvelopeIdentity"
-    keys: "EnvelopeKeys"
-
-
-def _bridge_mint_error_response(error: _BridgeMintError) -> JSONResponse:
-    """Map a bridge-mint failure value to its token-endpoint response: one place, RFC 6749 §5.2 shape
-    (top-level ``error``, no-store headers) for every case, with a status truthful about where the
-    failure is. The caller's request is 400, a transient gateway outage is 503, a gateway
-    misconfiguration is 500, and an upstream problem is 502. The identity-resolution statuses match how
-    admission statuses the same conditions on the egress side, so mint and admit never disagree under
-    one outage."""
-    match error:
-        case "no_identity":
-            status, code, desc = (
-                400,
-                "invalid_request",
-                "this server issues a gateway-bound credential; complete the interactive sign-in, or "
-                "send a litellm credential (x-litellm-api-key or Authorization) on the token request",
-            )
-        case "invalid_refresh":
-            status, code, desc = (
-                400,
-                "invalid_grant",
-                "the refresh credential is not a valid, live refresh envelope for this server; "
-                "re-run authorization_code to obtain a new one",
-            )
-        case "identity_unavailable":
-            status, code, desc = (
-                503,
-                "temporarily_unavailable",
-                "the authentication database is temporarily unreachable; retry shortly",
-            )
-        case "identity_unresolvable":
-            status, code, desc = (
-                500,
-                "server_error",
-                "the gateway could not resolve the litellm identity for this request",
-            )
-        case "not_configured":
-            status, code, desc = (
-                500,
-                "server_error",
-                "the gateway is not configured to mint a gateway-bound credential (master_key is not set)",
-            )
-        case "no_upstream_token":
-            status, code, desc = (
-                502,
-                "server_error",
-                "the upstream token response has no usable access_token",
-            )
-        case "upstream_token_expired":
-            status, code, desc = (
-                502,
-                "server_error",
-                "the upstream token response reports an already-expired lifetime",
-            )
-        case "too_large":
-            status, code, desc = (
-                502,
-                "server_error",
-                "the upstream token is too large to seal into a gateway-bound credential",
-            )
-        case _:
-            assert_never(error)
-    return JSONResponse(
-        status_code=status, content={"error": code, "error_description": desc}, headers=TOKEN_NO_CACHE_HEADERS
-    )
-
-
-def _key_resolution_failure_to_mint_error(failure: _KeyResolutionFailure) -> _BridgeMintError:
-    """Lift an identity-resolution failure into the mint taxonomy, preserving origin so the status stays
-    truthful: the caller's missing credential is 400, a transient DB outage is 503, and a gateway that
-    cannot resolve identity is 500."""
-    match failure:
-        case "no_active_key":
-            return "no_identity"
-        case "unavailable":
-            return "identity_unavailable"
-        case "unresolvable":
-            return "identity_unresolvable"
-        case _:
-            assert_never(failure)
-
-
-def _upstream_rejection_to_mint_error(rejection: _UpstreamGrantRejection) -> _BridgeMintError:
-    """Lift an upstream-response rejection into the mint taxonomy; both are upstream faults (502)."""
-    match rejection:
-        case "no_access_token":
-            return "no_upstream_token"
-        case "expired_lifetime":
-            return "upstream_token_expired"
-        case _:
-            assert_never(rejection)
-
-
-async def _prepare_bridge_mint(
-    request: Request,
-    mcp_server: MCPServer,
-    bridge_identity: _BridgeAuthorizationCode | None = None,
-) -> "_BridgeMintReady | _BridgeMintError":
-    """Phase 1 for the authorization_code grant, BEFORE the upstream exchange: confirm the gateway can
-    mint (master_key set), resolve the litellm identity, and derive the envelope keys. Returns a ready
-    context or a precise failure value. Running before the exchange is what makes every failure here fail
-    closed without consuming the single-use code.
-
-    Two identity sources, one envelope. The interactive DCR client authenticates via SSO at the bridged
-    authorize, so its identity arrives as ``bridge_identity`` (the user recovered from the gateway
-    authorization code) and mints a user subject. The scripted two-header client presents a litellm key
-    on the token request instead, so its identity is the active key's hash and mints a key_hash subject.
-    A missing or invalid presented key keeps its resolution origin so the mapper statuses it truthfully;
-    neither source present is ``no_identity``. The refresh_token grant has its own phase-1
-    (:func:`_prepare_bridge_refresh`), which recovers identity from the presented refresh envelope."""
-    from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import (  # noqa: PLC0415  # inline import avoids a module-load circular import
-        envelope_keys_from_master_key,
-    )
-    from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import (  # noqa: PLC0415  # inline import avoids a module-load circular import
-        key_hash_identity,
-        user_identity,
-    )
-    from litellm.proxy.proxy_server import (  # noqa: PLC0415  # inline import avoids a module-load circular import
-        master_key,
-    )
-
-    if not master_key:
-        return "not_configured"
-    keys = envelope_keys_from_master_key(master_key)
-    if bridge_identity is not None:
-        identity = user_identity(server_id=mcp_server.server_id, user_id=bridge_identity.litellm_user_id)
-        return _BridgeMintReady(identity=identity, keys=keys)
-    resolved = await _resolve_active_litellm_key(request)
-    if not isinstance(resolved, _ResolvedKey):
-        return _key_resolution_failure_to_mint_error(resolved)
-    identity = key_hash_identity(server_id=mcp_server.server_id, key_hash=resolved.key_hash)
-    return _BridgeMintReady(identity=identity, keys=keys)
-
-
-@dataclass(frozen=True, slots=True)
-class _BridgeRefreshReady:
-    """A validated refresh request: the identity+keys to mint the renewed pair under, the upstream refresh
-    token (unwrapped from the client's refresh envelope) to exchange with the upstream IdP, and the scope
-    sealed alongside it at mint. The upstream refresh token is a ``SecretStr`` like every other credential
-    in this layer, so a repr or a traceback that captures this value never exposes the raw upstream refresh
-    token in plaintext. ``upstream_scope`` carries the originally-granted scope so the renewal re-requests
-    it when the client (a DCR/MCP client that typically omits scope on refresh) sends none, keeping the
-    renewed token's scope stable against an upstream that would otherwise narrow or drop it."""
-
-    ready: "_BridgeMintReady"
-    upstream_refresh_token: SecretStr
-    upstream_scope: str | None = None
-
-
-def _refresh_key_failure_to_mint_error(failure: _KeyResolutionFailure) -> _BridgeMintError:
-    """Lift an identity-resolution failure on the refresh path into the mint taxonomy. Unlike the mint
-    path, a resolved-but-inactive (or unknown) key is ``invalid_grant`` rather than ``invalid_request``:
-    the client did present an identity (sealed in the refresh envelope), but it is no longer live, so the
-    refresh is invalid and the client must re-authenticate. A transient outage is still 503 and a gateway
-    fault still 500, matching the mint path and admission."""
-    match failure:
-        case "no_active_key":
-            return "invalid_refresh"
-        case "unavailable":
-            return "identity_unavailable"
-        case "unresolvable":
-            return "identity_unresolvable"
-        case _:
-            assert_never(failure)
-
-
-async def _prepare_bridge_refresh(
-    mcp_server: MCPServer, refresh_value: str | None
-) -> "_BridgeRefreshReady | _BridgeMintError":
-    """Phase 1 for the refresh_token grant, BEFORE the upstream exchange: open the client's refresh
-    envelope, re-validate the sealed litellm identity so a revoked key cannot keep refreshing, and
-    recover the upstream refresh token to exchange. Identity comes entirely from the sealed envelope, not
-    the HTTP request, so the request object is not needed here. The client presents a refresh envelope,
-    never a raw upstream refresh token, so a missing value, a non-envelope, an unopenable envelope, or one
-    minted for another server is ``invalid_grant``. Running before the exchange means a rejected refresh
-    never consumes or rotates the upstream refresh token."""
-    from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import (  # noqa: PLC0415  # inline import avoids a module-load circular import
-        BridgeRefreshOpened,
-        envelope_keys_from_master_key,
-        open_bridge_refresh_envelope,
-    )
-    from litellm.proxy.proxy_server import (  # noqa: PLC0415  # inline import avoids a module-load circular import
-        master_key,
-    )
-
-    if not master_key:
-        return "not_configured"
-    if not refresh_value:
-        return "invalid_refresh"
-    keys = envelope_keys_from_master_key(master_key)
-    opened = open_bridge_refresh_envelope(refresh_value, keys, datetime.now(timezone.utc), mcp_server.server_id)
-    if not isinstance(opened, BridgeRefreshOpened):
-        return "invalid_refresh"
-    failure = await _revalidate_active_subject(opened.identity)
-    if failure is not None:
-        return _refresh_key_failure_to_mint_error(failure)
-    return _BridgeRefreshReady(
-        ready=_BridgeMintReady(identity=opened.identity, keys=keys),
-        upstream_refresh_token=opened.refresh.refresh_token,
-        upstream_scope=opened.refresh.scope,
-    )
-
-
-def _finish_bridge_mint(
-    ready: "_BridgeMintReady", mcp_server: MCPServer, token_response: object, now: datetime
-) -> "JSONResponse | _BridgeMintError":
-    """Phase 3, AFTER the upstream exchange: seal the upstream grant into the client-held access envelope
-    using the pre-resolved identity and keys, and, when the upstream returned a refresh token, seal a
-    long-lived refresh envelope alongside it so the client can renew without re-authenticating. Shared by
-    the authorization_code and refresh_token paths, so a renewal that the upstream rotates re-issues a
-    fresh refresh envelope. The only hard failures here are properties of the upstream access token (no
-    usable token, an already-expired lifetime, or a token too large to seal); a refresh token that cannot
-    be sealed degrades to an access-only response rather than failing the whole exchange."""
-    from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import (  # noqa: PLC0415  # inline import avoids a module-load circular import
-        build_bridge_token_response,
-    )
-    from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import (  # noqa: PLC0415  # inline import avoids a module-load circular import
-        SealedEnvelope,
-        UpstreamTokenGrant,
-    )
-
-    grant = _bridge_grant_from_token_response(token_response)
-    if not isinstance(grant, UpstreamTokenGrant):
-        return _upstream_rejection_to_mint_error(grant)
-    sealed = build_bridge_token_response(ready.identity, grant, ready.keys, now)
-    if not isinstance(sealed, SealedEnvelope):
-        return "too_large"
-    # Report expires_in from the JWT's own second-truncated exp, rounding the elapsed portion up, so the
-    # client is never told the bearer lives past the point admission (which uses that exp) rejects it.
-    expires_in = max(0, int(sealed.expires_at.timestamp()) - math.ceil(now.timestamp()))
-    refresh_envelope = _mint_refresh_envelope_value(ready.identity, token_response, ready.keys, now, mcp_server)
-    body = {
-        "access_token": sealed.token.get_secret_value(),
-        "token_type": "Bearer",
-        "expires_in": expires_in,
-        # A refresh envelope rides along only when the upstream returned a refresh token to seal; when it
-        # rotates on renewal, the client receives the new one and the old envelope's upstream token dies.
-        **({"refresh_token": refresh_envelope} if refresh_envelope is not None else {}),
-    }
-    return JSONResponse(body, headers=TOKEN_NO_CACHE_HEADERS)
-
-
-def _upstream_refresh_credential(token_response: object) -> "RefreshCredential | None":
-    """Extract the upstream refresh grant from a token response, or ``None`` when there is none to seal.
-    Each field is isinstance-checked so nothing untyped reaches the refresh envelope; ``refresh_expires_in``
-    (the refresh token's own lifetime, when the upstream reports it) is classified like ``expires_in`` and
-    bounds the refresh envelope's TTL. An upstream that reports the refresh token itself as already elapsed
-    (``refresh_expires_in`` non-positive) yields ``None`` rather than a refresh envelope: sealing a dead
-    token would hand the client a full-TTL-capped envelope the IdP will reject, so the exchange degrades to
-    an access-only response (the client re-authenticates at access expiry), mirroring how
-    :func:`_bridge_grant_from_token_response` refuses an already-elapsed access token instead of capping it."""
-    from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import (  # noqa: PLC0415  # inline import avoids a module-load circular import
-        RefreshCredential,
-    )
-
-    if not isinstance(token_response, dict):
-        return None
-    refresh = token_response.get("refresh_token")
-    if not isinstance(refresh, str) or not refresh:
-        return None
-    lifetime = _classify_upstream_lifetime(token_response.get("refresh_expires_in"))
-    if lifetime == "expired":
-        return None
-    scope = token_response.get("scope")
-    return RefreshCredential(
-        refresh_token=SecretStr(refresh),
-        scope=scope if isinstance(scope, str) and scope else None,
-        expires_in=lifetime if isinstance(lifetime, int) else None,
-    )
-
-
-def _mint_refresh_envelope_value(
-    identity: "EnvelopeIdentity", token_response: object, keys: "EnvelopeKeys", now: datetime, mcp_server: MCPServer
-) -> str | None:
-    """Seal the upstream refresh grant (if any) into a refresh envelope and return its bearer string, or
-    ``None`` when the upstream returned no refresh token or the refresh token is too large to seal. A
-    too-large refresh token degrades to an access-only response (logged) rather than failing an exchange
-    that already succeeded upstream: the client simply re-authenticates when the access envelope expires."""
-    from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import (  # noqa: PLC0415  # inline import avoids a module-load circular import
-        build_bridge_refresh_token_response,
-    )
-    from litellm.proxy._experimental.mcp_server.outbound_credentials.envelope import (  # noqa: PLC0415  # inline import avoids a module-load circular import
-        SealedEnvelope,
-    )
-
-    refresh_credential = _upstream_refresh_credential(token_response)
-    if refresh_credential is None:
-        return None
-    sealed = build_bridge_refresh_token_response(identity, refresh_credential, keys, now)
-    if isinstance(sealed, SealedEnvelope):
-        return sealed.token.get_secret_value()
-    verbose_logger.warning(
-        "bridge mint: the upstream refresh token is too large to seal into a refresh envelope for "
-        "server=%s; issuing an access-only response, so the client re-authenticates at access expiry",
-        mcp_server.server_id,
-    )
-    return None
-
-
-def _upstream_oauth_error(response: httpx.Response) -> str | None:
-    """The RFC 6749 5.2 ``error`` code from an upstream token-endpoint error body, or ``None`` when the
-    body is not a JSON object carrying a string ``error``. Reading the field beats substring-matching the
-    raw text, which would false-match a code that only appears inside ``error_description`` (a false
-    invalid_grant would trigger a needless authorization_code re-run)."""
-    try:
-        body = json.loads(response.text)
-    except (ValueError, TypeError):
-        return None
-    if not isinstance(body, dict):
-        return None
-    error = body.get("error")
-    return error if isinstance(error, str) else None
+def _token_credential_source(mcp_server: MCPServer) -> CredentialSource:
+    """Mirrors the resolved-client rule in :func:`exchange_token_with_server`: when the server has a
+    stored client_id the gateway presents its own credentials upstream, so a credential rejection is
+    the operator's fault, not the caller's."""
+    return "gateway_stored" if mcp_server.client_id else "caller_supplied"
 
 
 async def exchange_token_with_server(
@@ -1469,32 +801,25 @@ async def exchange_token_with_server(
                 return _bridge_mint_error_response(prepared)
             bridge_mint_ready = prepared
     async_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.Oauth2Check)
-    response = await async_client.post(
-        mcp_server.token_url,
-        headers={"Accept": "application/json", **client_auth.headers},
-        data=token_data,
-    )
-    if response is None:
-        raise HTTPException(
-            status_code=502,
-            detail="MCP upstream token endpoint returned no response",
-        )
-
     try:
-        response.raise_for_status()
+        response = await async_client.post(
+            mcp_server.token_url,
+            headers={"Accept": "application/json", **client_auth.headers},
+            data=token_data,
+        )
+        if response is not None:
+            response.raise_for_status()
     except httpx.HTTPStatusError as exc:
-        if "invalid_target" in exc.response.text:
-            verbose_logger.warning(
-                "MCP server %s: the upstream authorization server rejected the token request with "
-                "invalid_target; it may require RFC 8707 resource indicators, which the gateway "
-                "does not send yet (tracked as LIT-4339)",
-                mcp_server.server_id,
-            )
+        fault = classify_upstream_token_rejection(
+            exc.response,
+            credential_source=_token_credential_source(mcp_server),
+            log_context=mcp_server.server_id,
+        )
         upstream_rejected_bridge_refresh = (
             is_bridge
             and grant_type == "refresh_token"
-            and exc.response.status_code == 400
-            and _upstream_oauth_error(exc.response) == "invalid_grant"
+            and isinstance(fault, CallerRejected)
+            and fault.code == "invalid_grant"
         )
         if upstream_rejected_bridge_refresh:
             verbose_logger.info(
@@ -1504,7 +829,12 @@ async def exchange_token_with_server(
                 mcp_server.server_id,
             )
             return _bridge_mint_error_response("invalid_refresh")
-        raise
+        return render_token_fault(fault)
+    if response is None:
+        raise HTTPException(
+            status_code=502,
+            detail="MCP upstream token endpoint returned no response",
+        )
     token_response = response.json()
 
     # Validate token response against server-configured rules before any storage.
@@ -1556,8 +886,12 @@ async def exchange_token_with_server(
         minted = _finish_bridge_mint(bridge_mint_ready, mcp_server, token_response, datetime.now(timezone.utc))
         return minted if isinstance(minted, JSONResponse) else _bridge_mint_error_response(minted)
 
+    raw_access_token = token_response.get("access_token") if isinstance(token_response, dict) else None
+    if not isinstance(raw_access_token, str) or not raw_access_token:
+        return render_token_fault(UpstreamProtocolFault(note="the upstream token response has no usable access_token"))
+
     result = {
-        "access_token": token_response["access_token"],
+        "access_token": raw_access_token,
         "token_type": token_response.get("token_type", "Bearer"),
     }
 
@@ -1813,21 +1147,6 @@ async def _persist_dcr_client_registration(
         return "failed"
 
 
-_MAX_UPSTREAM_ERROR_CHARS = 500
-
-
-def _safe_upstream_error_detail(response: httpx.Response) -> str:
-    """Bounded plaintext summary of an upstream registration failure for the client.
-
-    RFC 7591 error bodies are small JSON objects (``error`` / ``error_description``); relaying the
-    text lets the client read the real reason instead of a bare 500, and the length bound keeps a
-    hostile or oversized upstream body from bloating the gateway response."""
-    body = response.text
-    if not body:
-        return response.reason_phrase or "upstream registration failed"
-    return body[:_MAX_UPSTREAM_ERROR_CHARS]
-
-
 async def register_client_with_server(
     request: Request,
     mcp_server: MCPServer,
@@ -1887,19 +1206,24 @@ async def register_client_with_server(
     }
 
     async_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.Oauth2Register)
-    response = await async_client.post(
-        mcp_server.registration_url,
-        headers=headers,
-        json=register_data,
-    )
+    try:
+        response = await async_client.post(
+            mcp_server.registration_url,
+            headers=headers,
+            json=register_data,
+        )
+        if response is not None:
+            response.raise_for_status()
+    except httpx.HTTPStatusError as exc:
+        status_code, detail = dcr_fault_detail(
+            classify_upstream_dcr_rejection(exc.response, log_context=mcp_server.server_id)
+        )
+        raise HTTPException(status_code=status_code, detail=detail) from exc
     if response is None:
         raise HTTPException(
             status_code=502,
             detail="MCP upstream registration endpoint returned no response",
         )
-    if bridge_relay and response.status_code >= 400:
-        raise HTTPException(status_code=response.status_code, detail=_safe_upstream_error_detail(response))
-    response.raise_for_status()
 
     token_response = response.json()
 

litellm/proxy/_experimental/mcp_server/faults/__init__.py

@@ -0,0 +1,38 @@
+"""Typed fault values for upstream OAuth/DCR failures (phase 1 of the MCP error-handling framework).
+
+The invariant this package exists to enforce: an upstream failure is classified ONCE into a single
+fault value, and the response status, wire error code, and prose are all derived from that value.
+Deriving all three from one classification makes contradictory pairings (a caller-fault error code on
+a server-fault status) unrepresentable, and gives the trust-boundary rule one enforcement point:
+spec-defined machine fields may cross to callers, upstream prose and raw bodies go to server logs.
+"""
+
+from litellm.proxy._experimental.mcp_server.faults.classify import (
+    classify_upstream_dcr_rejection,
+    classify_upstream_token_rejection,
+)
+from litellm.proxy._experimental.mcp_server.faults.render_oauth import (
+    dcr_fault_detail,
+    render_token_fault,
+)
+from litellm.proxy._experimental.mcp_server.faults.types import (
+    CallerRejected,
+    CredentialSource,
+    GatewayRejected,
+    UpstreamOAuthFault,
+    UpstreamProtocolFault,
+    UpstreamReportedFault,
+)
+
+__all__ = [
+    "CallerRejected",
+    "CredentialSource",
+    "GatewayRejected",
+    "UpstreamOAuthFault",
+    "UpstreamProtocolFault",
+    "UpstreamReportedFault",
+    "classify_upstream_dcr_rejection",
+    "classify_upstream_token_rejection",
+    "dcr_fault_detail",
+    "render_token_fault",
+]

litellm/proxy/_experimental/mcp_server/faults/classify.py

@@ -0,0 +1,133 @@
+"""The single place that reads upstream OAuth/DCR failure responses.
+
+Every accessor here is total: an upstream that lies about its content encoding, sends an undecodable
+body, or omits the spec fields yields a classified fault, never an exception. Nothing outside this
+module should touch a failed upstream response's body.
+"""
+
+from __future__ import annotations
+
+import httpx
+
+from litellm._logging import verbose_logger
+from litellm.proxy._experimental.mcp_server.faults.types import (
+    GATEWAY_CAPABILITY_CODES,
+    GATEWAY_CREDENTIAL_CODES,
+    MAX_WIRE_FIELD_CHARS,
+    CallerRejected,
+    CredentialSource,
+    GatewayRejected,
+    UpstreamOAuthFault,
+    UpstreamProtocolFault,
+    UpstreamReportedFault,
+)
+
+
+def _safe_text(response: httpx.Response) -> str:
+    try:
+        return response.text
+    except Exception:
+        return ""
+
+
+def _safe_json(response: httpx.Response) -> object:
+    try:
+        return response.json()
+    except Exception:
+        return None
+
+
+def _bounded_field(value: object) -> str | None:
+    if not isinstance(value, str) or not value:
+        return None
+    return value[:MAX_WIRE_FIELD_CHARS]
+
+
+def _log_out_of_contract(endpoint_kind: str, response: httpx.Response, log_context: str) -> None:
+    verbose_logger.warning(
+        "MCP upstream %s endpoint (%s) returned HTTP %s outside the OAuth error contract (first %s chars): %s",
+        endpoint_kind,
+        log_context,
+        response.status_code,
+        MAX_WIRE_FIELD_CHARS,
+        _safe_text(response)[:MAX_WIRE_FIELD_CHARS],
+    )
+
+
+def _classify_oauth_error_code(
+    code: str,
+    description: str | None,
+    error_uri: str | None,
+    credential_source: CredentialSource,
+    log_context: str,
+) -> UpstreamOAuthFault:
+    """Blame assignment for a contract-conformant OAuth error code, shared by the token and DCR
+    classifiers. Codes by which the upstream blames itself keep that blame; ``invalid_target`` is a
+    gateway capability gap (RFC 8707 resource indicators, LIT-4339) no matter whose credentials were
+    presented; credential-indicting codes follow the credential source; everything else, including
+    codes we do not recognize, is the caller's to act on. The upstream's HTTP status is deliberately
+    never consulted: status derives from this classification at render time, which is what keeps
+    status and code from contradicting each other."""
+    if code == "server_error" or code == "temporarily_unavailable":
+        return UpstreamReportedFault(code=code)
+    if code in GATEWAY_CAPABILITY_CODES:
+        verbose_logger.warning(
+            "MCP server %s: the upstream authorization server rejected the request with "
+            "invalid_target; it may require RFC 8707 resource indicators, which the gateway "
+            "does not send yet (tracked as LIT-4339)",
+            log_context,
+        )
+        return GatewayRejected(code=code)
+    if credential_source == "gateway_stored" and code in GATEWAY_CREDENTIAL_CODES:
+        verbose_logger.warning(
+            "MCP server %s: upstream authorization server rejected the gateway's configured client "
+            "credentials (%s): %s",
+            log_context,
+            code,
+            description or "<no description>",
+        )
+        return GatewayRejected(code=code)
+    return CallerRejected(code=code, description=description, error_uri=error_uri)
+
+
+def classify_upstream_token_rejection(
+    response: httpx.Response,
+    credential_source: CredentialSource,
+    log_context: str,
+) -> UpstreamOAuthFault:
+    """Classify a token-endpoint rejection into exactly one fault: a body with an RFC 6749 §5.2
+    ``error`` field goes through blame assignment (:func:`_classify_oauth_error_code`); anything
+    without a usable ``error`` field is an upstream protocol fault."""
+    parsed = _safe_json(response)
+    fields = parsed if isinstance(parsed, dict) else {}
+    code = _bounded_field(fields.get("error"))
+    if code is None:
+        _log_out_of_contract("token", response, log_context)
+        return UpstreamProtocolFault(note=f"upstream token endpoint returned HTTP {response.status_code}")
+    return _classify_oauth_error_code(
+        code,
+        description=_bounded_field(fields.get("error_description")),
+        error_uri=_bounded_field(fields.get("error_uri")),
+        credential_source=credential_source,
+        log_context=log_context,
+    )
+
+
+def classify_upstream_dcr_rejection(response: httpx.Response, log_context: str) -> UpstreamOAuthFault:
+    """Classify a dynamic-client-registration rejection. RFC 7591 §3.2.2 errors carry
+    ``error`` / ``error_description`` and go through the same blame assignment as token errors
+    (registration sends no client credentials, so credential codes stay caller-actionable); anything
+    without a usable ``error`` field is an upstream protocol fault."""
+    parsed = _safe_json(response)
+    fields = parsed if isinstance(parsed, dict) else {}
+    code = _bounded_field(fields.get("error"))
+    if code is None:
+        _log_out_of_contract("registration", response, log_context)
+        return UpstreamProtocolFault(note=f"upstream registration failed with HTTP {response.status_code}")
+    return _classify_oauth_error_code(
+        code,
+        description=_bounded_field(fields.get("error_description")),
+        error_uri=None,
+        credential_source="caller_supplied",
+        log_context=log_context,
+    )

litellm/proxy/_experimental/mcp_server/faults/render_oauth.py

@@ -0,0 +1,89 @@
+"""Render upstream OAuth/DCR faults onto the wire. The only place that chooses statuses and bodies
+for these faults, so every consumer emits the same contract: RFC 6749 §5.2-shaped JSON with the §5.1
+no-store headers on token endpoints, HTTPException details on registration. Status, code, and prose
+all derive from the fault tag; exhaustive matches keep a new fault arm from shipping unrendered.
+"""
+
+from __future__ import annotations
+
+from fastapi.responses import JSONResponse
+from typing_extensions import assert_never
+
+from litellm.proxy._experimental.mcp_server.faults.types import UpstreamOAuthFault
+from litellm.proxy._experimental.mcp_server.oauth_utils import TOKEN_NO_CACHE_HEADERS
+
+
+def _gateway_rejected_description(code: str) -> str:
+    if code == "invalid_target":
+        return (
+            "the upstream authorization server rejected the request (invalid_target); "
+            "it may require RFC 8707 resource indicators, which the gateway does not send yet"
+        )
+    return (
+        f"the upstream authorization server rejected the gateway's configured client credentials "
+        f"({code}); verify the MCP server's client_id and client_secret"
+    )
+
+
+def _upstream_reported_status_and_description(code: str) -> tuple[int, str]:
+    if code == "temporarily_unavailable":
+        return 503, "the upstream authorization server is temporarily unavailable; retry shortly"
+    return 502, "the upstream authorization server reported an internal error"
+
+
+def render_token_fault(fault: UpstreamOAuthFault) -> JSONResponse:
+    """RFC 6749 §5.2 response for a token-endpoint fault. Caller-actionable rejections relay the
+    upstream's code on the status that code implies (401 for invalid_client per §5.2, else 400);
+    gateway-side faults are 502 ``server_error`` with gateway-authored prose so a caller is never
+    blamed for, or shown the internals of, a failure only the operator can fix."""
+    match fault.tag:
+        case "caller_rejected":
+            content = {
+                "error": fault.code,
+                **({"error_description": fault.description} if fault.description else {}),
+                **({"error_uri": fault.error_uri} if fault.error_uri else {}),
+            }
+            status_code = 401 if fault.code == "invalid_client" else 400
+            return JSONResponse(status_code=status_code, content=content, headers=TOKEN_NO_CACHE_HEADERS)
+        case "gateway_rejected":
+            return JSONResponse(
+                status_code=502,
+                content={
+                    "error": "server_error",
+                    "error_description": _gateway_rejected_description(fault.code),
+                },
+                headers=TOKEN_NO_CACHE_HEADERS,
+            )
+        case "upstream_reported_fault":
+            status_code, description = _upstream_reported_status_and_description(fault.code)
+            return JSONResponse(
+                status_code=status_code,
+                content={"error": fault.code, "error_description": description},
+                headers=TOKEN_NO_CACHE_HEADERS,
+            )
+        case "upstream_protocol_fault":
+            return JSONResponse(
+                status_code=502,
+                content={"error": "server_error", "error_description": fault.note},
+                headers=TOKEN_NO_CACHE_HEADERS,
+            )
+        case _:
+            assert_never(fault.tag)
+
+
+def dcr_fault_detail(fault: UpstreamOAuthFault) -> tuple[int, str]:
+    """Status and detail string for a registration fault, raised as HTTPException by the caller.
+    RFC 7591 §3.2.2 defines registration errors as 400, so a contract-conformant rejection is 400
+    regardless of the status the upstream chose; everything else is a 502 upstream fault."""
+    match fault.tag:
+        case "caller_rejected":
+            detail = f"{fault.code}: {fault.description}" if fault.description else fault.code
+            return 400, detail
+        case "gateway_rejected":
+            return 502, _gateway_rejected_description(fault.code)
+        case "upstream_reported_fault":
+            return _upstream_reported_status_and_description(fault.code)
+        case "upstream_protocol_fault":
+            return 502, fault.note
+        case _:
+            assert_never(fault.tag)

litellm/proxy/_experimental/mcp_server/faults/types.py

@@ -0,0 +1,79 @@
+"""Fault taxonomy for upstream OAuth token and DCR registration failures.
+
+Each fault is a frozen model on a ``tag`` literal. The tag alone decides the HTTP status, the wire
+error code, and whose prose the caller sees, so those three facts can never disagree the way they can
+when an upstream's status and error code are relayed independently.
+"""
+
+from __future__ import annotations
+
+from typing import Literal, TypeAlias
+
+from pydantic import BaseModel, ConfigDict
+
+MAX_WIRE_FIELD_CHARS = 500
+"""Bound on every upstream-derived string that crosses to a caller or into a log line."""
+
+CredentialSource: TypeAlias = Literal["gateway_stored", "caller_supplied"]
+"""Whose client credentials the gateway presented upstream: the MCP server's stored configuration or
+credentials the caller supplied on the request. Decides whether a credential rejection is the
+caller's problem to fix or the gateway operator's."""
+
+GATEWAY_CREDENTIAL_CODES: frozenset[str] = frozenset({"invalid_client", "unauthorized_client"})
+"""RFC 6749 error codes that indict the OAuth client's credentials or grant authorization. When the
+gateway presented its own stored credentials, these are gateway-side faults the caller cannot act on;
+when the caller supplied the credentials, they are the caller's to fix."""
+
+GATEWAY_CAPABILITY_CODES: frozenset[str] = frozenset({"invalid_target"})
+"""Codes that indict a gateway capability regardless of whose credentials were presented:
+``invalid_target`` means the upstream wants RFC 8707 resource indicators, which the gateway does not
+send yet (LIT-4339). Never the caller's fault."""
+
+UPSTREAM_FAULT_CODES: frozenset[str] = frozenset({"server_error", "temporarily_unavailable"})
+"""Codes by which the upstream blames itself. Relaying them as caller faults would invert blame, so
+they classify as upstream-reported faults and render on the 5xx their meaning implies."""
+
+
+class CallerRejected(BaseModel):
+    """The upstream spoke the OAuth error contract and the failure is actionable by our caller
+    (e.g. ``invalid_grant``: re-run authorization). The code and its bounded prose relay on the
+    4xx status the code itself implies."""
+
+    model_config = ConfigDict(frozen=True)
+    tag: Literal["caller_rejected"] = "caller_rejected"
+    code: str
+    description: str | None = None
+    error_uri: str | None = None
+
+
+class GatewayRejected(BaseModel):
+    """The upstream rejected the request for a cause only the gateway operator can address: the
+    server's stored client credentials or a gateway capability gap. Not actionable by the caller:
+    rendered as 502 with gateway-authored prose naming the code; the upstream's prose goes to
+    server logs only."""
+
+    model_config = ConfigDict(frozen=True)
+    tag: Literal["gateway_rejected"] = "gateway_rejected"
+    code: str
+
+
+class UpstreamReportedFault(BaseModel):
+    """The upstream blamed itself in the OAuth vocabulary. Rendered on the 5xx the code implies
+    (``server_error`` 502, ``temporarily_unavailable`` 503) so blame and status agree."""
+
+    model_config = ConfigDict(frozen=True)
+    tag: Literal["upstream_reported_fault"] = "upstream_reported_fault"
+    code: Literal["server_error", "temporarily_unavailable"]
+
+
+class UpstreamProtocolFault(BaseModel):
+    """The upstream broke the error contract: no JSON ``error`` field, an undecodable body, or a
+    success response without a usable token. Rendered as 502 with a gateway-authored note; the
+    upstream body never crosses to the caller."""
+
+    model_config = ConfigDict(frozen=True)
+    tag: Literal["upstream_protocol_fault"] = "upstream_protocol_fault"
+    note: str
+
+
+UpstreamOAuthFault: TypeAlias = CallerRejected | GatewayRejected | UpstreamReportedFault | UpstreamProtocolFault

litellm/proxy/_experimental/mcp_server/mcp_server_manager.py

@@ -186,6 +186,144 @@ _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES: tuple[MCPAuth, ...] = (
 )
 
 
+def _blank_to_none(value: str | None) -> str | None:
+    """Collapse an absent, empty, or whitespace-only string to ``None``.
+
+    OAuth endpoint fields are consumed by truthiness-based merges (``row or discovered``) and by the
+    corroboration gate. A whitespace-only value is truthy to ``or`` but is not a usable endpoint, so
+    without this the merge would keep the blank value for redirects while the gate treats it as
+    unpinned and backfills the other fields, yielding a broken half-discovered config. Normalizing
+    the pinned fields once, at each build entry point, gives every downstream consumer a single
+    notion of "blank" so those code paths cannot disagree.
+    """
+    if not isinstance(value, str):
+        return None
+    return value.strip() or None
+
+
+def _normalized_authorize_endpoint(url: str) -> str:
+    """Compare authorize endpoints on scheme, host, and path only. The default port is elided and
+    the host is lowercased so ``https://IDP.example.com:443/authorize/`` and
+    ``https://idp.example.com/authorize`` are the same identity; query and trailing slash are not."""
+    parsed = urlparse(url)
+    scheme = parsed.scheme.lower()
+    host = (parsed.hostname or "").lower()
+    default_port = {"https": 443, "http": 80}.get(scheme)
+    try:
+        port = parsed.port
+    except ValueError:
+        port = None
+    authority = host if port is None or port == default_port else f"{host}:{port}"
+    return f"{scheme}://{authority}{parsed.path.rstrip('/')}"
+
+
+def _endpoints_corroborate_authorization_url(
+    source_authorization_url: str | None,
+    trusted_authorization_url: str | None,
+) -> bool:
+    """Whether a source's ``token_url``/``registration_url`` may be paired with a trusted authorize
+    endpoint. This is the single trust rule for adopting OAuth endpoints from any non-manual source.
+
+    Discovery is rooted at the MCP resource (RFC 9728), so a compromised upstream can advertise an
+    attacker-run authorization server. When ``authorization_url`` is admin-pinned, pairing it with a
+    ``token_url`` from a different source is the RFC 9700 authorization-server mix-up: the user signs
+    in at the trusted authorize endpoint while the gateway redeems the code, with the stored client
+    secret and PKCE verifier, at the attacker's token endpoint. Endpoints are trustworthy together
+    only when they share an authorization server, so a source's endpoints are adopted only when the
+    same source advertised an ``authorization_endpoint`` matching the pinned value. With no pinned
+    value (``trusted_authorization_url is None``) there is nothing to protect: the authorize endpoint
+    comes from the same source as the token endpoint, so they corroborate each other by construction.
+    """
+    if not (trusted_authorization_url and trusted_authorization_url.strip()):
+        return True
+    return bool(source_authorization_url) and _normalized_authorize_endpoint(
+        source_authorization_url
+    ) == _normalized_authorize_endpoint(trusted_authorization_url)
+
+
+def _carry_forward_resolved_oauth_endpoints(new_server: MCPServer, previous_server: MCPServer | None) -> None:
+    """Keep the last known good OAuth endpoints when a rebuild's re-discovery comes back empty.
+
+    A rebuild wholesale-replaces the registry entry, so without this a transient upstream outage
+    during re-discovery downgrades a working server (``authorization_url`` set) to a broken one
+    (``None``, /authorize 400s) with no configuration change. Mirrors the ``short_prefix``
+    carry-forward. Skipped when the server's ``url`` or ``auth_type`` changed, since the previous
+    endpoints may then belong to a different upstream. ``registration_url`` IS carried even though
+    ``_persist_discovered_oauth_endpoints`` refuses to write it to the row: carrying only restores
+    the same in-memory value the previous build already ran with, while persisting it would flip
+    ``_dcr_bridge_relays_client_registration`` (which keys off the stored column) for dcr_bridge
+    servers that never had one configured.
+
+    Carry-forward is a non-manual endpoint source, so the same trust rule as discovery applies: the
+    previous ``token_url``/``registration_url``/``scopes`` are carried only when the previous
+    ``authorization_url`` corroborates the authorize endpoint this build will use, i.e. when the
+    incoming build has no pinned authorize endpoint (``None`` -> we adopt the previous one too, a
+    consistent group) or pins the same one. An admin re-pointing ``authorization_url`` to a different
+    server must not keep serving the old server's token endpoint or granted scopes.
+    """
+    if previous_server is None:
+        return
+    if previous_server.url != new_server.url or previous_server.auth_type != new_server.auth_type:
+        return
+    may_carry = _endpoints_corroborate_authorization_url(
+        previous_server.authorization_url, new_server.authorization_url
+    )
+    if new_server.authorization_url is None and previous_server.authorization_url:
+        new_server.authorization_url = previous_server.authorization_url
+    if may_carry and new_server.token_url is None and previous_server.token_url:
+        new_server.token_url = previous_server.token_url
+    if may_carry and new_server.registration_url is None and previous_server.registration_url:
+        new_server.registration_url = previous_server.registration_url
+    if may_carry and not new_server.scopes and previous_server.scopes:
+        new_server.scopes = previous_server.scopes
+
+
+def _restrict_discovery_to_corroborated_authorization_server(
+    metadata: MCPOAuthMetadata | None,
+    manual_authorization_url: str | None,
+    server_identifier: str,
+    is_dcr_bridge: bool,
+) -> MCPOAuthMetadata | None:
+    """Reject discovered token/registration endpoints a manually pinned authorize endpoint cannot
+    vouch for (the RFC 9700 authorization-server mix-up).
+
+    Discovery is rooted at the MCP resource, so a compromised upstream can advertise an attacker
+    ``token_endpoint``: with ``authorization_url`` admin-pinned but ``token_url`` blank, the merge
+    would pair the trusted authorize endpoint with that attacker token endpoint, and the gateway would
+    post the authorization code and client secret there. So the discovered ``token_url`` and
+    ``registration_url`` are kept only if the document corroborates the pin (its
+    ``authorization_endpoint`` matches). ``scopes`` are deliberately NOT gated here: per the MCP
+    authorization spec Scope Selection Strategy and RFC 9700 §2.3, the scopes a client requests are
+    resource-driven (the WWW-Authenticate challenge or the RFC 9728 protected-resource
+    ``scopes_supported``), and scope inflation by a compromised resource is bounded by the
+    authorization server and user consent (RFC 6749 §3.3), not by the client second-guessing the
+    request. With no pin there is no trust anchor to protect, so discovery is returned as-is.
+    """
+    if metadata is None or not (manual_authorization_url and manual_authorization_url.strip()):
+        return metadata
+    if _endpoints_corroborate_authorization_url(metadata.authorization_url, manual_authorization_url):
+        return metadata
+    if not metadata.token_url and not metadata.registration_url:
+        return metadata
+    bridge_note = (
+        " The discovered registration_url is rejected with it, so this dcr_bridge server stays on the"
+        " short-circuit registration arm."
+        if is_dcr_bridge and metadata.registration_url
+        else ""
+    )
+    verbose_logger.warning(
+        "MCP OAuth discovery for server %s advertised authorization_endpoint %s, which does not match the "
+        "manually configured authorization_url %s; rejecting the discovered token_url/registration_url so "
+        "authorization codes and client credentials only follow the configured authorization server. "
+        "Configure Token URL manually if the mismatch is intentional.%s",
+        server_identifier,
+        _normalized_authorize_endpoint(metadata.authorization_url) if metadata.authorization_url else "<absent>",
+        _normalized_authorize_endpoint(manual_authorization_url),
+        bridge_note,
+    )
+    return metadata.model_copy(update={"token_url": None, "registration_url": None})
+
+
 def invalidate_user_env_vars_cache(user_id: str, server_id: str) -> None:
     """Drop a cached entry after the user stores or clears their env var values
     so the next request reads the fresh value instead of a stale one."""
@@ -999,12 +1137,15 @@ class MCPServerManager:
             )
 
             auth_type = server_config.get("auth_type", None)
+            manual_authorization_url = _blank_to_none(server_config.get("authorization_url"))
+            manual_token_url = _blank_to_none(server_config.get("token_url"))
+            manual_registration_url = _blank_to_none(server_config.get("registration_url"))
             if server_url and (
                 auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES
                 or self._obo_needs_endpoint_discovery(
                     auth_type,
                     server_config.get("token_exchange_endpoint"),
-                    server_config.get("token_url"),
+                    manual_token_url,
                 )
             ):
                 mcp_oauth_metadata = await self._descovery_metadata(
@@ -1014,20 +1155,29 @@ class MCPServerManager:
             else:
                 mcp_oauth_metadata = None
 
+            gated_oauth_metadata = (
+                _restrict_discovery_to_corroborated_authorization_server(
+                    mcp_oauth_metadata,
+                    manual_authorization_url,
+                    server_name or server_id,
+                    bool(server_config.get("dcr_bridge")),
+                )
+                if auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES
+                else mcp_oauth_metadata
+            )
+
             # Filter blank scopes (e.g. YAML ``scopes: [""]``) the same way the DB-build path does, so
             # an all-blank list normalizes to None rather than a ``("",)`` tuple that skips the
             # entra_obo fail-closed scope precondition and POSTs an empty scope to the IdP.
             resolved_scopes = self._extract_scopes(server_config.get("scopes")) or (
-                mcp_oauth_metadata.scopes if mcp_oauth_metadata else None
+                gated_oauth_metadata.scopes if gated_oauth_metadata else None
             )
-            resolved_authorization_url = server_config.get("authorization_url") or (
-                mcp_oauth_metadata.authorization_url if mcp_oauth_metadata else None
+            resolved_authorization_url = manual_authorization_url or (
+                gated_oauth_metadata.authorization_url if gated_oauth_metadata else None
             )
-            resolved_token_url = server_config.get("token_url") or (
-                mcp_oauth_metadata.token_url if mcp_oauth_metadata else None
-            )
-            resolved_registration_url = server_config.get("registration_url") or (
-                mcp_oauth_metadata.registration_url if mcp_oauth_metadata else None
+            resolved_token_url = manual_token_url or (gated_oauth_metadata.token_url if gated_oauth_metadata else None)
+            resolved_registration_url = manual_registration_url or (
+                gated_oauth_metadata.registration_url if gated_oauth_metadata else None
             )
 
             config_oauth2_flow = server_config.get("oauth2_flow", None)
@@ -1343,6 +1493,7 @@ class MCPServerManager:
         *,
         credentials_are_encrypted: bool = True,
         env_vars_are_encrypted: Optional[bool] = None,
+        persist_discovered_endpoints: bool = True,
     ) -> MCPServer:
         _mcp_info: MCPInfo = mcp_server.mcp_info or {}
         env_dict = _deserialize_json_dict(getattr(mcp_server, "env", None))
@@ -1419,13 +1570,17 @@ class MCPServerManager:
 
         auth_type = cast(MCPAuthType, mcp_server.auth_type)
         server_url = mcp_server.url
+        manual_authorization_url = _blank_to_none(mcp_server.authorization_url)
+        manual_token_url = _blank_to_none(mcp_server.token_url)
+        manual_registration_url = _blank_to_none(mcp_server.registration_url)
+        has_all_upstream_oauth_fields = bool(manual_authorization_url and manual_token_url and scopes)
         needs_discovery = bool(server_url) and (
-            (auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES and not mcp_server.authorization_url)
+            (auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES and not has_all_upstream_oauth_fields)
             or self._obo_needs_endpoint_discovery(
                 auth_type,
                 mcp_server.token_exchange_endpoint
                 or (credentials_dict.get("token_exchange_endpoint") if credentials_dict else None),
-                mcp_server.token_url,
+                manual_token_url,
             )
         )
         mcp_oauth_metadata = (
@@ -1436,8 +1591,25 @@ class MCPServerManager:
             if needs_discovery
             else None
         )
+        if needs_discovery and mcp_oauth_metadata is None:
+            verbose_logger.warning(
+                "MCP OAuth discovery yielded no metadata for server %s (%s); "
+                "OAuth endpoints/scopes stay unresolved until a rebuild succeeds",
+                mcp_server.server_id,
+                server_url,
+            )
+        gated_oauth_metadata = (
+            _restrict_discovery_to_corroborated_authorization_server(
+                mcp_oauth_metadata,
+                manual_authorization_url,
+                mcp_server.server_id,
+                bool(getattr(mcp_server, "dcr_bridge", None)),
+            )
+            if auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES
+            else mcp_oauth_metadata
+        )
 
-        resolved_scopes = scopes or (mcp_oauth_metadata.scopes if mcp_oauth_metadata else None)
+        resolved_scopes = scopes or (gated_oauth_metadata.scopes if gated_oauth_metadata else None)
 
         new_server = MCPServer(
             server_id=mcp_server.server_id,
@@ -1457,9 +1629,9 @@ class MCPServerManager:
             client_secret=client_secret_value or getattr(mcp_server, "client_secret", None),
             oauth2_flow=self._explicit_oauth2_flow(getattr(mcp_server, "oauth2_flow", None)),
             scopes=resolved_scopes,
-            authorization_url=mcp_server.authorization_url or getattr(mcp_oauth_metadata, "authorization_url", None),
-            token_url=mcp_server.token_url or getattr(mcp_oauth_metadata, "token_url", None),
-            registration_url=mcp_server.registration_url or getattr(mcp_oauth_metadata, "registration_url", None),
+            authorization_url=manual_authorization_url or getattr(gated_oauth_metadata, "authorization_url", None),
+            token_url=manual_token_url or getattr(gated_oauth_metadata, "token_url", None),
+            registration_url=manual_registration_url or getattr(gated_oauth_metadata, "registration_url", None),
             token_endpoint_auth_method=(
                 credentials_dict.get("token_endpoint_auth_method") if credentials_dict else None
             ),
@@ -1506,12 +1678,21 @@ class MCPServerManager:
             max_concurrent_requests=getattr(mcp_server, "max_concurrent_requests", None),
         )
         _warn_internal_delegate_pkce_if_applicable(new_server, source="database")
-        await self._persist_discovered_obo_token_url(
-            server_id=mcp_server.server_id,
-            auth_type=auth_type,
-            existing_token_url=mcp_server.token_url,
-            discovered_token_url=new_server.token_url,
-        )
+        if persist_discovered_endpoints:
+            await self._persist_discovered_obo_token_url(
+                server_id=mcp_server.server_id,
+                auth_type=auth_type,
+                existing_token_url=manual_token_url,
+                discovered_token_url=new_server.token_url,
+            )
+            await self._persist_discovered_oauth_endpoints(
+                server_id=mcp_server.server_id,
+                auth_type=auth_type,
+                existing_authorization_url=manual_authorization_url,
+                existing_token_url=manual_token_url,
+                existing_scopes=scopes,
+                metadata=gated_oauth_metadata,
+            )
         return new_server
 
     async def _persist_discovered_obo_token_url(
@@ -1549,6 +1730,69 @@ class MCPServerManager:
         except Exception as exc:  # noqa: BLE001 - best-effort; a failed write re-discovers next build
             verbose_logger.warning("Failed to persist discovered OBO token_url for MCP server %s: %s", server_id, exc)
 
+    async def _persist_discovered_oauth_endpoints(
+        self,
+        *,
+        server_id: str,
+        auth_type: MCPAuthType | None,
+        existing_authorization_url: str | None,
+        existing_token_url: str | None,
+        existing_scopes: list[str] | None,
+        metadata: MCPOAuthMetadata | None,
+    ) -> None:
+        """Write freshly discovered OAuth endpoints back onto the DB row.
+
+        Same rationale as ``_persist_discovered_obo_token_url`` but for the interactive oauth2
+        family: discovered ``authorization_url``/``token_url``/``scopes`` otherwise live only on
+        the in-memory registry entry, which is rebuilt on every client connect (the DCR reuse path
+        calls ``update_server``) and on every post-write DB reload, so one failed re-discovery
+        serves 400 "authorization url is not set" from /authorize until a later rebuild succeeds.
+        Only fills row fields that are currently empty, never persists origin-fallback guesses
+        (RFC 9728/8414-advertised metadata only), and deliberately skips ``registration_url``
+        because ``_dcr_bridge_relays_client_registration`` keys off that column. Best-effort: a
+        failed write re-discovers on the next build. Scopes go through ``update_mcp_server`` so
+        they merge into the credentials blob without touching the stored client credentials.
+        """
+        if auth_type not in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES:
+            return
+        if metadata is None or metadata.from_origin_fallback:
+            return
+        authorization_url_update = (
+            {"authorization_url": metadata.authorization_url}
+            if metadata.authorization_url and not existing_authorization_url
+            else {}
+        )
+        token_url_update = {"token_url": metadata.token_url} if metadata.token_url and not existing_token_url else {}
+        scopes_update = {"credentials": {"scopes": metadata.scopes}} if metadata.scopes and not existing_scopes else {}
+        updates: dict[str, object] = {**authorization_url_update, **token_url_update, **scopes_update}
+        if not updates:
+            return
+        from litellm.proxy._experimental.mcp_server.db import (  # noqa: PLC0415  # db.py imports this module at load
+            update_mcp_server,
+        )
+        from litellm.proxy._types import UpdateMCPServerRequest  # noqa: PLC0415  # heavy module; import at call time
+        from litellm.proxy.proxy_server import prisma_client  # noqa: PLC0415  # runtime value, set after startup
+
+        if prisma_client is None:
+            return
+        try:
+            await update_mcp_server(
+                prisma_client=prisma_client,
+                data=UpdateMCPServerRequest.model_validate({"server_id": server_id, **updates}),
+                touched_by="mcp_oauth_discovery",
+            )
+            verbose_logger.info(
+                "Persisted discovered OAuth endpoints for MCP server %s: %s",
+                server_id,
+                sorted(updates),
+            )
+        except Exception as exc:  # noqa: BLE001 - best-effort; a failed write re-discovers next build
+            verbose_logger.warning(
+                "Failed to persist discovered OAuth endpoints for MCP server %s: %s",
+                server_id,
+                exc,
+            )
+
     async def _maybe_register_openapi_tools(self, server: MCPServer, *, initialize_mapping: bool = True):
         """Register OpenAPI tools if the server has a spec_path configured."""
         if server.spec_path:
@@ -1607,6 +1851,10 @@ class MCPServerManager:
                 existing_prefix = self.registry[mcp_server.server_id].short_prefix
                 if existing_prefix and not new_server.short_prefix:
                     new_server.short_prefix = existing_prefix
+                _carry_forward_resolved_oauth_endpoints(
+                    new_server=new_server,
+                    previous_server=self.registry[mcp_server.server_id],
+                )
                 self._assign_unique_short_prefix(new_server)
                 self.registry[mcp_server.server_id] = new_server
                 await self._maybe_register_openapi_tools(new_server)
@@ -2969,16 +3217,20 @@ class MCPServerManager:
                 ) = await self._attempt_well_known_discovery(server_url)
 
             metadata = None
+            used_origin_fallback = False
             if allow_origin_fallback and not authorization_servers:
                 try:
                     parsed_url = urlparse(server_url)
                     if parsed_url.scheme and parsed_url.netloc:
                         authorization_servers = [f"{parsed_url.scheme}://{parsed_url.netloc}"]
+                        used_origin_fallback = True
                 except Exception:
                     authorization_servers = []
 
             if authorization_servers:
                 metadata = await self._fetch_authorization_server_metadata(authorization_servers, server_url)
+                if metadata is not None and used_origin_fallback:
+                    metadata.from_origin_fallback = True
 
             preferred_scopes = scopes or resource_scopes
             if metadata is None and preferred_scopes:
@@ -4489,6 +4741,7 @@ class MCPServerManager:
                 # (if any) so the prefix is stable across reloads.
                 if existing_server is not None and existing_server.short_prefix:
                     new_server.short_prefix = existing_server.short_prefix
+                _carry_forward_resolved_oauth_endpoints(new_server=new_server, previous_server=existing_server)
                 new_registry[server.server_id] = new_server
             except Exception as e:
                 verbose_logger.exception(

litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py

@@ -175,16 +175,18 @@ mcp_oauth2_token_cache = MCPOAuth2TokenCache()
 def _compute_per_user_token_ttl(server: "MCPServer", expires_in: Optional[int]) -> int:
     """Compute Redis TTL for a per-user token.
 
-    Uses server.token_storage_ttl_seconds when configured; otherwise derives
-    TTL from expires_in minus the expiry buffer; falls back to the default TTL.
+    Uses server.token_storage_ttl_seconds when configured, capped at the token's
+    remaining lifetime (expires_in minus the expiry buffer) so a cached entry never
+    outlives the token itself; otherwise derives TTL from expires_in minus the
+    expiry buffer; falls back to the default TTL.
     """
+    lifetime_bound = expires_in - MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS if expires_in is not None else None
     if server.token_storage_ttl_seconds is not None:
-        return max(server.token_storage_ttl_seconds, 1)
-    if expires_in is not None:
-        return max(
-            expires_in - MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS,
-            1,
-        )
+        if lifetime_bound is None:
+            return max(server.token_storage_ttl_seconds, 1)
+        return max(min(server.token_storage_ttl_seconds, lifetime_bound), 1)
+    if lifetime_bound is not None:
+        return max(lifetime_bound, 1)
     return MCP_PER_USER_TOKEN_DEFAULT_TTL
 
 

litellm/proxy/_lazy_openapi_snapshot.json

@@ -7613,6 +7613,18 @@
               ],
               "title": "Messages"
             },
+            "metadata": {
+              "anyOf": [
+                {
+                  "additionalProperties": true,
+                  "type": "object"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Metadata"
+            },
             "text": {
               "title": "Text",
               "type": "string"

litellm/proxy/_types.py

@@ -1118,6 +1118,7 @@ class GenerateKeyRequest(KeyRequestBase):
 class GenerateKeyResponse(KeyRequestBase):
     key: str  # type: ignore
     key_name: Optional[str] = None
+    key_type: str | None = None
     expires: Optional[datetime] = None
     user_id: Optional[str] = None
     token_id: Optional[str] = None
@@ -2421,6 +2422,16 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
             "is active as a reminder that hard enforcement is relaxed."
         ),
     )
+    skip_user_budget_on_team_key: bool | None = Field(
+        None,
+        description=(
+            "If True, restores the legacy behavior where a user's personal "
+            "max_budget is NOT enforced when their key belongs to a team; only "
+            "the team (and team-member) budgets apply. Defaults to False, meaning "
+            "the user's personal max_budget is always enforced regardless of "
+            "whether the key belongs to a team (see GitHub issue #12905)."
+        ),
+    )
     user_url_validation: Optional[bool] = Field(
         None,
         description=(

litellm/proxy/auth/auth_checks.py

@@ -626,26 +626,29 @@ async def common_checks(
             )
 
         async def _user_max_budget_check() -> None:
-            # 4.1 personal budget, if personal key
-            if (
-                (team_object is None or team_object.team_id is None)
-                and user_object is not None
-                and user_object.max_budget is not None
-            ):
-                from litellm.proxy.proxy_server import get_current_spend
+            if user_object is None or user_object.max_budget is None:
+                return
+            skip_for_team = (
+                general_settings.get("skip_user_budget_on_team_key") is True
+                and team_object is not None
+                and team_object.team_id is not None
+            )
+            if skip_for_team:
+                return
+            from litellm.proxy.proxy_server import get_current_spend
 
-                user_budget = user_object.max_budget
-                user_spend = await get_current_spend(
-                    counter_key=f"spend:user:{user_object.user_id}",
-                    fallback_spend=user_object.spend or 0.0,
+            user_budget = user_object.max_budget
+            user_spend = await get_current_spend(
+                counter_key=f"spend:user:{user_object.user_id}",
+                fallback_spend=user_object.spend or 0.0,
+                max_budget=user_budget,
+            )
+            if math.isfinite(user_budget) and user_spend >= user_budget:
+                raise litellm.BudgetExceededError(
+                    current_cost=user_spend,
                     max_budget=user_budget,
+                    message=f"ExceededBudget: User={user_object.user_id} over budget. Spend={user_spend}, Budget={user_budget}",
                 )
-                if math.isfinite(user_budget) and user_spend >= user_budget:
-                    raise litellm.BudgetExceededError(
-                        current_cost=user_spend,
-                        max_budget=user_budget,
-                        message=f"ExceededBudget: User={user_object.user_id} over budget. Spend={user_spend}, Budget={user_budget}",
-                    )
 
         # Each scope reads a distinct counter key with no cross-scope ordering
         # dependency, so the per-scope Redis-first reads run concurrently instead

litellm/proxy/auth/auth_utils.py

@@ -10,7 +10,7 @@ from fastapi import HTTPException, Request, status
 import litellm
 from litellm import Router, provider_list
 from litellm._logging import verbose_proxy_logger
-from litellm.constants import STANDARD_CUSTOMER_ID_HEADERS
+from litellm.constants import MINIMUM_CUSTOM_KEY_LENGTH, STANDARD_CUSTOMER_ID_HEADERS
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
 from litellm.litellm_core_utils.url_utils import SSRFError, validate_url
 from litellm.proxy._types import *
@@ -1533,4 +1533,6 @@ def get_model_from_request(
 
 
 def abbreviate_api_key(api_key: str) -> str:
+    if len(api_key) < MINIMUM_CUSTOM_KEY_LENGTH:
+        return "sk-..."
     return f"sk-...{api_key[-4:]}"

litellm/proxy/auth/user_api_key_auth.py

@@ -1191,13 +1191,15 @@ async def _user_api_key_auth_builder(
             return await handle_oauth2_proxy_request(request=request)
 
         if general_settings.get("enable_jwt_auth", False) is True:
-            from litellm.proxy.proxy_server import premium_user
-
-            if premium_user is not True:
-                raise ValueError(f"JWT Auth is an enterprise only feature. {CommonProxyErrors.not_premium_user.value}")
             is_jwt = jwt_handler.is_jwt(token=api_key)
             verbose_proxy_logger.debug("is_jwt: %s", is_jwt)
             if is_jwt:
+                from litellm.proxy.proxy_server import premium_user
+
+                if premium_user is not True:
+                    raise ValueError(
+                        f"JWT Auth is an enterprise only feature. {CommonProxyErrors.not_premium_user.value}"
+                    )
                 # Try JWT-to-Virtual-Key mapping first to avoid
                 # unnecessary DB queries in auth_builder
                 do_standard_jwt_auth = True
@@ -2442,6 +2444,7 @@ async def _reserve_budget_after_common_checks(
         proxy_logging_obj=proxy_logging_obj,
         end_user_id=end_user_id,
         end_user_object=end_user_object,
+        skip_user_budget_on_team_key=general_settings.get("skip_user_budget_on_team_key") is True,
     )
 
 

litellm/proxy/client/cli/commands/auth.py

@@ -277,6 +277,28 @@ def prompt_team_selection_fallback(
             return None
 
 
+def _response_error_detail(response: requests.Response) -> str | None:
+    try:
+        body = response.json()
+    except ValueError:
+        return None
+    detail = body.get("detail") if isinstance(body, dict) else None
+    if isinstance(detail, str) and detail:
+        return detail
+    return None
+
+
+def _polling_error_message(response: requests.Response) -> str:
+    detail = _response_error_detail(response)
+    if detail:
+        return f"Polling error: HTTP {response.status_code}: {detail}"
+    return f"Polling error: HTTP {response.status_code}"
+
+
+def _is_permanent_polling_error(status_code: int) -> bool:
+    return 400 <= status_code < 500 and status_code != 429
+
+
 # Polling-based authentication - no local server needed
 def _poll_for_ready_data(
     url: str,
@@ -308,8 +330,14 @@ def _poll_for_ready_data(
                         click.echo(pending_message)
                 elif other_status_message and other_status_log_every > 0 and attempt % other_status_log_every == 0:
                     click.echo(other_status_message)
+            elif _is_permanent_polling_error(response.status_code):
+                detail = _response_error_detail(response)
+                raise ValueError(
+                    f"The proxy rejected the login session with HTTP {response.status_code}"
+                    + (f": {detail}" if detail else f" and no error detail (from {url})")
+                )
             elif http_error_log_every > 0 and attempt % http_error_log_every == 0:
-                click.echo(f"Polling error: HTTP {response.status_code}")
+                click.echo(_polling_error_message(response))
         except requests.RequestException as e:
             if connection_error_log_every > 0 and attempt % connection_error_log_every == 0:
                 click.echo(f"Connection error (will retry): {e}")
@@ -342,12 +370,45 @@ def _normalize_teams(teams, team_details):
 
 
 def _start_cli_sso_flow(base_url: str) -> Dict[str, Any]:
-    response = requests.post(f"{base_url}/sso/cli/start", timeout=10)
-    response.raise_for_status()
-    data = response.json()
+    start_url = f"{base_url}/sso/cli/start"
+    try:
+        response = requests.post(start_url, timeout=10)
+    except requests.RequestException as e:
+        raise ValueError(
+            f"Could not reach the proxy at {start_url}: {e}. "
+            "Check that the proxy is running and that --base-url points at it."
+        ) from e
+
+    if response.status_code in (404, 405):
+        raise ValueError(
+            f"POST {start_url} returned HTTP {response.status_code}. "
+            "Either --base-url is wrong, or the proxy is older than this CLI and does not support "
+            "the CLI SSO login flow; upgrade the proxy or use a CLI version that matches it."
+        )
+    if response.status_code != 200:
+        detail = _response_error_detail(response)
+        raise ValueError(
+            f"Starting CLI login failed: HTTP {response.status_code} from {start_url}"
+            + (f": {detail}" if detail else "")
+        )
+
+    try:
+        data = response.json()
+    except ValueError:
+        content_type = response.headers.get("content-type", "unknown")
+        raise ValueError(
+            f"The proxy returned a non-JSON response from {start_url} (content-type: {content_type}). "
+            "A proxy, load balancer, or auth gateway in front of LiteLLM may be intercepting the request. "
+            f"Response starts with: {response.text[:200]!r}"
+        )
+
     required_fields = ("login_id", "poll_secret", "user_code")
-    if not all(isinstance(data.get(field), str) for field in required_fields):
-        raise ValueError("Invalid CLI SSO start response")
+    missing_fields = tuple(field for field in required_fields if not isinstance(data.get(field), str))
+    if missing_fields:
+        raise ValueError(
+            f"The response from {start_url} is missing required field(s): {', '.join(missing_fields)}. "
+            "The proxy version may not match this CLI; upgrade whichever is older."
+        )
     return data
 
 
@@ -577,6 +638,10 @@ def login(ctx: click.Context):
             return
         else:
             click.echo("❌ Authentication timed out. Please try again.")
+            click.echo(
+                "The proxy never reported the browser sign-in as finished. If you did complete it, "
+                "check the proxy logs for /sso/callback errors and confirm SSO is configured on the proxy."
+            )
             return
 
     except KeyboardInterrupt:

litellm/proxy/enterprise_billing/billing_metrics.py

@@ -0,0 +1,323 @@
+"""
+Push-based OTLP metering for enterprise litellm deployments.
+
+Owns a dedicated OpenTelemetry meter provider and an OTLP/HTTP exporter
+authenticated to our global collector with a TLS client certificate. The
+collector front end terminates mutual TLS: the client certificate presented
+here is validated against our CA at the edge, and the verified subject is
+what identifies the deployment. It is intentionally isolated from the global
+meter provider so the customer's own OTEL metrics are untouched and ours
+never leak into their backend.
+
+The deployment's identity rides on the TLS client certificate, not on the
+payload; the secret license key is never sent as an attribute or header.
+"""
+
+import os
+import tempfile
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Optional, Union
+
+from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
+from opentelemetry.metrics import Counter
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
+from opentelemetry.sdk.resources import Resource
+
+from litellm._logging import verbose_proxy_logger
+from litellm.proxy.middleware.billable_request_metrics_middleware import (
+    BillableCategory,
+)
+
+if TYPE_CHECKING:
+    from litellm.proxy._types import EnterpriseLicenseData
+
+ENDPOINT_ENV = "LITELLM_BILLING_METRICS_ENDPOINT"
+CLIENT_CERT_ENV = "LITELLM_BILLING_METRICS_CLIENT_CERT"
+CLIENT_KEY_ENV = "LITELLM_BILLING_METRICS_CLIENT_KEY"
+CA_CERT_ENV = "LITELLM_BILLING_METRICS_CA_CERT"
+EXPORT_INTERVAL_ENV = "LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS"
+DEFAULT_EXPORT_INTERVAL_MS = 60_000
+SHUTDOWN_FLUSH_TIMEOUT_MS = 5_000
+_METRICS_PATH = "/v1/metrics"
+
+# The cert env vars take a path or the PEM itself. Secret stores that inject
+# values as env content cannot mount them as files, so inline PEM is written out.
+_PEM_PREFIX = "-----BEGIN"
+_PEM_DIR_PREFIX = "litellm-billing-mtls-"
+_PEM_FILE_MODE = 0o600
+_CLIENT_CERT_FILENAME = "client.crt"
+_CLIENT_KEY_FILENAME = "client.key"
+_CA_CERT_FILENAME = "ca.crt"
+
+METRIC_NAME = "litellm.enterprise.billable_requests"
+METER_NAME = "litellm.enterprise.billing"
+
+AttributeValue = Union[str, int]
+
+
+@dataclass(frozen=True, slots=True)
+class BillingMetricsConfig:
+    endpoint: str
+    client_cert_path: str
+    client_key_path: str
+    ca_cert_path: Optional[str]
+    export_interval_ms: int
+    litellm_version: str
+    license_id: Optional[str]
+
+
+def _metrics_endpoint(endpoint: str) -> str:
+    """The OTLP/HTTP metric exporter wants the full URL including the signal path."""
+    trimmed = endpoint.rstrip("/")
+    return trimmed if trimmed.endswith(_METRICS_PATH) else f"{trimmed}{_METRICS_PATH}"
+
+
+def _resource_attributes(config: BillingMetricsConfig) -> dict[str, AttributeValue]:
+    base: dict[str, AttributeValue] = {
+        "service.name": "litellm-proxy",
+        "litellm.version": config.litellm_version,
+    }
+    license_attr: dict[str, AttributeValue] = {"litellm.license.id": config.license_id} if config.license_id else {}
+    return {**base, **license_attr}
+
+
+def _billable_attributes(
+    category: BillableCategory, route: str, status_code: int, model_id: Optional[str]
+) -> dict[str, AttributeValue]:
+    base: dict[str, AttributeValue] = {
+        "litellm.endpoint.category": category.value,
+        "http.route": route,
+        "http.response.status_code": status_code,
+    }
+    model_attr: dict[str, AttributeValue] = {"litellm.model_id": model_id} if model_id else {}
+    return {**base, **model_attr}
+
+
+def build_mtls_meter_provider(config: BillingMetricsConfig) -> MeterProvider:
+    """OTLP/HTTP exporter presenting a TLS client certificate.
+
+    The collector's load balancer terminates mutual TLS and validates the client
+    certificate against our CA. Server verification uses the system trust store
+    (the collector presents a public web-PKI certificate); ca_cert_path overrides
+    it only for private/test collectors.
+    """
+    exporter = OTLPMetricExporter(
+        endpoint=_metrics_endpoint(config.endpoint),
+        # None -> exporter falls back to the system trust store.
+        certificate_file=config.ca_cert_path,
+        client_certificate_file=config.client_cert_path,
+        client_key_file=config.client_key_path,
+    )
+    reader = PeriodicExportingMetricReader(exporter, export_interval_millis=config.export_interval_ms)
+    return MeterProvider(metric_readers=[reader], resource=Resource.create(_resource_attributes(config)))
+
+
+class BillingMetricsRecorder:
+    """Increments one OTLP counter per billable request. The meter provider is injected (see the factory)."""
+
+    def __init__(self, provider: MeterProvider) -> None:
+        self._provider = provider
+        self._counter: Counter = provider.get_meter(METER_NAME).create_counter(
+            name=METRIC_NAME,
+            unit="{request}",
+            description="Count of 2xx HTTP requests to billable LLM/MCP/A2A endpoints",
+        )
+
+    def record(self, *, category: BillableCategory, route: str, status_code: int, model_id: Optional[str]) -> None:
+        self._counter.add(1, _billable_attributes(category, route, status_code, model_id))
+
+    def shutdown(self) -> None:
+        """Final flush + exporter-thread stop. Without this, up to one export
+        interval of billable counts is dropped on every proxy restart."""
+        self._provider.shutdown(timeout_millis=SHUTDOWN_FLUSH_TIMEOUT_MS)
+
+
+def _export_interval_ms() -> int:
+    raw = os.getenv(EXPORT_INTERVAL_ENV)
+    if raw is None:
+        return DEFAULT_EXPORT_INTERVAL_MS
+    try:
+        return int(raw)
+    except ValueError:
+        verbose_proxy_logger.warning(
+            "Invalid %s=%r, falling back to %d ms", EXPORT_INTERVAL_ENV, raw, DEFAULT_EXPORT_INTERVAL_MS
+        )
+        return DEFAULT_EXPORT_INTERVAL_MS
+
+
+@dataclass(frozen=True, slots=True)
+class _CredentialPaths:
+    client_cert_path: str
+    client_key_path: str
+    ca_cert_path: Optional[str]
+
+
+def _is_pem_content(value: str) -> bool:
+    return value.lstrip().startswith(_PEM_PREFIX)
+
+
+def _write_pem(directory: str, filename: str, pem: str) -> str:
+    path = os.path.join(directory, filename)
+    with open(path, "w", encoding="utf-8") as handle:
+        handle.write(pem if pem.endswith("\n") else f"{pem}\n")
+    os.chmod(path, _PEM_FILE_MODE)
+    return path
+
+
+def _resolve_credential_paths(*, client_cert: str, client_key: str, ca_cert: Optional[str]) -> _CredentialPaths:
+    """
+    Accept either a filesystem path or inline PEM content for each credential.
+
+    Secret stores that inject values as environment content rather than mounted
+    files (ECS tasks reading AWS Secrets Manager, Cloud Run reading Secret
+    Manager) can only deliver the certificate as a string. The OTLP exporter
+    takes paths, so inline PEM is written to a private directory once, when the
+    recorder is built. Raises OSError if that write fails; the caller disables
+    metering rather than propagating.
+    """
+    inline = tuple(value for value in (client_cert, client_key, ca_cert) if value and _is_pem_content(value))
+    if not inline:
+        return _CredentialPaths(client_cert, client_key, ca_cert)
+
+    # mkdtemp is 0o700, so the 0o600 key file it holds is unreachable by other users.
+    directory = tempfile.mkdtemp(prefix=_PEM_DIR_PREFIX)
+    return _CredentialPaths(
+        client_cert_path=(
+            _write_pem(directory, _CLIENT_CERT_FILENAME, client_cert) if _is_pem_content(client_cert) else client_cert
+        ),
+        client_key_path=(
+            _write_pem(directory, _CLIENT_KEY_FILENAME, client_key) if _is_pem_content(client_key) else client_key
+        ),
+        ca_cert_path=(
+            _write_pem(directory, _CA_CERT_FILENAME, ca_cert) if ca_cert and _is_pem_content(ca_cert) else ca_cert
+        ),
+    )
+
+
+def load_billing_metrics_config(
+    *, license_data: Optional["EnterpriseLicenseData"], litellm_version: str
+) -> Optional[BillingMetricsConfig]:
+    endpoint = os.getenv(ENDPOINT_ENV)
+    client_cert = os.getenv(CLIENT_CERT_ENV)
+    client_key = os.getenv(CLIENT_KEY_ENV)
+    # Optional: only for private/test collectors whose server cert is not on the
+    # public web PKI. The production collector needs no CA override.
+    ca_cert = os.getenv(CA_CERT_ENV)
+
+    missing = [
+        name
+        for name, value in (
+            (ENDPOINT_ENV, endpoint),
+            (CLIENT_CERT_ENV, client_cert),
+            (CLIENT_KEY_ENV, client_key),
+        )
+        if not value
+    ]
+    if not endpoint or not client_cert or not client_key:
+        verbose_proxy_logger.warning(
+            "Enterprise billing metrics disabled: licensed deployment missing config (%s)",
+            ", ".join(missing),
+        )
+        return None
+
+    try:
+        paths = _resolve_credential_paths(client_cert=client_cert, client_key=client_key, ca_cert=ca_cert)
+    except OSError as exc:
+        verbose_proxy_logger.warning(
+            "Enterprise billing metrics disabled: could not write inline certificate content to disk: %s", exc
+        )
+        return None
+
+    # Report the variable names, never their values. A value that is neither a
+    # readable path nor recognizable PEM is still secret material, and this
+    # warning would otherwise copy a client key straight into the proxy logs.
+    unreadable = [
+        env_name
+        for env_name, path in (
+            (CLIENT_CERT_ENV, paths.client_cert_path),
+            (CLIENT_KEY_ENV, paths.client_key_path),
+            (CA_CERT_ENV, paths.ca_cert_path),
+        )
+        if path and not os.path.isfile(path)
+    ]
+    if unreadable:
+        verbose_proxy_logger.warning(
+            "Enterprise billing metrics disabled: %s did not resolve to a readable certificate file. "
+            "Set each to a file path, or to inline PEM content beginning with '%s'.",
+            ", ".join(unreadable),
+            _PEM_PREFIX,
+        )
+        return None
+
+    return BillingMetricsConfig(
+        endpoint=endpoint,
+        client_cert_path=paths.client_cert_path,
+        client_key_path=paths.client_key_path,
+        ca_cert_path=paths.ca_cert_path,
+        export_interval_ms=_export_interval_ms(),
+        litellm_version=litellm_version,
+        license_id=(license_data or {}).get("user_id"),
+    )
+
+
+class _ActiveRecorderRegistry:
+    """One-slot registry linking the factory-built recorder to the shutdown
+    hook; the middleware instance holding the recorder is not reachable from
+    proxy_shutdown_event."""
+
+    def __init__(self) -> None:
+        self._recorder: Optional[BillingMetricsRecorder] = None
+
+    def set(self, recorder: BillingMetricsRecorder) -> None:
+        self._recorder = recorder
+
+    def pop(self) -> Optional[BillingMetricsRecorder]:
+        recorder = self._recorder
+        self._recorder = None
+        return recorder
+
+
+_ACTIVE_RECORDER = _ActiveRecorderRegistry()
+
+
+def build_billing_metrics_recorder(
+    *, premium: bool, license_data: Optional["EnterpriseLicenseData"], litellm_version: str
+) -> Optional[BillingMetricsRecorder]:
+    """Build the recorder, or None when the deployment is not licensed or metering is unconfigured."""
+    if not premium:
+        # Debug, not warning: unlicensed is the common case and a warning here
+        # would be noise on every OSS proxy. Every other disable path warns.
+        verbose_proxy_logger.debug("Enterprise billing metrics disabled: deployment is not licensed")
+        return None
+
+    config = load_billing_metrics_config(license_data=license_data, litellm_version=litellm_version)
+    if config is None:
+        return None
+
+    try:
+        recorder = BillingMetricsRecorder(build_mtls_meter_provider(config))
+    except Exception as exc:  # noqa: BLE001 -- metering must never break proxy startup
+        verbose_proxy_logger.warning("Enterprise billing metrics disabled: failed to initialize exporter: %s", exc)
+        return None
+    _ACTIVE_RECORDER.set(recorder)
+    # The only positive signal that this component meters. Without it, a silent
+    # return above is indistinguishable from a working exporter in the logs, and
+    # a component that carries the cert but no license would look healthy.
+    verbose_proxy_logger.info(
+        "Enterprise billing metrics enabled: exporting to %s every %d ms",
+        config.endpoint,
+        config.export_interval_ms,
+    )
+    return recorder
+
+
+def shutdown_billing_metrics_recorder() -> None:
+    """Flush and stop the active recorder, if any. Idempotent; never raises."""
+    recorder = _ACTIVE_RECORDER.pop()
+    if recorder is None:
+        return
+    try:
+        recorder.shutdown()
+    except Exception as exc:  # noqa: BLE001 -- shutdown must never block or fail proxy exit
+        verbose_proxy_logger.warning("Enterprise billing metrics: final flush failed: %s", exc)

litellm/proxy/guardrails/guardrail_endpoints.py

@@ -2238,7 +2238,10 @@ async def apply_guardrail(
         if litellm_logging_obj is not None:
             _patch_logging_obj_for_guardrail(litellm_logging_obj, request)
 
-        request_data: dict = {"messages": request.messages} if request.messages else {}
+        request_data: dict = {
+            **({"messages": request.messages} if request.messages is not None else {}),
+            **({"metadata": request.metadata} if request.metadata is not None else {}),
+        }
         _input_type = _resolve_guardrail_input_type(active_guardrail, request.input_type)
         guardrailed_inputs = await active_guardrail.apply_guardrail(
             inputs={"texts": [request.text]},

litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py

@@ -26,8 +26,13 @@ from typing import (
     cast,
 )
 
+import copy
+from collections.abc import Mapping
+from datetime import datetime, timezone
+
 import httpx
 from fastapi import HTTPException
+from pydantic import TypeAdapter, ValidationError
 
 import litellm
 from litellm._logging import verbose_proxy_logger
@@ -42,10 +47,13 @@ from litellm.llms.custom_httpx.http_handler import (
 )
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.secret_managers.main import get_secret_str
-from litellm.types.guardrails import GuardrailEventHooks
+from litellm.types.guardrails import BedrockChecksConfigModel, GuardrailEventHooks
 from litellm.types.llms.openai import AllMessageValues, ChatCompletionUserMessage
 from litellm.types.proxy.guardrails.guardrail_hooks.bedrock_guardrails import (
+    BedrockChecksMessage,
+    BedrockChecksViolation,
     BedrockContentItem,
+    BedrockGuardrailChecksResponse,
     BedrockGuardrailOutput,
     BedrockGuardrailQualifier,
     BedrockGuardrailResponse,
@@ -55,6 +63,8 @@ from litellm.types.proxy.guardrails.guardrail_hooks.bedrock_guardrails import (
 from litellm.types.utils import GenericGuardrailAPIInputs
 
 if TYPE_CHECKING:
+    from botocore.awsrequest import AWSPreparedRequest
+
     from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 
 from litellm.types.utils import (
@@ -72,6 +82,22 @@ from litellm.types.utils import (
 
 GUARDRAIL_NAME = "bedrock"
 _BEDROCK_DYNAMIC_BODY_DENYLIST = frozenset({"content", "source"})
+# Resource-less, detect-only InvokeGuardrailChecks API (no guardrail resource required).
+_BEDROCK_INVOKE_GUARDRAIL_CHECKS_PATH = "/guardrail-checks/invoke"
+# InvokeGuardrailChecks accepts at most 10 content blocks per message. A message with
+# more text blocks is split across multiple messages so ALL content is scanned --
+# never truncated (truncation would let a user hide content past the limit).
+_BEDROCK_CHECKS_MAX_CONTENT_BLOCKS = 10
+_BEDROCK_CHECKS_KNOWN_KEYS = frozenset({"contentFilter", "promptAttack", "sensitiveInformation"})
+# Keys in a sensitiveInformation result that pinpoint the PII location. They are
+# stripped before the response is handed to standard logging / telemetry so the
+# detected PII span cannot be reconstructed from logs.
+_BEDROCK_CHECKS_PII_LOCATION_KEYS = (
+    "beginOffset",
+    "endOffset",
+    "messageIndex",
+    "contentIndex",
+)
 
 # Maps an OpenAI message content-block ``type`` to the Bedrock guardrail qualifier
 # it represents, so callers can drive contextual grounding by tagging their content.
@@ -149,6 +175,10 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
         guardrailIdentifier: Optional[str] = None,
         guardrailVersion: Optional[str] = None,
         disable_exception_on_block: Optional[bool] = False,
+        checks: BedrockChecksConfigModel | Mapping[str, object] | None = None,
+        content_filter_threshold: float | None = 0.5,
+        prompt_attack_threshold: float | None = 0.5,
+        pii_confidence_threshold: float | None = 0.5,
         **kwargs,
     ):
         self.async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback)
@@ -157,6 +187,15 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
         self.guardrail_provider = "bedrock"
         self.experimental_use_latest_role_message_only = bool(kwargs.get("experimental_use_latest_role_message_only"))
 
+        # Resource-less, detect-only InvokeGuardrailChecks mode. Present `checks`
+        # routes the guardrail to InvokeGuardrailChecks; absent => ApplyGuardrail.
+        self.checks: dict[str, Any] | None = self._normalize_checks(checks)
+        # Per-check block thresholds; a score >= threshold blocks. None => the
+        # check is detect-only (logged, never blocks).
+        self.content_filter_threshold = content_filter_threshold
+        self.prompt_attack_threshold = prompt_attack_threshold
+        self.pii_confidence_threshold = pii_confidence_threshold
+
         # store kwargs as optional_params
         self.optional_params = kwargs
 
@@ -165,16 +204,35 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
         If True, will not raise an exception when the guardrail is blocked.
         """
 
+        # `checks` (InvokeGuardrailChecks) and `guardrailIdentifier`/`guardrailVersion`
+        # (ApplyGuardrail) are two different APIs; configuring both is ambiguous.
+        if self.checks is not None and (self.guardrailIdentifier is not None or self.guardrailVersion is not None):
+            raise ValueError(
+                "Bedrock guardrail accepts either 'guardrailIdentifier'/'guardrailVersion' (ApplyGuardrail) "
+                "or 'checks' (InvokeGuardrailChecks), not both."
+            )
+
         # Set supported event hooks to include MCP hooks
         kwargs.setdefault("supported_event_hooks", list(self.get_supported_event_hooks()))
 
         super().__init__(**kwargs)
         BaseAWSLLM.__init__(self)
 
+        # InvokeGuardrailChecks is detect-only: it never returns rewritten content,
+        # so masking has no effect in checks mode.
+        if self.checks is not None and (
+            getattr(self, "mask_request_content", False) or getattr(self, "mask_response_content", False)
+        ):
+            verbose_proxy_logger.warning(
+                "Bedrock Guardrail: mask_request_content/mask_response_content have no "
+                "effect with 'checks' (InvokeGuardrailChecks is detect-only)."
+            )
+
         verbose_proxy_logger.debug(
-            "Bedrock Guardrail initialized with guardrailIdentifier: %s, guardrailVersion: %s",
+            "Bedrock Guardrail initialized with guardrailIdentifier: %s, guardrailVersion: %s, checks: %s",
             self.guardrailIdentifier,
             self.guardrailVersion,
+            list(self.checks.keys()) if self.checks else None,
         )
 
     @classmethod
@@ -187,6 +245,34 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
             GuardrailEventHooks.during_mcp_call,
         ]
 
+    @staticmethod
+    def _normalize_checks(checks: BedrockChecksConfigModel | Mapping[str, object] | None) -> dict[str, Any] | None:
+        """Normalize the configured `checks` into a plain dict for the API body.
+
+        Accepts a pydantic ``BedrockChecksConfigModel`` or a raw dict; drops None /
+        unknown keys. Returns None when no usable check is configured (=> ApplyGuardrail).
+        """
+        if checks is None:
+            return None
+        raw = checks.model_dump(exclude_none=True) if isinstance(checks, BedrockChecksConfigModel) else dict(checks)
+        unknown_keys = set(raw.keys()) - _BEDROCK_CHECKS_KNOWN_KEYS
+        if unknown_keys:
+            verbose_proxy_logger.warning(
+                "BedrockGuardrail: unrecognized check key(s) %s will be ignored; "
+                "recognized keys will still be used for InvokeGuardrailChecks. "
+                "Known keys: %s.",
+                sorted(unknown_keys),
+                sorted(_BEDROCK_CHECKS_KNOWN_KEYS),
+            )
+        cleaned = {key: value for key, value in raw.items() if key in _BEDROCK_CHECKS_KNOWN_KEYS and value is not None}
+        if not cleaned and raw:
+            raise ValueError(
+                f"BedrockGuardrail: 'checks' block contained only unrecognized or empty keys {sorted(raw.keys())}. "
+                f"Known keys: {sorted(_BEDROCK_CHECKS_KNOWN_KEYS)}. "
+                "Fix the guardrail config or remove the 'checks' block to use ApplyGuardrail mode."
+            )
+        return cleaned or None
+
     def _create_bedrock_input_content_request(self, messages: Optional[List[AllMessageValues]]) -> BedrockRequest:
         """
         Create a bedrock request for the input content - the LLM request.
@@ -574,6 +660,7 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
         aws_region_name: str,
         api_key: Optional[str] = None,
         extra_headers: Optional[dict] = None,
+        request_path: str | None = None,
     ):
         headers = {"Content-Type": "application/json"}
         if extra_headers is not None:
@@ -585,10 +672,12 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
             aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
             aws_region_name=aws_region_name,
         )
-        proxy_endpoint_url = (
-            f"{proxy_endpoint_url}/guardrail/{self.guardrailIdentifier}/version/{self.guardrailVersion}/apply"
-        )
-        # api_base = f"https://bedrock-runtime.{aws_region_name}.amazonaws.com/guardrail/{self.guardrailIdentifier}/version/{self.guardrailVersion}/apply"
+        # Default to the ApplyGuardrail resource path. Callers pass an explicit
+        # request_path for the resource-less InvokeGuardrailChecks endpoint (where
+        # guardrailIdentifier/guardrailVersion are None and must not be interpolated).
+        if request_path is None:
+            request_path = f"/guardrail/{self.guardrailIdentifier}/version/{self.guardrailVersion}/apply"
+        proxy_endpoint_url = f"{proxy_endpoint_url}{request_path}"
         encoded_data = json.dumps(data).encode("utf-8")
 
         # first check api-key, if none, fall back to sigV4
@@ -635,14 +724,43 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
     async def make_bedrock_api_request(
         self,
         source: Literal["INPUT", "OUTPUT"],
-        messages: Optional[List[AllMessageValues]] = None,
-        response: Optional[Union[Any, litellm.ModelResponse]] = None,
-        request_data: Optional[dict] = None,
-        logging_event_type: Optional[GuardrailEventHooks] = None,
+        messages: list[AllMessageValues] | None = None,
+        response: litellm.ModelResponse | None = None,
+        request_data: dict | None = None,
+        logging_event_type: GuardrailEventHooks | None = None,
     ) -> BedrockGuardrailResponse:
-        from datetime import datetime
+        """Dispatch to the configured Bedrock guardrail API.
 
-        start_time = datetime.now()
+        ``checks`` selects the resource-less, detect-only InvokeGuardrailChecks API;
+        otherwise the ApplyGuardrail API is used. Both return a ``BedrockGuardrailResponse``
+        (the checks path returns an empty one on a pass, which downstream masking treats
+        as a no-op) and raise on a blocked request.
+        """
+        if self.checks is not None:
+            return await self._make_invoke_guardrail_checks_request(
+                source=source,
+                messages=messages,
+                response=response,
+                request_data=request_data,
+                logging_event_type=logging_event_type,
+            )
+        return await self._make_apply_guardrail_request(
+            source=source,
+            messages=messages,
+            response=response,
+            request_data=request_data,
+            logging_event_type=logging_event_type,
+        )
+
+    async def _make_apply_guardrail_request(
+        self,
+        source: Literal["INPUT", "OUTPUT"],
+        messages: list[AllMessageValues] | None = None,
+        response: litellm.ModelResponse | None = None,
+        request_data: dict | None = None,
+        logging_event_type: GuardrailEventHooks | None = None,
+    ) -> BedrockGuardrailResponse:
+        start_time = datetime.now(timezone.utc)
         credentials, aws_region_name = self._load_credentials()
         bedrock_request_data: dict = dict(
             self.convert_to_bedrock_format(source=source, messages=messages, response=response)
@@ -683,51 +801,12 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
         else:
             event_type = GuardrailEventHooks.pre_call if source == "INPUT" else GuardrailEventHooks.post_call
 
-        try:
-            httpx_response = await self.async_handler.post(
-                url=prepared_request.url,
-                data=prepared_request.body,  # type: ignore
-                headers=prepared_request.headers,  # type: ignore
-            )
-        except HTTPException:
-            # Propagate HTTPException (e.g. from non-200 path) as-is
-            raise
-        except Exception as e:
-            # If this is an HTTP error with a response body (e.g. httpx.HTTPStatusError),
-            # extract the AWS error message and propagate it
-            response = getattr(e, "response", None)
-            if isinstance(response, httpx.Response):
-                try:
-                    (
-                        status_code,
-                        detail_message,
-                    ) = self._parse_bedrock_guardrail_error_response(response)
-                    self.add_standard_logging_guardrail_information_to_request_data(
-                        guardrail_provider=self.guardrail_provider,
-                        guardrail_json_response={"error": detail_message},
-                        request_data=request_data or {},
-                        guardrail_status="guardrail_failed_to_respond",
-                        start_time=start_time.timestamp(),
-                        end_time=datetime.now().timestamp(),
-                        duration=(datetime.now() - start_time).total_seconds(),
-                        event_type=event_type,
-                    )
-                    raise HTTPException(status_code=status_code, detail=detail_message) from e
-                except HTTPException:
-                    raise
-            # Endpoint down, timeout, or other HTTP/network errors
-            verbose_proxy_logger.error("Bedrock AI: failed to make guardrail request: %s", str(e))
-            self.add_standard_logging_guardrail_information_to_request_data(
-                guardrail_provider=self.guardrail_provider,
-                guardrail_json_response={"error": str(e)},
-                request_data=request_data or {},
-                guardrail_status="guardrail_failed_to_respond",
-                start_time=start_time.timestamp(),
-                end_time=datetime.now().timestamp(),
-                duration=(datetime.now() - start_time).total_seconds(),
-                event_type=event_type,
-            )
-            raise
+        httpx_response = await self._sign_and_post(
+            prepared_request=prepared_request,
+            request_data=request_data,
+            event_type=event_type,
+            start_time=start_time,
+        )
 
         #########################################################
         # Add guardrail information to request trace
@@ -743,8 +822,8 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
             request_data=request_data or {},
             guardrail_status=self._get_bedrock_guardrail_response_status(response=httpx_response),
             start_time=start_time.timestamp(),
-            end_time=datetime.now().timestamp(),
-            duration=(datetime.now() - start_time).total_seconds(),
+            end_time=datetime.now(timezone.utc).timestamp(),
+            duration=(datetime.now(timezone.utc) - start_time).total_seconds(),
             event_type=event_type,
             tracing_detail=tracing_detail or None,
         )
@@ -771,6 +850,338 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
 
         return bedrock_guardrail_response
 
+    async def _sign_and_post(
+        self,
+        prepared_request: "AWSPreparedRequest",
+        request_data: dict | None,
+        event_type: GuardrailEventHooks,
+        start_time: "datetime",
+    ) -> httpx.Response:
+        """POST a signed Bedrock request, logging+raising on network/HTTP errors.
+
+        Shared by both the ApplyGuardrail and InvokeGuardrailChecks paths so their
+        transport-error handling cannot drift. Returns the raw ``httpx.Response`` on
+        success (including non-2xx that httpx did not raise on); the 200-path logging,
+        status and tracing stay with each caller because the two APIs report differently.
+        """
+        try:
+            return await self.async_handler.post(
+                url=prepared_request.url,
+                data=prepared_request.body,
+                headers=prepared_request.headers,
+            )
+        except HTTPException:
+            # Propagate HTTPException (e.g. from non-200 path) as-is
+            raise
+        except Exception as e:
+            # If this is an HTTP error with a response body (e.g. httpx.HTTPStatusError),
+            # extract the AWS error message and propagate it
+            err_response = getattr(e, "response", None)
+            if isinstance(err_response, httpx.Response):
+                try:
+                    (
+                        status_code,
+                        detail_message,
+                    ) = self._parse_bedrock_guardrail_error_response(err_response)
+                    self.add_standard_logging_guardrail_information_to_request_data(
+                        guardrail_provider=self.guardrail_provider,
+                        guardrail_json_response={"error": detail_message},
+                        request_data=request_data or {},
+                        guardrail_status="guardrail_failed_to_respond",
+                        start_time=start_time.timestamp(),
+                        end_time=datetime.now(timezone.utc).timestamp(),
+                        duration=(datetime.now(timezone.utc) - start_time).total_seconds(),
+                        event_type=event_type,
+                    )
+                    raise HTTPException(status_code=status_code, detail=detail_message) from e
+                except HTTPException:
+                    raise
+            # Endpoint down, timeout, or other HTTP/network errors
+            verbose_proxy_logger.error("Bedrock AI: failed to make guardrail request: %s", str(e))
+            self.add_standard_logging_guardrail_information_to_request_data(
+                guardrail_provider=self.guardrail_provider,
+                guardrail_json_response={"error": str(e)},
+                request_data=request_data or {},
+                guardrail_status="guardrail_failed_to_respond",
+                start_time=start_time.timestamp(),
+                end_time=datetime.now(timezone.utc).timestamp(),
+                duration=(datetime.now(timezone.utc) - start_time).total_seconds(),
+                event_type=event_type,
+            )
+            raise
+
+    ###########  InvokeGuardrailChecks (resource-less, detect-only) ############
+
+    @staticmethod
+    def _chunk_texts_into_checks_messages(
+        role: Literal["user", "assistant", "system"], texts: list[str]
+    ) -> list[BedrockChecksMessage]:
+        """Group ``texts`` into role-tagged messages of <= the API content-block cap.
+
+        A source message with more text blocks than the per-message limit is split
+        across multiple messages so EVERY block is scanned. Truncating instead would
+        let a user hide prohibited content past the limit (guardrail bypass).
+        """
+        cap = _BEDROCK_CHECKS_MAX_CONTENT_BLOCKS
+        return [
+            BedrockChecksMessage(
+                role=role,
+                content=[{"text": text} for text in texts[start : start + cap]],
+            )
+            for start in range(0, len(texts), cap)
+        ]
+
+    def _build_invoke_guardrail_checks_messages(
+        self,
+        source: Literal["INPUT", "OUTPUT"],
+        messages: list[AllMessageValues] | None = None,
+        response: litellm.ModelResponse | None = None,
+    ) -> list[BedrockChecksMessage]:
+        """Build the role-tagged `messages` array for InvokeGuardrailChecks.
+
+        INPUT scans the request messages, OUTPUT scans the model response as an
+        ``assistant`` turn. Every non-empty text block of every message is scanned;
+        messages exceeding the per-message content-block cap are split into multiple
+        messages rather than truncated.
+
+        INPUT content is tagged ``user`` regardless of the caller-supplied role.
+        Bedrock excludes ``system`` content from prompt-attack evaluation, so
+        trusting a caller's ``system``/``developer`` label would let an injection
+        avoid the promptAttack check. At the proxy every INPUT message is
+        caller-controlled, so all of it is treated as untrusted user input, matching
+        AWS guidance to tag untrusted content as user input.
+        """
+        if source == "OUTPUT":
+            # Reuse the ApplyGuardrail output extractor (single source of truth for
+            # pulling assistant text out of a ModelResponse), then re-tag as an
+            # assistant turn for the role-based InvokeGuardrailChecks payload.
+            output_request = self._create_bedrock_output_content_request(response=response)
+            output_texts = [
+                text for item in output_request.get("content") or [] if (text := (item.get("text") or {}).get("text"))
+            ]
+            return self._chunk_texts_into_checks_messages("assistant", output_texts)
+
+        return [
+            checks_message
+            for message in messages or []
+            for checks_message in self._chunk_texts_into_checks_messages(
+                "user",
+                [block.text for block in self.get_content_items_for_message(message) or [] if block.text],
+            )
+        ]
+
+    async def _make_invoke_guardrail_checks_request(
+        self,
+        source: Literal["INPUT", "OUTPUT"],
+        messages: list[AllMessageValues] | None = None,
+        response: litellm.ModelResponse | None = None,
+        request_data: dict | None = None,
+        logging_event_type: GuardrailEventHooks | None = None,
+    ) -> BedrockGuardrailResponse:
+        """Run the resource-less InvokeGuardrailChecks API and enforce thresholds.
+
+        Detect-only: the API returns scores, never rewritten content. We map scores
+        to a block decision via the configured thresholds. On a pass we return an
+        empty ``BedrockGuardrailResponse`` (downstream masking treats it as a no-op).
+        """
+        start_time = datetime.now(timezone.utc)
+
+        checks_messages = self._build_invoke_guardrail_checks_messages(
+            source=source, messages=messages, response=response
+        )
+        if not checks_messages:
+            # Nothing to scan (e.g. tool-only turn) -> allow, like ApplyGuardrail does.
+            return BedrockGuardrailResponse()
+
+        credentials, aws_region_name = self._load_credentials()
+        body: dict[str, Any] = {"messages": checks_messages, "checks": self.checks}
+        api_key: str | None = request_data.get("api_key") if request_data else None
+
+        prepared_request = self._prepare_request(
+            credentials=credentials,
+            data=body,
+            optional_params=self.optional_params,
+            aws_region_name=aws_region_name,
+            api_key=api_key,
+            request_path=_BEDROCK_INVOKE_GUARDRAIL_CHECKS_PATH,
+        )
+        verbose_proxy_logger.debug("Bedrock InvokeGuardrailChecks request url: %s", prepared_request.url)
+
+        event_type = logging_event_type or (
+            GuardrailEventHooks.pre_call if source == "INPUT" else GuardrailEventHooks.post_call
+        )
+
+        httpx_response = await self._sign_and_post(
+            prepared_request=prepared_request,
+            request_data=request_data,
+            event_type=event_type,
+            start_time=start_time,
+        )
+
+        if httpx_response.status_code != 200:
+            status_code, detail_message = self._parse_bedrock_guardrail_error_response(httpx_response)
+            verbose_proxy_logger.error(
+                "Bedrock InvokeGuardrailChecks: error response. Status %s: %s",
+                httpx_response.status_code,
+                detail_message,
+            )
+            self.add_standard_logging_guardrail_information_to_request_data(
+                guardrail_provider=self.guardrail_provider,
+                guardrail_json_response={"error": detail_message},
+                request_data=request_data or {},
+                guardrail_status="guardrail_failed_to_respond",
+                start_time=start_time.timestamp(),
+                end_time=datetime.now(timezone.utc).timestamp(),
+                duration=(datetime.now(timezone.utc) - start_time).total_seconds(),
+                event_type=event_type,
+            )
+            raise HTTPException(status_code=status_code, detail=detail_message)
+
+        try:
+            json_response = TypeAdapter(BedrockGuardrailChecksResponse).validate_python(httpx_response.json())
+        except (ValidationError, ValueError) as e:
+            verbose_proxy_logger.error("Bedrock InvokeGuardrailChecks: unparseable 200 response: %s", str(e))
+            self.add_standard_logging_guardrail_information_to_request_data(
+                guardrail_provider=self.guardrail_provider,
+                guardrail_json_response={"error": str(e)},
+                request_data=request_data or {},
+                guardrail_status="guardrail_failed_to_respond",
+                start_time=start_time.timestamp(),
+                end_time=datetime.now(timezone.utc).timestamp(),
+                duration=(datetime.now(timezone.utc) - start_time).total_seconds(),
+                event_type=event_type,
+            )
+            raise HTTPException(
+                status_code=500,
+                detail={"error": "Bedrock InvokeGuardrailChecks returned an unexpected response shape"},
+            ) from e
+        violations = self._collect_invoke_checks_violations(json_response)
+
+        # Log a copy with PII location offsets stripped: offsets + the (separately
+        # logged) request messages would otherwise reconstruct the detected PII span.
+        self.add_standard_logging_guardrail_information_to_request_data(
+            guardrail_provider=self.guardrail_provider,
+            guardrail_json_response=self._sanitize_invoke_checks_response_for_logging(json_response),
+            request_data=request_data or {},
+            guardrail_status=self._get_invoke_checks_status(bool(violations)),
+            start_time=start_time.timestamp(),
+            end_time=datetime.now(timezone.utc).timestamp(),
+            duration=(datetime.now(timezone.utc) - start_time).total_seconds(),
+            event_type=event_type,
+            tracing_detail=self._build_invoke_checks_tracing_detail(violations) if violations else None,
+        )
+
+        if violations:
+            raise self._get_block_exception_for_checks(violations, request_data=request_data)
+
+        return BedrockGuardrailResponse()
+
+    def _collect_invoke_checks_violations(
+        self, response: BedrockGuardrailChecksResponse | None
+    ) -> list[BedrockChecksViolation]:
+        """Return the check results whose score meets/exceeds the configured threshold.
+
+        Only checks present in the configured ``checks`` block are evaluated; a
+        threshold of ``None`` makes that check detect-only (never contributes a
+        violation). A truncated sensitiveInformation result counts as a violation
+        (fail closed: omitted detections were never scored). Only the non-sensitive
+        label (category/type) and the numeric score are kept -- never offsets or
+        matched text.
+        """
+        results: dict[str, Any] = dict((response or {}).get("results") or {})
+        # (results key, score field, label field, threshold). PII uses
+        # confidenceScore/type; the other two use severityScore/category.
+        check_specs = [
+            (
+                "contentFilter",
+                "severityScore",
+                "category",
+                self.content_filter_threshold,
+            ),
+            ("promptAttack", "severityScore", "category", self.prompt_attack_threshold),
+            (
+                "sensitiveInformation",
+                "confidenceScore",
+                "type",
+                self.pii_confidence_threshold,
+            ),
+        ]
+
+        configured_checks = self.checks or {}
+        violations: list[BedrockChecksViolation] = []
+        for check_key, score_field, label_field, threshold in check_specs:
+            if threshold is None or check_key not in configured_checks:
+                continue
+            check_result = results.get(check_key) or {}
+            if check_key == "sensitiveInformation" and check_result.get("truncated"):
+                violations.append({"check": check_key, "truncated": True})
+            for entry in check_result.get("results") or []:
+                score = entry.get(score_field)
+                if isinstance(score, (int, float)) and float(score) >= threshold:
+                    violation: BedrockChecksViolation = (
+                        {"check": check_key, "category": entry.get("category"), "severityScore": float(score)}
+                        if score_field == "severityScore"
+                        else {"check": check_key, "type": entry.get("type"), "confidenceScore": float(score)}
+                    )
+                    violations.append(violation)
+        return violations
+
+    @staticmethod
+    def _sanitize_invoke_checks_response_for_logging(
+        response: BedrockGuardrailChecksResponse,
+    ) -> dict[str, Any]:
+        """Strip PII location offsets from a checks response before it is logged."""
+        sanitized: dict[str, Any] = copy.deepcopy(dict(response))
+        sensitive = (sanitized.get("results") or {}).get("sensitiveInformation") or {}
+        for entry in sensitive.get("results") or []:
+            if isinstance(entry, dict):
+                for key in _BEDROCK_CHECKS_PII_LOCATION_KEYS:
+                    entry.pop(key, None)
+        return sanitized
+
+    @staticmethod
+    def _get_invoke_checks_status(over_threshold: bool) -> GuardrailStatus:
+        return "guardrail_intervened" if over_threshold else "success"
+
+    @staticmethod
+    def _build_invoke_checks_tracing_detail(
+        violations: list[BedrockChecksViolation],
+    ) -> GuardrailTracingDetail:
+        tracing_detail: GuardrailTracingDetail = {}
+        categories = [
+            label
+            for label in (v.get("category") or v.get("type") for v in violations)
+            if isinstance(label, str) and label
+        ]
+        if categories:
+            tracing_detail["violation_categories"] = categories
+        tracing_detail["guardrail_action"] = "GUARDRAIL_INTERVENED" if violations else "NONE"
+        return tracing_detail
+
+    def _get_block_exception_for_checks(
+        self, violations: list[BedrockChecksViolation], request_data: dict | None = None
+    ) -> Union[HTTPException, ModifyResponseException]:
+        """Build the block exception for an over-threshold InvokeGuardrailChecks result.
+
+        Mirrors ``_get_http_exception_for_blocked_guardrail``'s return-type branching.
+        The detail carries only non-sensitive labels + scores (no offsets / raw input).
+        """
+        if self.disable_exception_on_block is True:
+            _request_data = request_data or {}
+            return ModifyResponseException(
+                message="Violated guardrail policy",
+                model=_request_data.get("model", "bedrock-guardrail"),
+                request_data=_request_data,
+                guardrail_name=self.guardrail_name,
+            )
+        return HTTPException(
+            status_code=400,
+            detail={
+                "error": "Violated guardrail policy",
+                "bedrock_guardrail_checks": violations,
+            },
+        )
+
     def _check_bedrock_response_for_exception(self, response) -> bool:
         """
         Return True if the Bedrock ApplyGuardrail response indicates an exception.

litellm/proxy/guardrails/guardrail_hooks/compresr/__init__.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from litellm.types.guardrails import (
+    GuardrailEventHooks,
+    Mode,
+    SupportedGuardrailIntegrations,
+)
+
+from .compresr import CompresrGuardrail
+
+if TYPE_CHECKING:
+    from litellm.types.guardrails import Guardrail, LitellmParams
+
+
+def _coerce_event_hook(
+    mode: str | list[str] | Mode,
+) -> GuardrailEventHooks | list[GuardrailEventHooks] | Mode:
+    if isinstance(mode, Mode):
+        return mode
+    if isinstance(mode, list):
+        return [GuardrailEventHooks(item) for item in mode]
+    return GuardrailEventHooks(mode)
+
+
+def _get_optional_value(litellm_params: LitellmParams, optional_params: object | None, attribute_name: str) -> object:
+    if optional_params is not None:
+        value = getattr(optional_params, attribute_name, None)
+        if value is not None:
+            return value
+    return getattr(litellm_params, attribute_name, None)
+
+
+def initialize_guardrail(litellm_params: LitellmParams, guardrail: Guardrail) -> CompresrGuardrail:
+    import litellm
+
+    optional_params = getattr(litellm_params, "optional_params", None)
+
+    _callback = CompresrGuardrail(
+        api_base=litellm_params.api_base,
+        api_key=litellm_params.api_key,
+        model=litellm_params.model,
+        target_compression_ratio=_get_optional_value(litellm_params, optional_params, "target_compression_ratio"),
+        coarse=_get_optional_value(litellm_params, optional_params, "coarse"),
+        min_chars_to_compress=_get_optional_value(litellm_params, optional_params, "min_chars_to_compress"),
+        compress_tool_outputs=_get_optional_value(litellm_params, optional_params, "compress_tool_outputs"),
+        compress_system=_get_optional_value(litellm_params, optional_params, "compress_system"),
+        compress_history=_get_optional_value(litellm_params, optional_params, "compress_history"),
+        compress_last_user=_get_optional_value(litellm_params, optional_params, "compress_last_user"),
+        enable_retrieval=_get_optional_value(litellm_params, optional_params, "enable_retrieval"),
+        max_bytes_per_call=_get_optional_value(litellm_params, optional_params, "max_bytes_per_call"),
+        allow_bypass_header=_get_optional_value(litellm_params, optional_params, "allow_bypass_header"),
+        dynamic=_get_optional_value(litellm_params, optional_params, "dynamic"),
+        dynamic_min_ratio=_get_optional_value(litellm_params, optional_params, "dynamic_min_ratio"),
+        dynamic_max_ratio=_get_optional_value(litellm_params, optional_params, "dynamic_max_ratio"),
+        compression_params=_get_optional_value(litellm_params, optional_params, "compression_params"),
+        guardrail_name=guardrail["guardrail_name"],
+        event_hook=_coerce_event_hook(litellm_params.mode),
+        default_on=litellm_params.default_on or False,
+        unreachable_fallback=litellm_params.unreachable_fallback,
+    )
+    litellm.logging_callback_manager.add_litellm_callback(  # pyright: ignore[reportUnknownMemberType]  # callback manager is untyped
+        _callback
+    )
+    return _callback
+
+
+guardrail_initializer_registry = {
+    SupportedGuardrailIntegrations.COMPRESR.value: initialize_guardrail,
+}
+
+guardrail_class_registry = {
+    SupportedGuardrailIntegrations.COMPRESR.value: CompresrGuardrail,
+}

litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/__init__.py

@@ -38,6 +38,7 @@ def initialize_guardrail(litellm_params: "LitellmParams", guardrail: "Guardrail"
         default_on=litellm_params.default_on,
         streaming_end_of_stream_only=_get_config_value(litellm_params, optional_params, "streaming_end_of_stream_only"),
         streaming_sampling_rate=_get_config_value(litellm_params, optional_params, "streaming_sampling_rate"),
+        streaming_transform_mode=_get_config_value(litellm_params, optional_params, "streaming_transform_mode"),
     )
 
     litellm.logging_callback_manager.add_litellm_callback(_generic_guardrail_api_callback)

litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/generic_guardrail_api.py

@@ -58,7 +58,7 @@ _HEADER_PRESENT_PLACEHOLDER = "[present]"
 
 def _header_value_allowed(
     header_name: str,
-    extra_allowlist: Optional[Set[str]] = None,
+    extra_allowlist: Set[str] | None = None,
 ) -> bool:
     """Return True if this header's value may be forwarded (allowlist, including globs and extra_headers)."""
     lower = header_name.lower()
@@ -74,8 +74,8 @@ def _header_value_allowed(
 
 def _sanitize_inbound_headers(
     headers: Any,
-    extra_allowlist: Optional[Set[str]] = None,
-) -> Optional[Dict[str, str]]:
+    extra_allowlist: Set[str] | None = None,
+) -> Dict[str, str] | None:
     """
     Sanitize inbound headers before passing them to a 3rd party guardrail service.
 
@@ -105,8 +105,8 @@ def _sanitize_inbound_headers(
 def _extract_inbound_headers(
     request_data: dict,
     logging_obj: Optional["LiteLLMLoggingObj"],
-    extra_allowlist: Optional[Set[str]] = None,
-) -> Optional[Dict[str, str]]:
+    extra_allowlist: Set[str] | None = None,
+) -> Dict[str, str] | None:
     """
     Extract inbound headers from available request context.
 
@@ -172,15 +172,16 @@ class GenericGuardrailAPI(CustomGuardrail):
 
     def __init__(
         self,
-        headers: Optional[Dict[str, Any]] = None,
-        api_base: Optional[str] = None,
-        api_key: Optional[str] = None,
-        additional_provider_specific_params: Optional[Dict[str, Any]] = None,
+        headers: Dict[str, Any] | None = None,
+        api_base: str | None = None,
+        api_key: str | None = None,
+        additional_provider_specific_params: Dict[str, Any] | None = None,
         unreachable_fallback: Literal["fail_closed", "fail_open"] = "fail_closed",
-        fail_on_error: Optional[bool] = True,
-        extra_headers: Optional[list] = None,
-        streaming_end_of_stream_only: Optional[bool] = None,
-        streaming_sampling_rate: Optional[int] = None,
+        fail_on_error: bool | None = True,
+        extra_headers: list | None = None,
+        streaming_end_of_stream_only: bool | None = None,
+        streaming_sampling_rate: int | None = None,
+        streaming_transform_mode: Literal["block_only", "incremental_diff"] | None = None,
         **kwargs,
     ):
         self.async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback)
@@ -221,6 +222,13 @@ class GenericGuardrailAPI(CustomGuardrail):
             raise ValueError(f"streaming_sampling_rate must be >= 1 (got {streaming_sampling_rate})")
         self.streaming_sampling_rate: int = 5 if streaming_sampling_rate is None else streaming_sampling_rate
 
+        # Read by UnifiedLLMGuardrails.async_post_call_streaming_iterator_hook.
+        # "block_only" (default) drops text rewrites on the streaming path;
+        # "incremental_diff" emits them as synthetic deltas.
+        self.streaming_transform_mode: Literal["block_only", "incremental_diff"] = (
+            "block_only" if streaming_transform_mode is None else streaming_transform_mode
+        )
+
         # Set supported event hooks
         kwargs.setdefault("supported_event_hooks", list(self.get_supported_event_hooks()))
 
@@ -280,7 +288,7 @@ class GenericGuardrailAPI(CustomGuardrail):
         input_type: Literal["request", "response"],
         logging_obj: Optional["LiteLLMLoggingObj"],
         error: Exception,
-        http_status_code: Optional[int] = None,
+        http_status_code: int | None = None,
     ) -> GenericGuardrailAPIInputs:
         status_suffix = f" http_status_code={http_status_code}" if http_status_code else ""
         verbose_proxy_logger.critical(
@@ -326,6 +334,8 @@ class GenericGuardrailAPI(CustomGuardrail):
             return_inputs["tools"] = guardrail_response.tools
         elif tools:
             return_inputs["tools"] = tools
+        if guardrail_response.stream_holdback_chars is not None:
+            return_inputs["stream_holdback_chars"] = guardrail_response.stream_holdback_chars
         return return_inputs
 
     def _handle_guardrail_request_error(
@@ -479,7 +489,7 @@ class GenericGuardrailAPI(CustomGuardrail):
             return self._handle_guardrail_request_error(e, inputs, input_type, logging_obj, is_unreachable=False)
 
     @staticmethod
-    def get_config_model() -> Optional[type["GuardrailConfigModel"]]:
+    def get_config_model() -> type["GuardrailConfigModel"] | None:
         from litellm.types.proxy.guardrails.guardrail_hooks.generic_guardrail_api import (
             GenericGuardrailAPIConfigModel,
         )

litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py

@@ -8,7 +8,7 @@ Unified Guardrail, leveraging LiteLLM's /applyGuardrail endpoint
 
 import copy
 import json
-from typing import TYPE_CHECKING, Any, AsyncGenerator, List, Optional, Union
+from typing import TYPE_CHECKING, Any, AsyncGenerator, List, Union
 
 from fastapi import HTTPException
 
@@ -21,7 +21,13 @@ from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_fo
 from litellm.llms import load_guardrail_translation_mappings
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.types.guardrails import GuardrailEventHooks
-from litellm.types.utils import CallTypes, CallTypesLiteral
+from litellm.types.utils import (
+    CallTypes,
+    CallTypesLiteral,
+    Delta,
+    ModelResponseStream,
+    StreamingChoices,
+)
 
 if TYPE_CHECKING:
     # Imported lazily at runtime (inside the streaming hook) to avoid a
@@ -34,7 +40,12 @@ A2A_CALL_TYPES = (CallTypes.asend_message, CallTypes.send_message)
 GUARDRAIL_NAME = "unified_llm_guardrails"
 
 
-def _get_a2a_request_id(responses_so_far: List[Any], request_data: dict) -> Optional[str]:
+class _StreamTerminated(Exception):
+    """Internal signal that the incremental transform stream has already emitted
+    its terminal chunks (block message or in-stream error) and must stop."""
+
+
+def _get_a2a_request_id(responses_so_far: List[Any], request_data: dict) -> str | None:
     """Get JSON-RPC request id from first A2A chunk or request body for in-stream error reporting."""
     for item in responses_so_far:
         if isinstance(item, dict) and "id" in item:
@@ -216,7 +227,7 @@ class UnifiedLLMGuardrails(CustomLogger):
 
         verbose_proxy_logger.debug("async_post_call_success_hook response: %s", response)
 
-        call_type: Optional[CallTypesLiteral] = None
+        call_type: CallTypesLiteral | None = None
         if user_api_key_dict.request_route is not None:
             call_types = get_call_types_for_route(user_api_key_dict.request_route)
             if call_types is not None and len(call_types) > 0:  # type: ignore
@@ -292,6 +303,498 @@ class UnifiedLLMGuardrails(CustomLogger):
         for chunk in block_chunks:
             yield chunk
 
+    @staticmethod
+    def _resolve_transform_call_type(
+        user_api_key_dict: UserAPIKeyAuth,
+        mappings: dict,
+    ) -> str | None:
+        """Resolve the call type for the incremental_diff path, or None if the
+        route is unresolvable / unsupported.
+
+        Incremental transformation needs a route we can resolve before the first
+        chunk and a handler that supports the streaming text-diff protocol (v1:
+        the OpenAI chat completions handler only). Returning None makes the caller
+        fall back to block_only.
+        """
+        from litellm.llms.openai.chat.guardrail_translation.handler import (
+            OpenAIChatCompletionsHandler,
+        )
+
+        if user_api_key_dict.request_route is None:
+            return None
+        call_types = get_call_types_for_route(user_api_key_dict.request_route)
+        if not call_types:
+            return None
+        call_type = call_types[0].value
+        try:
+            mapped = CallTypes(call_type)
+        except ValueError:
+            return None
+        handler_cls = mappings.get(mapped)
+        if handler_cls is None or not issubclass(handler_cls, OpenAIChatCompletionsHandler):
+            return None
+        return call_type
+
+    async def _emit_streaming_http_error(
+        self,
+        exc: HTTPException,
+        call_type: str | None,
+        responses_so_far: list[Any],
+        request_data: dict,
+    ) -> AsyncGenerator[Any, None]:
+        """Surface a mid-stream HTTPException. For A2A (NDJSON) call types the
+        response has already started, so emit an in-stream JSON-RPC error chunk;
+        otherwise re-raise so the proxy can report it.
+        """
+        if call_type is not None and CallTypes(call_type) in A2A_CALL_TYPES:
+            request_id = _get_a2a_request_id(responses_so_far, request_data)
+            detail = exc.detail if isinstance(exc.detail, dict) else {"message": str(exc.detail)}
+            error_chunk = (
+                json.dumps(
+                    {
+                        "jsonrpc": "2.0",
+                        "id": request_id,
+                        "error": {
+                            "code": -32603,
+                            "message": detail.get("error", detail.get("message", str(exc.detail))),
+                            "data": {k: v for k, v in detail.items() if k not in ("error", "message")},
+                        },
+                    }
+                )
+                + "\n"
+            )
+            yield error_chunk
+            return
+        raise exc
+
+    def _build_transform_chunk(
+        self,
+        *,
+        reference_chunk: Any,
+        mutated_text_per_choice: dict[int, str],
+        emitted_text_per_choice: dict[int, str],
+        holdback_per_choice: dict[int, int],
+        finish_reason_per_choice: dict[int, str | None],
+        is_final: bool,
+    ) -> ModelResponseStream | None:
+        """Build the synthetic chunk carrying the newly-guardrailed deltas.
+
+        For each choice, the new delta is the mutated accumulated text past what
+        has already been emitted, minus a trailing holdback (forced to 0 on the
+        final flush). ``emitted_text_per_choice`` holds the exact bytes already
+        sent per choice and is extended in place. Returns None when there is no
+        text to emit (e.g. a tool-call-only turn) or nothing new and this is not
+        the final chunk.
+
+        Raises HTTPException(400, stream_transform_underflow) when the guardrail's
+        transform is not a forward extension of what has already been streamed
+        (shorter than, or rewrites, the already-sent prefix), since emitted bytes
+        cannot be retracted. This makes the framework fail closed rather than
+        silently leave un-transformed text on the wire; a guardrail that needs to
+        rewrite recent output must withhold it first via ``stream_holdback_chars``.
+        """
+        if not mutated_text_per_choice:
+            # Fix #4 — on the final flush a deferred finish_reason (from a mixed
+            # content+tool_calls chunk whose passthrough suppressed it) still
+            # needs to reach the client, even if the guardrail returned no text
+            # to emit. Build a terminator chunk carrying finish_reason per choice.
+            if is_final and finish_reason_per_choice:
+                terminator_choices: list[StreamingChoices] = []
+                for choice_idx, finish_reason in finish_reason_per_choice.items():
+                    if finish_reason is None:
+                        continue
+                    terminator_choices.append(
+                        StreamingChoices(
+                            index=choice_idx,
+                            delta=Delta(content="", role=None, tool_calls=None),
+                            finish_reason=finish_reason,
+                        )
+                    )
+                if terminator_choices:
+                    return ModelResponseStream(
+                        id=getattr(reference_chunk, "id", None),
+                        created=getattr(reference_chunk, "created", None),
+                        model=getattr(reference_chunk, "model", None),
+                        choices=terminator_choices,
+                    )
+            return None
+
+        deltas: dict[int, str] = {}
+        for choice_idx, text in mutated_text_per_choice.items():
+            already = emitted_text_per_choice.get(choice_idx, "")
+            if not text.startswith(already):
+                raise HTTPException(
+                    status_code=400,
+                    detail={
+                        "error": "stream_transform_underflow",
+                        "message": (
+                            f"Guardrail streaming transform for choice {choice_idx} is not a forward "
+                            f"extension of the {len(already)} chars already streamed to the client "
+                            "(it is shorter than, or rewrites, the emitted prefix); emitted bytes "
+                            "cannot be retracted. Withhold recent output via stream_holdback_chars "
+                            "before rewriting it."
+                        ),
+                    },
+                )
+            holdback = 0 if is_final else max(0, holdback_per_choice.get(choice_idx, 0))
+            end = max(len(already), len(text) - holdback)
+            deltas[choice_idx] = text[len(already) : end]
+
+        # Iterate the mutated choices (not just those in reference_chunk) so a
+        # choice with pending text is never dropped for n > 1. finish_reason is
+        # taken per choice from the accumulated map (a choice can finish in an
+        # earlier chunk than the stream's last one); tool_calls are dropped since
+        # v1 does not transform streamed tool calls (they pass through raw).
+        synthetic_choices: list[StreamingChoices] = []
+        for choice_idx in mutated_text_per_choice:
+            delta_text = deltas.get(choice_idx, "")
+            finish_reason = finish_reason_per_choice.get(choice_idx) if is_final else None
+            # Skip a choice with nothing to say: no new content and no
+            # finish_reason to deliver. This avoids emitting an empty delta for an
+            # already-finished choice (e.g. one that terminated via a passed-through
+            # tool-call chunk, which already carried its own finish_reason).
+            if not delta_text and finish_reason is None:
+                continue
+            # role="assistant" on this choice's first emitted delta only.
+            role = "assistant" if not emitted_text_per_choice.get(choice_idx) else None
+            synthetic_choices.append(
+                StreamingChoices(
+                    index=choice_idx,
+                    delta=Delta(content=delta_text, role=role, tool_calls=None),
+                    finish_reason=finish_reason,
+                )
+            )
+
+        if not synthetic_choices:
+            return None
+
+        for choice_idx in mutated_text_per_choice:
+            emitted_text_per_choice[choice_idx] = emitted_text_per_choice.get(choice_idx, "") + deltas.get(
+                choice_idx, ""
+            )
+
+        return ModelResponseStream(
+            id=getattr(reference_chunk, "id", None),
+            created=getattr(reference_chunk, "created", None),
+            model=getattr(reference_chunk, "model", None),
+            choices=synthetic_choices,
+        )
+
+    async def _emit_transform_round(
+        self,
+        *,
+        endpoint_translation: Any,
+        guardrail_to_apply: CustomGuardrail,
+        request_data: dict,
+        user_api_key_dict: UserAPIKeyAuth,
+        call_type: str,
+        reference_chunk: Any,
+        responses_so_far: list[Any],
+        responses_yielded: list[Any],
+        emitted_text_per_choice: dict[int, str],
+        finish_reason_per_choice: dict[int, str | None],
+        is_final: bool,
+    ) -> AsyncGenerator[Any, None]:
+        """Run one guardrail processing round and emit the resulting diff chunk.
+
+        Raises ``_StreamTerminated`` (after emitting the terminal block message or
+        in-stream error) when the guardrail blocks or an underflow occurs.
+        """
+        from litellm.integrations.custom_guardrail import ModifyResponseException
+        from litellm.llms.base_llm.guardrail_translation.base_translation import (
+            StreamTransformSink,
+        )
+
+        sink = StreamTransformSink()
+        try:
+            await endpoint_translation.process_output_streaming_response(
+                responses_so_far=responses_so_far,
+                guardrail_to_apply=guardrail_to_apply,
+                litellm_logging_obj=request_data.get("litellm_logging_obj"),
+                user_api_key_dict=user_api_key_dict,
+                request_data=request_data,
+                stream_transform_sink=sink,
+            )
+            synthetic = self._build_transform_chunk(
+                reference_chunk=reference_chunk,
+                mutated_text_per_choice=sink.mutated_text_per_choice,
+                emitted_text_per_choice=emitted_text_per_choice,
+                holdback_per_choice=sink.holdback_per_choice,
+                finish_reason_per_choice=finish_reason_per_choice,
+                is_final=is_final,
+            )
+        except ModifyResponseException as e:
+            if e.original_response is None:
+                e.original_response = responses_so_far
+            async for block_chunk in self._handle_streaming_block(
+                e,
+                endpoint_translation,
+                stream_started=bool(responses_yielded),
+                responses_so_far=responses_yielded,
+            ):
+                yield block_chunk
+            raise _StreamTerminated()
+        except HTTPException as e:
+            async for error_item in self._emit_streaming_http_error(e, call_type, responses_so_far, request_data):
+                yield error_item
+            raise _StreamTerminated()
+
+        if synthetic is not None:
+            responses_yielded.append(synthetic)
+            yield synthetic
+
+    async def _run_incremental_transform_stream(
+        self,
+        *,
+        guardrail_to_apply: CustomGuardrail,
+        response: Any,
+        request_data: dict,
+        user_api_key_dict: UserAPIKeyAuth,
+        call_type: str,
+        sampling_rate: int,
+        end_of_stream_only: bool,
+        mappings: dict,
+    ) -> AsyncGenerator[Any, None]:
+        """Emit guardrail text transformations as new deltas on the stream.
+
+        Raw chunks are withheld and accumulated; on each sampled processing round
+        (and once at end of stream) the guardrailed accumulated text is diffed
+        against what has already been emitted and the new portion is sent as a
+        synthetic chunk. A BLOCK terminates the stream via the shared block
+        handler; an underflow surfaces as an HTTPException.
+        """
+        endpoint_translation = mappings[CallTypes(call_type)]()
+        responses_so_far: list[Any] = []
+        responses_yielded: list[Any] = []
+        emitted_text_per_choice: dict[int, str] = {}
+        finish_reason_per_choice: dict[int, str | None] = {}
+        chunk_counter = 0
+        last_chunk: Any | None = None
+
+        def _round(reference_chunk: Any, is_final: bool) -> AsyncGenerator[Any, None]:
+            return self._emit_transform_round(
+                endpoint_translation=endpoint_translation,
+                guardrail_to_apply=guardrail_to_apply,
+                request_data=request_data,
+                user_api_key_dict=user_api_key_dict,
+                call_type=call_type,
+                reference_chunk=reference_chunk,
+                responses_so_far=responses_so_far,
+                responses_yielded=responses_yielded,
+                emitted_text_per_choice=emitted_text_per_choice,
+                finish_reason_per_choice=finish_reason_per_choice,
+                is_final=is_final,
+            )
+
+        saw_tool_calls = False
+        saw_text_content = False
+
+        try:
+            async for item in response:
+                # v1 transforms only text. A chunk carrying tool_calls is passed
+                # through raw so function-calling turns are not dropped, but ONLY
+                # its tool-call fields are forwarded: content is stripped so any
+                # response text (in the same delta, or in another choice of an n>1
+                # chunk) can never bypass the transform. The original chunk is kept
+                # in responses_so_far so its text is still accumulated + redacted +
+                # emitted as synthetic deltas, and so the guardrail inspects the
+                # assembled tool calls at end of stream (see the block inspection
+                # below), matching block_only. finish_reason rides on the raw
+                # tool-only chunk, so it is not recorded for the text flush.
+                if self._chunk_has_tool_calls(item):
+                    saw_tool_calls = True
+                    responses_so_far.append(item)
+                    last_chunk = item
+                    # Fix #3 — flush accumulated text BEFORE the tool-call
+                    # passthrough. Without this, a stream of text chunks that
+                    # hasn't yet hit a sampled round can be trailed by a
+                    # tool-call chunk carrying finish_reason="tool_calls"; an
+                    # SSE-compliant client stops reading at that finish_reason
+                    # and drops the end-of-stream text flush that would follow.
+                    if saw_text_content:
+                        async for out in _round(item, is_final=False):
+                            yield out
+                    # Fix #1 — pass finish_reason_per_choice into the
+                    # passthrough so a mixed content+tool_call chunk defers its
+                    # finish_reason to the final text terminator (see the
+                    # _tool_call_passthrough_chunk docstring).
+                    tool_only = self._tool_call_passthrough_chunk(
+                        item, finish_reason_per_choice=finish_reason_per_choice
+                    )
+                    responses_yielded.append(tool_only)
+                    yield tool_only
+                    continue
+
+                chunk_counter += 1
+                responses_so_far.append(item)
+                last_chunk = item
+                self._record_finish_reasons(item, finish_reason_per_choice)
+                if self._chunk_carries_text(item):
+                    saw_text_content = True
+                # Skip the sampled round for a terminal chunk: the end-of-stream
+                # flush below processes it once with holdback forced to 0, so a
+                # sampled round here would guardrail the same content twice.
+                if (
+                    not end_of_stream_only
+                    and not self._chunk_has_finish_reason(item)
+                    and chunk_counter % sampling_rate == 0
+                ):
+                    async for out in _round(item, is_final=False):
+                        yield out
+
+            # v1 does not transform streamed tool calls, but they must still go
+            # through the guardrail's block decision. Run the block_only inspection
+            # over the full assembled response so tool calls cannot bypass it.
+            #
+            # Pass a deep copy of responses_so_far — the block path routes through
+            # ``_process_streaming_block_only`` which mutates ``delta.content``
+            # in-place on the chunk objects it receives. For an n>1 chunk carrying
+            # text on one choice and tool_calls (with finish_reason) on another,
+            # ``has_stream_ended`` reads ``choices[0]`` alone and can miss the
+            # terminal signal, letting the block path rewrite the raw accumulator.
+            # The subsequent final ``_round`` would then re-read the already-mutated
+            # text, producing double-application for a non-idempotent guardrail or a
+            # ``stream_transform_underflow`` 400 from mismatched prefixes. A shallow
+            # list copy wouldn't help — the mutation is on the chunk objects
+            # themselves — so we deepcopy.
+            if saw_tool_calls:
+                async for out in self._inspect_full_response_for_block(
+                    endpoint_translation=endpoint_translation,
+                    guardrail_to_apply=guardrail_to_apply,
+                    request_data=request_data,
+                    user_api_key_dict=user_api_key_dict,
+                    responses_so_far=copy.deepcopy(responses_so_far),
+                    responses_yielded=responses_yielded,
+                ):
+                    yield out
+
+            if last_chunk is not None:
+                async for out in _round(last_chunk, is_final=True):
+                    yield out
+        except _StreamTerminated:
+            return
+
+    async def _inspect_full_response_for_block(
+        self,
+        *,
+        endpoint_translation: Any,
+        guardrail_to_apply: CustomGuardrail,
+        request_data: dict,
+        user_api_key_dict: UserAPIKeyAuth,
+        responses_so_far: list[Any],
+        responses_yielded: list[Any],
+    ) -> AsyncGenerator[Any, None]:
+        """Run the block-only guardrail inspection over the full assembled
+        response (text + tool calls) so nothing bypasses the block decision.
+
+        The guardrail's returned transforms are discarded here (v1 does not
+        transform tool calls); only its block decision matters. A block is
+        surfaced the same way as elsewhere: ModifyResponseException terminates the
+        stream via the shared block handler; a GenericGuardrailAPI block raises and
+        propagates, matching block_only.
+        """
+        from litellm.integrations.custom_guardrail import ModifyResponseException
+
+        try:
+            await endpoint_translation.process_output_streaming_response(
+                responses_so_far=responses_so_far,
+                guardrail_to_apply=guardrail_to_apply,
+                litellm_logging_obj=request_data.get("litellm_logging_obj"),
+                user_api_key_dict=user_api_key_dict,
+                request_data=request_data,
+                stream_transform_sink=None,
+            )
+        except ModifyResponseException as e:
+            if e.original_response is None:
+                e.original_response = responses_so_far
+            async for block_chunk in self._handle_streaming_block(
+                e,
+                endpoint_translation,
+                stream_started=bool(responses_yielded),
+                responses_so_far=responses_yielded,
+            ):
+                yield block_chunk
+            raise _StreamTerminated()
+
+    @staticmethod
+    def _chunk_has_tool_calls(item: Any) -> bool:
+        for choice in getattr(item, "choices", None) or []:
+            delta = getattr(choice, "delta", None)
+            if getattr(delta, "tool_calls", None):
+                return True
+        return False
+
+    @staticmethod
+    def _chunk_carries_text(item: Any) -> bool:
+        """True if any choice in this chunk has non-empty string ``delta.content``."""
+        for choice in getattr(item, "choices", None) or []:
+            delta = getattr(choice, "delta", None)
+            content = getattr(delta, "content", None)
+            if isinstance(content, str) and content != "":
+                return True
+        return False
+
+    @staticmethod
+    def _tool_call_passthrough_chunk(
+        item: Any,
+        finish_reason_per_choice: "dict[int, str | None] | None" = None,
+    ) -> ModelResponseStream:
+        """Copy of a chunk carrying tool calls with all text content stripped.
+
+        Only tool_calls, role and finish_reason are forwarded; content is set to
+        None so response text can never be delivered raw (it flows through the
+        transform instead). Applies per choice so an n>1 chunk mixing a text
+        choice and a tool-call choice does not leak the text choice.
+
+        For a choice that carries BOTH text content AND tool_calls, ``finish_reason``
+        is suppressed on the passthrough and recorded on
+        ``finish_reason_per_choice`` (when provided) so the final synthetic text
+        chunk delivers it. Emitting the passthrough's ``finish_reason`` before the
+        text flush would let a spec-compliant SSE client stop reading at
+        ``finish_reason`` and silently drop the guardrailed text, defeating the
+        redaction purpose.
+        """
+        synthetic_choices: list[StreamingChoices] = []
+        for choice in getattr(item, "choices", None) or []:
+            delta = getattr(choice, "delta", None)
+            idx = getattr(choice, "index", 0) or 0
+            original_finish = getattr(choice, "finish_reason", None)
+            has_text = isinstance(getattr(delta, "content", None), str) and getattr(delta, "content", "") != ""
+            if has_text and original_finish is not None and finish_reason_per_choice is not None:
+                finish_reason_per_choice[idx] = original_finish
+                passthrough_finish: str | None = None
+            else:
+                passthrough_finish = original_finish
+            synthetic_choices.append(
+                StreamingChoices(
+                    index=idx,
+                    delta=Delta(
+                        content=None,
+                        role=getattr(delta, "role", None),
+                        tool_calls=getattr(delta, "tool_calls", None),
+                    ),
+                    finish_reason=passthrough_finish,
+                )
+            )
+        return ModelResponseStream(
+            id=getattr(item, "id", None),
+            created=getattr(item, "created", None),
+            model=getattr(item, "model", None),
+            choices=synthetic_choices,
+        )
+
+    @staticmethod
+    def _record_finish_reasons(item: Any, finish_reason_per_choice: dict[int, str | None]) -> None:
+        for choice in getattr(item, "choices", None) or []:
+            finish_reason = getattr(choice, "finish_reason", None)
+            if finish_reason is not None:
+                finish_reason_per_choice[getattr(choice, "index", 0) or 0] = finish_reason
+
+    @staticmethod
+    def _chunk_has_finish_reason(item: Any) -> bool:
+        choices = getattr(item, "choices", None) or []
+        return any(getattr(choice, "finish_reason", None) is not None for choice in choices)
+
     async def async_post_call_streaming_iterator_hook(
         self,
         user_api_key_dict: UserAPIKeyAuth,
@@ -334,6 +837,10 @@ class UnifiedLLMGuardrails(CustomLogger):
         sampling_rate = _streaming_flag("streaming_sampling_rate", 5)
         # Only apply the guardrail at end of stream (not per chunk).
         end_of_stream_only = _streaming_flag("streaming_end_of_stream_only", False)
+        # "block_only" (default) drops guardrail text rewrites on the streaming
+        # path; "incremental_diff" emits them as synthetic deltas (see
+        # _run_incremental_transform_stream).
+        streaming_transform_mode = _streaming_flag("streaming_transform_mode", "block_only")
         # Withhold every chunk until end-of-stream moderation passes, then
         # release the original chunks (clean) or only the block message
         # (blocked) -- moderating the whole response *before* any content
@@ -380,6 +887,35 @@ class UnifiedLLMGuardrails(CustomLogger):
         if endpoint_guardrail_translation_mappings is None:
             endpoint_guardrail_translation_mappings = load_guardrail_translation_mappings()
 
+        # Streaming text transformation (incremental_diff) diverges enough from the
+        # block_only path that it runs as its own iterator. It requires a route we
+        # can resolve up front to an OpenAI-chat handler (the only supported v1
+        # surface); anything else falls back to the block_only behavior below.
+        if streaming_transform_mode == "incremental_diff":
+            transform_call_type = self._resolve_transform_call_type(
+                user_api_key_dict=user_api_key_dict,
+                mappings=endpoint_guardrail_translation_mappings,
+            )
+            if transform_call_type is not None:
+                async for transformed_item in self._run_incremental_transform_stream(
+                    guardrail_to_apply=guardrail_to_apply,
+                    response=response,
+                    request_data=request_data,
+                    user_api_key_dict=user_api_key_dict,
+                    call_type=transform_call_type,
+                    sampling_rate=sampling_rate,
+                    end_of_stream_only=end_of_stream_only,
+                    mappings=endpoint_guardrail_translation_mappings,
+                ):
+                    yield transformed_item
+                return
+            verbose_proxy_logger.warning(
+                "UnifiedLLMGuardrails: streaming_transform_mode=incremental_diff is only supported "
+                "for the OpenAI chat completions streaming path with a resolvable request route; "
+                "falling back to block_only for %s",
+                getattr(guardrail_to_apply, "guardrail_name", None),
+            )
+
         # Infer call type from first chunk
         call_type = None
         chunk_counter = 0

litellm/proxy/guardrails/guardrail_initializers.py

@@ -16,6 +16,10 @@ def initialize_bedrock(litellm_params: LitellmParams, guardrail: Guardrail):
         event_hook=litellm_params.mode,
         guardrailIdentifier=litellm_params.guardrailIdentifier,
         guardrailVersion=litellm_params.guardrailVersion,
+        checks=litellm_params.checks,
+        content_filter_threshold=litellm_params.content_filter_threshold,
+        prompt_attack_threshold=litellm_params.prompt_attack_threshold,
+        pii_confidence_threshold=litellm_params.pii_confidence_threshold,
         default_on=litellm_params.default_on,
         disable_exception_on_block=litellm_params.disable_exception_on_block,
         mask_request_content=litellm_params.mask_request_content,

litellm/proxy/management_endpoints/key_management_endpoints.py

@@ -32,6 +32,7 @@ from litellm._uuid import uuid
 from litellm.constants import (
     LENGTH_OF_LITELLM_GENERATED_KEY,
     LITELLM_PROXY_ADMIN_NAME,
+    MINIMUM_CUSTOM_KEY_LENGTH,
     UI_SESSION_TOKEN_TEAM_ID,
 )
 from litellm.litellm_core_utils.duration_parser import duration_in_seconds
@@ -42,7 +43,7 @@ from litellm.proxy._experimental.mcp_server.db import (
     rotate_mcp_user_env_vars_master_key,
 )
 from litellm.proxy._types import *
-from litellm.proxy._types import LiteLLM_VerificationToken
+from litellm.proxy._types import LiteLLM_VerificationToken, hash_token
 from litellm.proxy.auth.auth_checks import (
     _delete_cache_key_object,
     can_team_access_model,
@@ -468,7 +469,10 @@ def handle_key_type(data: GenerateKeyRequest, data_json: dict) -> dict:
     Handle the key type.
     """
     key_type = data.key_type
-    data_json.pop("key_type", None)
+    if key_type is None:
+        data_json.pop("key_type", None)
+        return data_json
+    data_json["key_type"] = key_type.value
     if key_type == LiteLLMKeyType.LLM_API:
         data_json["allowed_routes"] = ["llm_api_routes"]
     elif key_type == LiteLLMKeyType.MANAGEMENT:
@@ -1019,6 +1023,14 @@ async def _common_key_generation_helper(
             detail={"error": f"Invalid key format. LiteLLM Virtual Key must start with 'sk-'. Received: {_masked}"},
         )
 
+    if data.key is not None and len(data.key) < MINIMUM_CUSTOM_KEY_LENGTH:
+        raise HTTPException(
+            status_code=400,
+            detail={
+                "error": f"Invalid key format. LiteLLM Virtual Key must be at least {MINIMUM_CUSTOM_KEY_LENGTH} characters long."
+            },
+        )
+
     # check org key limits - done here to handle inheriting org id from team
     if data.organization_id is not None:
         from litellm.proxy.proxy_server import prisma_client, user_api_key_cache
@@ -1471,7 +1483,7 @@ async def generate_key_fn(
     Parameters:
     - duration: Optional[str] - Specify the length of time the token is valid for. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
     - key_alias: Optional[str] - User defined key alias
-    - key: Optional[str] - User defined key value. If not set, a 16-digit unique sk-key is created for you.
+    - key: Optional[str] - User defined key value. Must start with 'sk-' and be at least 16 characters long. If not set, a 16-digit unique sk-key is created for you.
     - team_id: Optional[str] - The team id of the key
     - user_id: Optional[str] - The user id of the key
     - agent_id: Optional[str] - The agent id associated with the key.
@@ -1685,7 +1697,7 @@ async def generate_service_account_key_fn(
     Parameters:
     - duration: Optional[str] - Specify the length of time the token is valid for. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
     - key_alias: Optional[str] - User defined key alias
-    - key: Optional[str] - User defined key value. If not set, a 16-digit unique sk-key is created for you.
+    - key: Optional[str] - User defined key value. Must start with 'sk-' and be at least 16 characters long. If not set, a 16-digit unique sk-key is created for you.
     - team_id: Optional[str] - The team id of the key
     - user_id: Optional[str] - [NON-FUNCTIONAL] THIS WILL BE IGNORED. The user id of the key
     - budget_id: Optional[str] - The budget id associated with the key. Created by calling `/budget/new`.
@@ -3566,6 +3578,7 @@ async def generate_key_helper_fn(
     created_by: Optional[str] = None,
     updated_by: Optional[str] = None,
     allowed_routes: Optional[list] = None,
+    key_type: str | None = None,
     sso_user_id: Optional[str] = None,
     object_permission_id: Optional[str] = None,  # object_permission_id <-> LiteLLM_ObjectPermissionTable
     object_permission: Optional[LiteLLM_ObjectPermissionBase] = None,
@@ -3706,6 +3719,7 @@ async def generate_key_helper_fn(
             "created_by": created_by,
             "updated_by": updated_by,
             "allowed_routes": allowed_routes or [],
+            "key_type": key_type,
             "object_permission_id": object_permission_id,
             "router_settings": router_settings_json,
             "access_group_ids": access_group_ids or [],
@@ -3772,7 +3786,10 @@ async def generate_key_helper_fn(
                 return user_data
 
             ## CREATE KEY
-            verbose_proxy_logger.debug("prisma_client: Creating Key= %s", key_data)
+            verbose_proxy_logger.debug(
+                "prisma_client: Creating Key= %s",
+                {**key_data, "token": hash_token(token=token)},
+            )
             create_key_response = await prisma_client.insert_data(data=key_data, table_name="key")
 
             key_data["token_id"] = getattr(create_key_response, "token", None)
@@ -4348,7 +4365,6 @@ async def get_new_token(data: Optional[RegenerateKeyRequest]) -> str:
     if data and data.new_key is not None:
         # Reject custom key values if disabled by admin
         await _check_custom_key_allowed(data.new_key)
-        new_token = data.new_key
         if not data.new_key.startswith("sk-"):
             raise HTTPException(
                 status_code=status.HTTP_400_BAD_REQUEST,
@@ -4356,6 +4372,12 @@ async def get_new_token(data: Optional[RegenerateKeyRequest]) -> str:
                     "error": "New key must start with 'sk-'. This is to distinguish a key hash (used by litellm for logging / internal logic) from the actual key."
                 },
             )
+        if len(data.new_key) < MINIMUM_CUSTOM_KEY_LENGTH:
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail={"error": f"New key must be at least {MINIMUM_CUSTOM_KEY_LENGTH} characters long."},
+            )
+        new_token = data.new_key
     else:
         new_token = f"sk-{secrets.token_urlsafe(LENGTH_OF_LITELLM_GENERATED_KEY)}"
     return new_token
@@ -4462,7 +4484,7 @@ async def _execute_virtual_key_regeneration(
 
     new_token = await get_new_token(data=data)
     new_token_hash = hash_token(new_token)
-    new_token_key_name = f"sk-...{new_token[-4:]}"
+    new_token_key_name = abbreviate_api_key(api_key=new_token)
     update_data = {"token": new_token_hash, "key_name": new_token_key_name}
 
     non_default_values = {}
@@ -4542,7 +4564,7 @@ async def regenerate_key_fn(
     - data: Optional[RegenerateKeyRequest] - Request body containing optional parameters to update
         - key: Optional[str] - The key to regenerate.
         - new_master_key: Optional[str] - The new master key to use, if key is the master key.
-        - new_key: Optional[str] - The new key to use, if key is not the master key. If both set, new_master_key will be used.
+        - new_key: Optional[str] - The new key to use, if key is not the master key. Must start with 'sk-' and be at least 16 characters long. If both set, new_master_key will be used.
         - key_alias: Optional[str] - User-friendly key alias
         - user_id: Optional[str] - User ID associated with key
         - team_id: Optional[str] - Team ID associated with key

litellm/proxy/management_endpoints/mcp_management_endpoints.py

@@ -1475,6 +1475,7 @@ if MCP_AVAILABLE:
             temporary_server = await global_mcp_server_manager.build_mcp_server_from_table(
                 temp_record,
                 credentials_are_encrypted=False,
+                persist_discovered_endpoints=False,
             )
             _cache_temporary_mcp_server(
                 temporary_server,

litellm/proxy/management_endpoints/team_endpoints.py

@@ -1621,6 +1621,7 @@ async def update_team(
     - allowed_passthrough_routes: Optional[List[str]] - List of allowed pass through routes for the team.
     - model_rpm_limit: Optional[Dict[str, int]] - The RPM (Requests Per Minute) limit per model for this team. Example: {"gpt-4": 100, "gpt-3.5-turbo": 200}
     - model_tpm_limit: Optional[Dict[str, int]] - The TPM (Tokens Per Minute) limit per model for this team. Example: {"gpt-4": 10000, "gpt-3.5-turbo": 20000}
+    - mcp_rpm_limit: Optional[Dict[str, int]] - Per-MCP-server RPM limit for this team, keyed by MCP server name (alias if set, else the configured name). Example: {"github": 100, "slack": 200}. Applied across all keys for this team.
     Example - update team TPM Limit
     - allowed_vector_store_indexes: Optional[List[dict]] - List of allowed vector store indexes for the key. Example - [{"index_name": "my-index", "index_permissions": ["write", "read"]}]. If specified, the key will only be able to use these specific vector store indexes. Create index, using `/v1/indexes` endpoint.
     - secret_manager_settings: Optional[dict] - Secret manager settings for the team. [Docs](https://docs.litellm.ai/docs/secret_managers/overview)

litellm/proxy/management_endpoints/ui_sso.py

@@ -241,13 +241,34 @@ def _check_cli_sso_start_rate_limit(
 
 
 def _get_cli_sso_flow_or_raise(login_id: Optional[str], cache: DualCache) -> dict:
+    if isinstance(login_id, str) and login_id.startswith("sk-"):
+        raise HTTPException(
+            status_code=400,
+            detail=(
+                "Your litellm CLI is out of date and uses a login flow this proxy no longer supports. "
+                "Upgrade it with `pip install -U 'litellm[proxy]'` and run `litellm-proxy login` again."
+            ),
+        )
     if not _is_valid_cli_sso_login_id(login_id):
-        raise HTTPException(status_code=400, detail="Invalid CLI login session")
+        raise HTTPException(status_code=400, detail="Invalid CLI login session id")
 
     cache_key = _get_cli_sso_flow_cache_key(cast(str, login_id))
     flow = cache.get_cache(key=cache_key)
     if not isinstance(flow, dict) or "poll_secret_hash" not in flow:
-        raise HTTPException(status_code=400, detail="Invalid CLI login session")
+        verbose_proxy_logger.warning(
+            "CLI SSO login session not found in cache for login_id=%s. If the proxy runs multiple replicas, "
+            "a shared Redis cache (enable_redis_auth_cache: true) is required for CLI login to work.",
+            login_id,
+        )
+        raise HTTPException(
+            status_code=400,
+            detail=(
+                "CLI login session not found or expired. Run `litellm-proxy login` again. "
+                "If this happens immediately after starting a login, the proxy is likely running multiple "
+                "replicas without a shared cache; configure Redis with `enable_redis_auth_cache: true` "
+                "so every replica can see the login session."
+            ),
+        )
     return flow
 
 
@@ -4034,30 +4055,41 @@ class MicrosoftSSOHandler:
         base_url = MicrosoftSSOHandler.get_graph_api_base_url()
         # Endpoint to get app role assignments for the given service principal
         endpoint = f"/servicePrincipals/{service_principal_id}/appRoleAssignedTo"
-        url = base_url + endpoint
+        next_link: str | None = base_url + endpoint
 
         headers = {
             "Authorization": f"Bearer {access_token}",
             "Content-Type": "application/json",
         }
 
-        response = await async_client.get(url, headers=headers)
-        response_json = response.json()
-        verbose_proxy_logger.debug(f"Response from service principal app role assigned to: {response_json}")
         group_ids: List[str] = []
         service_principal_teams: List[MicrosoftServicePrincipalTeam] = []
+        page_count = 0
 
-        for _object in response_json.get("value", []):
-            if _object.get("principalType") == "Group":
-                # Append the group ID to the list
-                group_ids.append(_object.get("principalId"))
-                # Append the service principal team to the list
-                service_principal_teams.append(
-                    MicrosoftServicePrincipalTeam(
-                        principalDisplayName=_object.get("principalDisplayName"),
-                        principalId=_object.get("principalId"),
+        while next_link is not None and page_count < MicrosoftSSOHandler.MAX_GRAPH_API_PAGES:
+            response = await async_client.get(next_link, headers=headers)
+            response_json = response.json()
+            verbose_proxy_logger.debug(f"Response from service principal app role assigned to: {response_json}")
+
+            for _object in response_json.get("value", []):
+                if _object.get("principalType") == "Group":
+                    # Append the group ID to the list
+                    group_ids.append(_object.get("principalId"))
+                    # Append the service principal team to the list
+                    service_principal_teams.append(
+                        MicrosoftServicePrincipalTeam(
+                            principalDisplayName=_object.get("principalDisplayName"),
+                            principalId=_object.get("principalId"),
+                        )
                     )
-                )
+
+            next_link = response_json.get("@odata.nextLink")
+            page_count += 1
+
+        if next_link is not None and page_count >= MicrosoftSSOHandler.MAX_GRAPH_API_PAGES:
+            verbose_proxy_logger.warning(
+                f"Reached maximum page limit of {MicrosoftSSOHandler.MAX_GRAPH_API_PAGES}. Some service principal group assignments may not be included."
+            )
 
         return group_ids, service_principal_teams
 

litellm/proxy/middleware/billable_request_metrics_middleware.py

@@ -0,0 +1,234 @@
+"""
+Counts billable HTTP requests on enterprise deployments.
+
+A billable request is an inbound request to an LLM inference, MCP, or A2A
+endpoint that returns a 2xx status. The actual export happens in an injected
+recorder (see litellm.proxy.enterprise_billing.billing_metrics); when no
+recorder is injected (non-enterprise, or metering misconfigured) this
+middleware is a transparent pass-through.
+"""
+
+import re
+import threading
+from enum import Enum
+from typing import Callable, Optional, Protocol, Sequence, runtime_checkable
+
+from starlette.types import ASGIApp, Message, Receive, Scope, Send
+
+from litellm._logging import verbose_proxy_logger
+from litellm.proxy._types import LiteLLMRoutes
+
+
+class BillableCategory(str, Enum):
+    LLM = "llm"
+    MCP = "mcp"
+    A2A = "a2a"
+
+
+@runtime_checkable
+class BillingRecorder(Protocol):
+    def record(self, *, category: BillableCategory, route: str, status_code: int, model_id: Optional[str]) -> None: ...
+
+
+_MODEL_ID_HEADER = b"x-litellm-model-id"
+
+# Ordered: a longer suffix that shares an ending with a shorter one must come
+# first, e.g. "/chat/completions" before "/completions". This is the POST
+# inference surface that writes a SpendLogs row on success, so the exported
+# count lines up with the admin UI usage page for inference traffic. Billing
+# is a deliberate lower bound on SpendLogs rows: management writes that also
+# log (batch/file/fine-tuning creation, interaction cancel) and non-POST calls
+# that log (passthrough reads) never bill, so drift only ever undercounts.
+_LLM_ROUTE_SUFFIXES: tuple[str, ...] = (
+    "/chat/completions",
+    "/completions",
+    "/embeddings",
+    "/responses",
+    "/rerank",
+    "/moderations",
+    "/images/generations",
+    "/images/edits",
+    "/images/variations",
+    "/audio/transcriptions",
+    "/audio/translations",
+    "/audio/speech",
+    "/videos",  # create; GET list is excluded by the POST gate
+    "/remix",  # /v1/videos/{id}/remix
+    "/ocr",
+    "/search",  # /v1/search and /v1/vector_stores/{id}/search
+    "/rag/query",
+    "/rag/ingest",
+    ":generateContent",  # Gemini-native /v1beta/models/{model}:generateContent
+    ":streamGenerateContent",
+)
+
+# Exact paths only: a suffix match would also catch non-inference resources that
+# share the ending, e.g. the OpenAI Assistants route /v1/threads/{id}/messages
+# writes no SpendLogs row and must not bill, unlike Anthropic /v1/messages.
+_LLM_ROUTE_EXACT: tuple[str, ...] = (
+    "/v1/messages",
+    "/interactions",  # Google Interactions create; /{id} reads and /cancel do not match
+    "/v1beta/interactions",
+)
+
+# Provider passthrough prefixes (e.g. /bedrock/..., /vertex-ai/...) carry real
+# inference calls that write SpendLogs rows, so they bill. Anchored to the
+# routes enum so new providers are picked up without touching this module.
+# /langfuse forwards observability traffic, not inference: it writes no
+# SpendLogs row and must not bill.
+_NON_BILLABLE_PASSTHROUGH_PREFIXES = frozenset({"/langfuse"})
+_PASSTHROUGH_PREFIXES: tuple[str, ...] = tuple(
+    prefix
+    for prefix in LiteLLMRoutes.mapped_pass_through_routes.value
+    if prefix not in _NON_BILLABLE_PASSTHROUGH_PREFIXES
+)
+
+
+def _classify_llm_route(path: str) -> Optional[str]:
+    exact_match = next((route for route in _LLM_ROUTE_EXACT if path == route), None)
+    if exact_match is not None:
+        return exact_match
+    suffix_match = next((suffix for suffix in _LLM_ROUTE_SUFFIXES if path == suffix or path.endswith(suffix)), None)
+    if suffix_match is not None:
+        return suffix_match
+    # Deep passthrough paths only: the bare prefix itself is not an inference call.
+    return next((prefix for prefix in _PASSTHROUGH_PREFIXES if path.startswith(f"{prefix}/")), None)
+
+
+_MCP_MANAGEMENT_PREFIX = "/v1/mcp"
+_MCP_DYNAMIC_TRANSPORT = re.compile(r"/(?:toolset/)?[^/]+/mcp")
+# The REST wrapper's tool-call endpoint executes a tool and fires the same MCP
+# spend logging as the /mcp transport; its list/test siblings do not bill.
+_MCP_REST_TOOL_CALL = "/mcp-rest/tools/call"
+
+_A2A_INVOKE_SUFFIX = "/message/send"
+_A2A_TRANSPORT_PREFIXES: tuple[str, ...] = ("/v1/a2a/", "/a2a/")
+# Bare POST /a2a/{agent_id} carries the JSON-RPC method in the body, not the
+# path. Only message/send and message/stream write a SpendLogs row there; the
+# task RPCs (tasks/get, tasks/cancel, tasks/pushNotificationConfig/*, ...) are
+# forwarded upstream and write none. A path-only classifier cannot separate
+# them, so the bare route does not bill: counting a task RPC would overcount,
+# while missing a bare-path message/send only undercounts, and undercounting is
+# the sole direction this metric is allowed to drift. The /mcp transport is
+# method-agnostic by contrast because its list path logs a SpendLogs row too.
+
+
+def _classify_mcp_route(path: str) -> Optional[str]:
+    if path == _MCP_MANAGEMENT_PREFIX or path.startswith(f"{_MCP_MANAGEMENT_PREFIX}/"):
+        return None
+    if path == "/mcp" or path.startswith("/mcp/"):
+        return "/mcp"
+    if path == _MCP_REST_TOOL_CALL:
+        return "/mcp"
+    if _MCP_DYNAMIC_TRANSPORT.fullmatch(path) is not None:
+        return "/mcp"
+    return None
+
+
+def _classify_a2a_route(path: str) -> Optional[str]:
+    if path.endswith(_A2A_INVOKE_SUFFIX) and any(path.startswith(prefix) for prefix in _A2A_TRANSPORT_PREFIXES):
+        return "/a2a"
+    return None
+
+
+def classify_billable_request(path: str, method: str = "POST") -> Optional[tuple[BillableCategory, str]]:
+    """Map a request path to its (category, normalized route), or None if not billable."""
+    normalized = path.rstrip("/") or "/"
+
+    mcp_route = _classify_mcp_route(normalized)
+    if mcp_route is not None:
+        return (BillableCategory.MCP, mcp_route)
+
+    a2a_route = _classify_a2a_route(normalized)
+    if a2a_route is not None:
+        return (BillableCategory.A2A, a2a_route)
+
+    # POST-only is a conservative gate: non-POST calls can still write a
+    # SpendLogs row (passthrough reads, resource GETs) but must not bill, so
+    # any classifier-vs-dashboard mismatch is an undercount, never an overcount.
+    if method.upper() != "POST":
+        return None
+
+    llm_route = _classify_llm_route(normalized)
+    if llm_route is not None:
+        return (BillableCategory.LLM, llm_route)
+    return None
+
+
+def _extract_model_id(headers: Sequence[tuple[bytes, bytes]]) -> Optional[str]:
+    return next(
+        (value.decode("latin-1") for name, value in headers if name.lower() == _MODEL_ID_HEADER and value),
+        None,
+    )
+
+
+class BillableRequestMetricsMiddleware:
+    """
+    Pure ASGI middleware that records one billable request per 2xx response to a
+    billable endpoint. Modeled on InFlightRequestsMiddleware: it wraps `send`,
+    reads the final status and the x-litellm-model-id header off the
+    `http.response.start` message, and never blocks or fails the request path.
+    """
+
+    def __init__(
+        self,
+        app: ASGIApp,
+        recorder: Optional[BillingRecorder] = None,
+        recorder_factory: Optional[Callable[[], Optional[BillingRecorder]]] = None,
+    ) -> None:
+        self.app = app
+        self.recorder = recorder
+        # The factory defers recorder construction to the first request, AFTER the
+        # startup event has loaded the YAML config's environment_variables (license
+        # and cert env vars). Building at import time captured recorder=None for
+        # deployments configured that way. Resolved exactly once; the result
+        # (including None) is cached.
+        self._recorder_factory = recorder_factory
+        self._resolved = recorder_factory is None
+        self._resolve_lock = threading.Lock()
+
+    def _resolve_recorder(self) -> Optional[BillingRecorder]:
+        if self._resolved:
+            return self.recorder
+        # The lock keeps concurrent first requests from each building their own
+        # MeterProvider (and leaking its background exporter thread).
+        with self._resolve_lock:
+            if not self._resolved:
+                factory = self._recorder_factory
+                self.recorder = factory() if factory is not None else self.recorder
+                self._resolved = True
+        return self.recorder
+
+    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+        if scope["type"] != "http":
+            await self.app(scope, receive, send)
+            return
+
+        recorder = self._resolve_recorder()
+        if recorder is None:
+            await self.app(scope, receive, send)
+            return
+
+        classification = classify_billable_request(scope.get("path", ""), scope.get("method", "POST"))
+        if classification is None:
+            await self.app(scope, receive, send)
+            return
+
+        category, route = classification
+        status_code = 0
+        model_id: Optional[str] = None
+
+        async def send_wrapper(message: Message) -> None:
+            nonlocal status_code, model_id
+            if message["type"] == "http.response.start":
+                status_code = message["status"]
+                model_id = _extract_model_id(message.get("headers", []))
+            await send(message)
+
+        await self.app(scope, receive, send_wrapper)
+
+        if 200 <= status_code < 300:
+            try:
+                recorder.record(category=category, route=route, status_code=status_code, model_id=model_id)
+            except Exception:  # noqa: BLE001 -- metering must never fail a request that was already served
+                verbose_proxy_logger.warning("billable request metering failed for %s", route, exc_info=True)

litellm/proxy/proxy_server.py

@@ -438,10 +438,30 @@ from litellm.proxy.management_helpers.audit_logs import (
     create_object_audit_log,
 )
 from litellm.proxy.memory.memory_endpoints import router as memory_router
+from litellm.proxy.middleware.billable_request_metrics_middleware import (
+    BillableRequestMetricsMiddleware,
+    BillingRecorder,
+)
 from litellm.proxy.plugin_routes import (
-    router as plugin_router,
     register_plugins_from_config,
 )
+from litellm.proxy.plugin_routes import (
+    router as plugin_router,
+)
+
+try:
+    from litellm.proxy.enterprise_billing.billing_metrics import (
+        build_billing_metrics_recorder as _build_billing_metrics_recorder,
+    )
+    from litellm.proxy.enterprise_billing.billing_metrics import (
+        shutdown_billing_metrics_recorder as _shutdown_billing_metrics_recorder,
+    )
+
+    build_billing_metrics_recorder: Optional[Callable[..., Optional[BillingRecorder]]] = _build_billing_metrics_recorder
+    shutdown_billing_metrics_recorder: Optional[Callable[[], None]] = _shutdown_billing_metrics_recorder
+except ImportError:
+    build_billing_metrics_recorder = None
+    shutdown_billing_metrics_recorder = None
 from litellm.proxy.middleware.in_flight_requests_middleware import (
     InFlightRequestsMiddleware,
 )
@@ -461,13 +481,11 @@ from litellm.proxy.openai_files_endpoints.files_endpoints import (
 )
 from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
     passthrough_endpoint_router,
+    vertex_ai_live_websocket_passthrough,
 )
 from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
     router as llm_passthrough_router,
 )
-from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
-    vertex_ai_live_websocket_passthrough,
-)
 from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
     initialize_pass_through_endpoints,
 )
@@ -552,21 +570,19 @@ from litellm.types.proxy.management_endpoints.ui_sso import (
 from litellm.types.realtime import RealtimeQueryParams
 from litellm.types.router import (
     DeploymentTypedDict,
-)
-from litellm.types.router import ModelInfo as RouterModelInfo
-from litellm.types.router import (
     RouterGeneralSettings,
+    RoutingPlugin,
     SearchToolTypedDict,
     updateDeployment,
 )
+from litellm.types.router import ModelInfo as RouterModelInfo
 from litellm.types.scheduler import DefaultPriorities
 from litellm.types.secret_managers.main import (
     KeyManagementSettings,
     KeyManagementSystem,
 )
-from litellm.types.utils import CredentialItem, CustomHuggingfaceTokenizer
+from litellm.types.utils import CredentialItem, CustomHuggingfaceTokenizer, RawRequestTypedDict, StandardLoggingPayload
 from litellm.types.utils import ModelInfo as ModelMapInfo
-from litellm.types.utils import RawRequestTypedDict, StandardLoggingPayload
 from litellm.utils import _add_custom_logger_callback_to_specific_event
 
 try:
@@ -767,6 +783,11 @@ async def proxy_shutdown_event():
     if db_writer_client is not None:
         await db_writer_client.close()  # type: ignore[reportGeneralTypeIssues]
 
+    # final flush of billable-request counts: without it, up to one export
+    # interval of enterprise billing data is dropped on every restart
+    if shutdown_billing_metrics_recorder is not None:
+        shutdown_billing_metrics_recorder()
+
     # flush remaining langfuse logs
     if "langfuse" in litellm.success_callback:
         try:
@@ -972,11 +993,11 @@ async def proxy_startup_event(app: FastAPI):
         if is_otel_v2_enabled():
             from opentelemetry import trace as _otel_trace
 
-            from litellm.litellm_core_utils.litellm_logging import _in_memory_loggers
             from litellm.integrations.otel.logger import (
                 OpenTelemetryV2,
                 publish_global_otel_v2_provider,
             )
+            from litellm.litellm_core_utils.litellm_logging import _in_memory_loggers
 
             registered = open_telemetry_logger if isinstance(open_telemetry_logger, OpenTelemetryV2) else None
             publish_global_otel_v2_provider(
@@ -1780,6 +1801,31 @@ app.add_middleware(
 )
 
 app.add_middleware(PrometheusAuthMiddleware)
+# Added before InFlightRequestsMiddleware so it nests *inside* it: Starlette
+# makes the last-added middleware outermost. The billable count is recorded
+# after the inner app returns, so if this sat outside the in-flight tracker a
+# request could be counted as drained while its record() had not yet run, and
+# proxy_shutdown_event could flush and stop the exporter underneath it.
+app.add_middleware(
+    BillableRequestMetricsMiddleware,
+    # Factory, not an instance: the recorder is resolved on the first request so
+    # it sees premium_user and the billing env vars AFTER proxy_startup_event has
+    # loaded the YAML config's environment_variables. Building it here at import
+    # time would permanently capture recorder=None for YAML-configured
+    # deployments. The lambda reads the module globals at call time.
+    recorder_factory=lambda: (
+        build_billing_metrics_recorder(
+            premium=premium_user,
+            # Read from the license check, not the premium_user_data module
+            # global: that global is bound once at import and goes stale when
+            # the license arrives via the YAML config's environment_variables.
+            license_data=_license_check.airgapped_license_data,
+            litellm_version=version,
+        )
+        if build_billing_metrics_recorder is not None
+        else None
+    ),
+)
 app.add_middleware(InFlightRequestsMiddleware)
 app.add_middleware(SecurityHeadersMiddleware)
 
@@ -3660,6 +3706,45 @@ def _attach_redis_usage_cache(redis_cache: RedisCache, enable_redis_auth_cache:
     litellm_config_cache.redis_cache = redis_cache
 
 
+def resolve_complexity_router_plugins(
+    model_name: str,
+    complexity_router_config: dict,
+    config_file_path: str | None,
+) -> None:
+    """
+    Resolves `complexity_router_config["plugins"]` dotted-path strings to live
+    instances via `get_instance_fn` (the same convention `litellm_settings.callbacks`
+    uses), in place. Raises at config-load time if a path resolves to something that
+    doesn't implement `RoutingPlugin`, rather than deferring to a confusing
+    `AttributeError` on the first request that reaches the plugin pipeline.
+    """
+    plugin_paths = complexity_router_config.get("plugins")
+    if not isinstance(plugin_paths, list):
+        return
+
+    resolved_plugins = [
+        get_instance_fn(value=plugin_path, config_file_path=config_file_path)
+        if isinstance(plugin_path, str)
+        else plugin_path
+        for plugin_path in plugin_paths
+    ]
+    for plugin_path, resolved_plugin in zip(plugin_paths, resolved_plugins):
+        # `@runtime_checkable` only checks that `run` exists as an attribute, not that
+        # it's a coroutine function -- a synchronous `def run(self, context)` would pass
+        # isinstance() here and only fail at request time with a confusing `TypeError:
+        # object RoutingContext can't be used in 'await' expression`.
+        if not isinstance(resolved_plugin, RoutingPlugin) or not inspect.iscoroutinefunction(
+            getattr(resolved_plugin, "run", None)
+        ):
+            raise ValueError(
+                f"complexity_router_config.plugins entry {plugin_path!r} on model {model_name!r} "
+                f"resolved to {resolved_plugin!r}, which does not implement the RoutingPlugin "
+                "interface (an async `run(context)` method). Fix the referenced module before "
+                "starting the proxy."
+            )
+    complexity_router_config["plugins"] = resolved_plugins
+
+
 class ProxyConfig:
     """
     Abstraction class on top of config loading/updating logic. Gives us one place to control all config updating logic.
@@ -4720,6 +4805,13 @@ class ProxyConfig:
                 for k, v in model["litellm_params"].items():
                     if isinstance(v, str) and v.startswith("os.environ/"):
                         model["litellm_params"][k] = get_secret(v)
+                complexity_router_config = model["litellm_params"].get("complexity_router_config")
+                if isinstance(complexity_router_config, dict):
+                    resolve_complexity_router_plugins(
+                        model_name=model.get("model_name", ""),
+                        complexity_router_config=complexity_router_config,
+                        config_file_path=config_file_path,
+                    )
                 print(f"\033[32m    {model.get('model_name', '')}\033[0m")  # noqa: T201
                 litellm_model_name = model["litellm_params"]["model"]
                 litellm_model_api_base = model["litellm_params"].get("api_base", None)
@@ -14805,6 +14897,7 @@ async def get_config_list(
         "forward_client_headers_to_llm_api": {"type": "Boolean"},
         "mcp_required_fields": {"type": "List"},
         "cancel_on_disconnect": {"type": "Boolean"},
+        "skip_user_budget_on_team_key": {"type": "Boolean"},
     }
 
     return_val = []

litellm/proxy/route_llm_request.py

@@ -6,6 +6,7 @@ from fastapi import HTTPException, status
 
 import litellm
 from litellm.proxy._types import UserAPIKeyAuth
+from litellm.router_utils.common_utils import _is_proxy_admin_request
 
 # Router-internal mock_testing_* flag names — kept in sync with
 # ``litellm.types.router.MockRouterTestingParams`` by the test
@@ -363,6 +364,7 @@ async def route_request(
 
     team_id = get_team_id_from_data(data)
     router_model_names = llm_router.model_names if llm_router is not None else []
+    is_proxy_admin_without_team = team_id is None and _is_proxy_admin_request(data)
 
     # Preprocess Google GenAI generate content requests
     if route_type in ["agenerate_content", "agenerate_content_stream"]:
@@ -517,6 +519,13 @@ async def route_request(
             data["model"] = team_model_name
             return getattr(llm_router, f"{route_type}")(**data)
 
+        elif (
+            is_proxy_admin_without_team
+            and data["model"] not in router_model_names
+            and data["model"] in llm_router.team_public_model_names
+        ):
+            return getattr(llm_router, f"{route_type}")(**data)
+
         elif data["model"] in router_model_names or llm_router.has_model_id(data["model"]):
             return getattr(llm_router, f"{route_type}")(**data)
 

litellm/proxy/schema.prisma

@@ -422,6 +422,7 @@ model LiteLLM_VerificationToken {
     budget_reset_at DateTime?
     allowed_cache_controls String[] @default([])
     allowed_routes   String[] @default([])
+    key_type         String?
     policies         String[] @default([])
     access_group_ids String[] @default([])
     model_spend      Json @default("{}")
@@ -516,6 +517,7 @@ model LiteLLM_DeletedVerificationToken {
     budget_reset_at         DateTime?
     allowed_cache_controls  String[] @default([])
     allowed_routes          String[] @default([])
+    key_type                String?
     policies                String[] @default([])
     access_group_ids String[] @default([])
     model_spend             Json     @default("{}")

litellm/proxy/spend_tracking/budget_reservation.py

@@ -123,6 +123,7 @@ async def reserve_budget_for_request(
     proxy_logging_obj: ProxyLogging,
     end_user_id: Optional[str] = None,
     end_user_object: Optional[Any] = None,
+    skip_user_budget_on_team_key: bool = False,
 ) -> Optional[dict]:
     if valid_token is None or not RouteChecks.is_llm_api_route(route=route):
         return None
@@ -141,6 +142,7 @@ async def reserve_budget_for_request(
         proxy_logging_obj=proxy_logging_obj,
         end_user_id=end_user_id,
         end_user_object=end_user_object,
+        skip_user_budget_on_team_key=skip_user_budget_on_team_key,
     )
     if not counters:
         return None
@@ -296,6 +298,7 @@ async def _get_budget_counters(
     proxy_logging_obj: ProxyLogging,
     end_user_id: Optional[str] = None,
     end_user_object: Optional[Any] = None,
+    skip_user_budget_on_team_key: bool = False,
 ) -> List[_BudgetCounter]:
     counters: List[_BudgetCounter] = []
 
@@ -344,8 +347,9 @@ async def _get_budget_counters(
             )
         )
 
+    is_team_key = team_object is not None and team_object.team_id is not None
     if (
-        (team_object is None or team_object.team_id is None)
+        not (is_team_key and skip_user_budget_on_team_key)
         and user_object is not None
         and user_object.user_id is not None
         and user_object.max_budget is not None

litellm/proxy/utils.py

@@ -833,7 +833,7 @@ class ProxyLogging:
     def get_combined_callback_list(self, dynamic_success_callbacks: Optional[List], global_callbacks: List) -> List:
         if dynamic_success_callbacks is None:
             return list(global_callbacks)
-        return list(set(dynamic_success_callbacks + global_callbacks))
+        return list(dict.fromkeys(dynamic_success_callbacks + global_callbacks))
 
     def _parse_pre_mcp_call_hook_response(
         self,
@@ -3592,7 +3592,10 @@ class PrismaClient:
         """
         start_time = time.time()
         try:
-            verbose_proxy_logger.debug("PrismaClient: insert_data: %s", data)
+            verbose_proxy_logger.debug(
+                "PrismaClient: insert_data: %s",
+                {**data, "token": self.hash_token(token=data["token"])} if data.get("token") is not None else data,
+            )
             if table_name == "key":
                 token = data["token"]
                 hashed_token = self.hash_token(token=token)

litellm/router.py

@@ -26,6 +26,7 @@ from typing import (
     AsyncGenerator,
     Callable,
     Dict,
+    FrozenSet,
     Generator,
     List,
     Literal,
@@ -108,6 +109,7 @@ from litellm.router_utils.clientside_credential_handler import (
     is_clientside_credential,
 )
 from litellm.router_utils.common_utils import (
+    _is_proxy_admin_request,
     filter_team_based_models,
     filter_web_search_deployments,
 )
@@ -494,6 +496,7 @@ class Router:
         self.model_name_to_deployment_indices: Dict[str, List[int]] = {}
         # Maps (team_id, team_public_model_name) -> list of indices in model_list
         self.team_model_to_deployment_indices: Dict[Tuple[str, str], List[int]] = {}
+        self.team_public_model_names: FrozenSet[str] = frozenset()
 
         # Initialize cache attributes that ``_invalidate_model_group_info_cache``
         # touches *before* the first ``set_model_list`` below (which calls
@@ -2983,7 +2986,7 @@ class Router:
         # here before it's wiped below, instead of relying on that attempt's
         # (possibly still-pending) failure event to do it.
         refund_stale_reservation_before_retry(self.cache, kwargs)
-        set_io_token_rate_limit_request_kwargs(kwargs)
+        set_io_token_rate_limit_request_kwargs(kwargs, store_in_context=deployment_has_io_token_limits(deployment))
 
         ## DEPLOYMENT-LEVEL TAGS
         deployment_tags = deployment.get("litellm_params", {}).get("tags")
@@ -7284,6 +7287,14 @@ class Router:
                     raise e
         return returned_healthy_deployments
 
+    @staticmethod
+    def _json_default_stable_id(value: object) -> str:
+        """json.dumps default= for _generate_model_id: plain str() on an arbitrary
+        object (e.g. a RoutingPlugin instance) falls back to object.__repr__'s
+        `<module.Class object at 0x...>`, so the hash -- and deployment id -- would
+        change every restart. Use the class name instead, stable across restarts."""
+        return f"{type(value).__module__}.{type(value).__qualname__}"
+
     def _generate_model_id(self, model_group: str, litellm_params: dict):
         """
         Helper function to consistently generate the same id for a deployment
@@ -7299,14 +7310,14 @@ class Router:
             if isinstance(k, str):
                 parts.append(k)
             elif isinstance(k, dict):
-                parts.append(json.dumps(k))
+                parts.append(json.dumps(k, default=self._json_default_stable_id))
             else:
                 parts.append(str(k))
 
             if isinstance(v, str):
                 parts.append(v)
             elif isinstance(v, dict):
-                parts.append(json.dumps(v))
+                parts.append(json.dumps(v, default=self._json_default_stable_id))
             else:
                 parts.append(str(v))
 
@@ -7792,6 +7803,7 @@ class Router:
         self.model_id_to_deployment_index_map = {}  # Reset the index
         self.model_name_to_deployment_indices = {}  # Reset the model_name index
         self.team_model_to_deployment_indices = {}  # Reset the team_model index
+        self.team_public_model_names = frozenset()
         # Reset per-strategy router registries so hot-reload doesn't leave
         # stale routers pointing at the old model_list.
         self.quality_routers = {}
@@ -8143,6 +8155,9 @@ class Router:
                 self.team_model_to_deployment_indices[key] = updated_indices
             else:
                 del self.team_model_to_deployment_indices[key]
+        self.team_public_model_names = frozenset(
+            public_model_name for _, public_model_name in self.team_model_to_deployment_indices
+        )
 
     def _update_team_model_index(self, model: dict, idx: int) -> None:
         """
@@ -8156,6 +8171,7 @@ class Router:
         team_public_model_name = (model.get("model_info") or {}).get("team_public_model_name")
         if team_id and team_public_model_name:
             key = (team_id, team_public_model_name)
+            self.team_public_model_names = self.team_public_model_names | frozenset({team_public_model_name})
             if key not in self.team_model_to_deployment_indices:
                 self.team_model_to_deployment_indices[key] = []
             if idx not in self.team_model_to_deployment_indices[key]:
@@ -9110,6 +9126,7 @@ class Router:
         """
         self.model_name_to_deployment_indices.clear()
         self.team_model_to_deployment_indices.clear()
+        self.team_public_model_names = frozenset()
 
         for idx, model in enumerate(model_list):
             model_name = model.get("model_name")
@@ -10018,7 +10035,10 @@ class Router:
         return [m for m in self.model_list if m["litellm_params"]["model"] == model]
 
     def _try_early_resolve_deployments_for_model_not_in_names(
-        self, model: str, request_team_id: Optional[str]
+        self,
+        model: str,
+        request_team_id: Optional[str],
+        include_team_models: bool = False,
     ) -> Optional[Tuple[str, Union[List, Dict]]]:
         """
         When ``model`` is not in ``self.model_names``, try team routes, pattern routes,
@@ -10033,6 +10053,30 @@ class Router:
             team_deployments = self._get_all_deployments(model_name=model, team_id=request_team_id)
             if team_deployments:
                 return model, team_deployments
+        elif include_team_models:
+            team_deployments = [
+                self.model_list[index]
+                for (_, public_model_name), indices in self.team_model_to_deployment_indices.items()
+                if public_model_name == model
+                for index in indices
+            ]
+            team_ids = {
+                team_id
+                for deployment in team_deployments
+                for team_id in [(deployment.get("model_info") or {}).get("team_id")]
+                if team_id is not None
+            }
+            if len(team_ids) > 1:
+                raise litellm.BadRequestError(
+                    message=(
+                        f"Model name '{model}' matches deployments from multiple teams. "
+                        "Specify the deployment ID directly to disambiguate."
+                    ),
+                    model=model,
+                    llm_provider="",
+                )
+            if team_deployments:
+                return model, team_deployments
 
         pattern_deployments = self.pattern_router.get_deployments_by_pattern(
             model=model,
@@ -10097,7 +10141,11 @@ class Router:
         if _model_from_alias is not None:
             model = _model_from_alias
 
-        early = self._try_early_resolve_deployments_for_model_not_in_names(model=model, request_team_id=request_team_id)
+        early = self._try_early_resolve_deployments_for_model_not_in_names(
+            model=model,
+            request_team_id=request_team_id,
+            include_team_models=_is_proxy_admin_request(request_kwargs),
+        )
         if early is not None:
             return early
 

litellm/router_strategy/complexity_router/complexity_router.py

@@ -18,7 +18,7 @@ from __future__ import annotations
 import asyncio
 import random
 import re
-from typing import TYPE_CHECKING, Any, Literal, Optional, Union, cast
+from typing import TYPE_CHECKING, Any, Literal, Union, cast
 
 from pydantic import BaseModel
 
@@ -98,9 +98,9 @@ def _sanitize_user_api_key_auth(auth: Any) -> Any:
     return auth
 
 
-def _classifier_call_metadata(metadata: dict[str, Any] | None) -> dict[str, Any] | None:
+def _classifier_call_metadata(metadata: dict[str, Any] | None) -> dict[str, Any]:
     if not metadata:
-        return metadata
+        return {}
     return {
         k: _sanitize_user_api_key_auth(v) if k == "user_api_key_auth" else v
         for k, v in metadata.items()
@@ -468,6 +468,38 @@ class ComplexityRouter(CustomLogger):
     def _tier_pools(self) -> dict[str, list[str]]:
         return {tier: (models if isinstance(models, list) else [models]) for tier, models in self.config.tiers.items()}
 
+    async def _pick_model_for_tier(
+        self,
+        tier: ComplexityTier,
+        raw_messages: list[dict[str, Any]] | None,
+        resolved_messages: list[dict[str, Any]] | None,
+        request_kwargs: dict,
+    ) -> str:
+        if not self.config.plugins:
+            return self.get_model_for_tier(tier)
+
+        from litellm.types.router import RoutingContext
+
+        tier_key = tier.value
+        metadata_key = "litellm_metadata" if "litellm_metadata" in request_kwargs else "metadata"
+        context = RoutingContext(
+            raw_messages=raw_messages or [],
+            structured_messages=resolved_messages or [],
+            candidate_models=list(self._tier_pools().get(tier_key, [])),
+            metadata=request_kwargs.get(metadata_key) or {},
+        )
+        for plugin in self.config.plugins:
+            context = await plugin.run(context)
+
+        if not context.candidate_models:
+            # A plugin narrowing a tier to zero candidates is a policy decision (e.g. no
+            # model this tenant's budget allows) -- falling back to default_model here
+            # (which was never checked against the plugins) would let that policy be
+            # silently bypassed. Raise instead, matching the Router-level plugin
+            # pipeline's own fail-closed behavior for the same situation.
+            raise ValueError(f"No candidate models left for tier {tier_key} after routing-plugin filtering")
+        return self._pick_from_tier_value(context.candidate_models, tier_key)
+
     def _ensure_adaptive_router(self) -> Any | None:
         if not self.config.adaptive:
             return None
@@ -731,8 +763,8 @@ class ComplexityRouter(CustomLogger):
         # embedding call. Forwarding it would let the embedding's cost callback finalize the
         # reservation, so the routed completion's own callback then skips incrementing the
         # key/team budget. Key/team attribution fields are preserved for spend logging.
-        metadata = _classifier_call_metadata(request_kwargs.get("metadata")) or {}
-        litellm_metadata = _classifier_call_metadata(request_kwargs.get("litellm_metadata")) or {}
+        metadata = _classifier_call_metadata(request_kwargs.get("metadata"))
+        litellm_metadata = _classifier_call_metadata(request_kwargs.get("litellm_metadata"))
         query_vector = (
             await encoder.aencode_queries([user_message], metadata=metadata, litellm_metadata=litellm_metadata)
         )[0]
@@ -809,6 +841,44 @@ class ComplexityRouter(CustomLogger):
 
         return user_message, system_prompt
 
+    @staticmethod
+    def _iter_metadata_dicts(request_kwargs: dict) -> list[dict]:
+        """Metadata may land on `metadata` or `litellm_metadata` depending on the
+        endpoint, mirroring DeploymentAffinityCheck's precedence."""
+        return [
+            metadata
+            for metadata_key in ("litellm_metadata", "metadata")
+            if isinstance(metadata := request_kwargs.get(metadata_key), dict)
+        ]
+
+    @staticmethod
+    def _get_session_id_from_request_kwargs(request_kwargs: dict) -> str | None:
+        """Resolve a client-supplied session_id."""
+        for metadata in ComplexityRouter._iter_metadata_dicts(request_kwargs):
+            session_id = metadata.get("session_id")
+            if session_id is not None:
+                return str(session_id)
+        return None
+
+    @staticmethod
+    def _get_user_api_key_hash_from_request_kwargs(request_kwargs: dict) -> str | None:
+        """Resolve the proxy-derived API key hash, the same trust boundary
+        DeploymentAffinityCheck uses for its own key-based affinity (not the
+        client-supplied OpenAI `user` param, which isn't authenticated)."""
+        for metadata in ComplexityRouter._iter_metadata_dicts(request_kwargs):
+            user_key = metadata.get("user_api_key_hash")
+            if user_key is not None:
+                return str(user_key)
+        return None
+
+    def _get_session_affinity_cache_key(self, session_id: str, request_kwargs: dict) -> str:
+        # Namespace by the caller's API key hash so two different callers reusing the
+        # same client-supplied session_id can't poison each other's routing pin. Falls
+        # back to "unscoped" only when there's no authenticated caller to scope by
+        # (e.g. direct Router usage without the proxy layer).
+        caller_scope = self._get_user_api_key_hash_from_request_kwargs(request_kwargs) or "unscoped"
+        return f"complexity_router_session_affinity:v1:{self.model_name}:{caller_scope}:{session_id}"
+
     async def async_pre_routing_hook(
         self,
         model: str,
@@ -816,10 +886,76 @@ class ComplexityRouter(CustomLogger):
         messages: list[dict[str, Any]] | None = None,
         input: Union[str, list] | None = None,
         specific_deployment: bool | None = False,
-    ) -> Optional[PreRoutingHookResponse]:
+    ) -> PreRoutingHookResponse | None:
         """
         Pre-routing hook called before the routing decision.
 
+        When `session_affinity` is enabled and a session_id is resolvable on the request,
+        pins the model chosen on the session's first turn and reuses it for every later
+        turn, skipping classification entirely. Otherwise delegates to `_classify_and_route`.
+
+        Skipped entirely when `plugins` are configured: reusing a stale pin would bypass
+        the plugin pipeline on every turn after the first, since a pinned model was never
+        re-checked against a policy plugin whose decision can change between turns (e.g. a
+        budget plugin, once the session's spend crosses its cap).
+        """
+        from litellm.types.router import PreRoutingHookResponse
+
+        use_session_affinity = self.config.session_affinity and not self.config.plugins
+        session_id = self._get_session_id_from_request_kwargs(request_kwargs) if use_session_affinity else None
+        cache_key = self._get_session_affinity_cache_key(session_id, request_kwargs) if session_id is not None else None
+
+        if cache_key is not None:
+            pinned_model = await self.litellm_router_instance.cache.async_get_cache(key=cache_key)
+            if isinstance(pinned_model, str):
+                # Refresh the TTL on every hit so an active session doesn't lose its
+                # pin mid-conversation just because it outlives the original write.
+                await self.litellm_router_instance.cache.async_set_cache(
+                    key=cache_key,
+                    value=pinned_model,
+                    ttl=self.config.session_affinity_ttl_seconds,
+                )
+                if self.config.adaptive:
+                    from litellm.router_strategy.adaptive_router.config import (
+                        ADAPTIVE_ROUTER_CHOSEN_MODEL_KEY,
+                    )
+
+                    kwargs_metadata = request_kwargs.setdefault("metadata", {})
+                    if isinstance(kwargs_metadata, dict):
+                        kwargs_metadata[ADAPTIVE_ROUTER_CHOSEN_MODEL_KEY] = pinned_model
+                verbose_router_logger.info(
+                    f"ComplexityRouter: routing decision cause=session_affinity_pin, routed_model={pinned_model}"
+                )
+                has_original_messages = messages is not None and len(messages) > 0
+                return PreRoutingHookResponse(
+                    model=pinned_model,
+                    messages=messages if has_original_messages else None,
+                )
+
+        response = await self._classify_and_route(
+            model=model,
+            request_kwargs=request_kwargs,
+            messages=messages,
+            input=input,
+            specific_deployment=specific_deployment,
+        )
+        if cache_key is not None and response is not None:
+            await self.litellm_router_instance.cache.async_set_cache(
+                key=cache_key,
+                value=response.model,
+                ttl=self.config.session_affinity_ttl_seconds,
+            )
+        return response
+
+    async def _classify_and_route(
+        self,
+        model: str,
+        request_kwargs: dict,
+        messages: list[dict[str, Any]] | None = None,
+        input: Union[str, list] | None = None,
+        specific_deployment: bool | None = False,
+    ) -> PreRoutingHookResponse | None:
+        """
         Classifies the request by complexity and returns the appropriate model.
         Supports chat completions (messages), Responses API (input), and other
         formats via the guardrail translation handler dispatch.
@@ -849,14 +985,26 @@ class ComplexityRouter(CustomLogger):
 
         if user_message is None:
             verbose_router_logger.debug("ComplexityRouter: No user message found, routing to default model")
+            if not self.config.plugins and self.config.default_model:
+                # No plugins configured: preserve the pre-existing default_model-first
+                # priority exactly (changing it would be a silent behavior change for
+                # every non-plugin user, not just a security fix).
+                routed_model = self.config.default_model
+            else:
+                # Plugins configured: default_model must never bypass them, so it's not
+                # checked here at all -- _pick_model_for_tier -> get_model_for_tier still
+                # falls back to it (after the MEDIUM tier) once the plugin pipeline runs.
+                routed_model = await self._pick_model_for_tier(
+                    ComplexityTier.MEDIUM, messages, resolved_messages, request_kwargs
+                )
             return PreRoutingHookResponse(
-                model=self.config.default_model or self.get_model_for_tier(ComplexityTier.MEDIUM),
+                model=routed_model,
                 messages=messages if has_original_messages else None,
             )
 
         override_tier = await self._resolve_keyword_tier_override(user_message, request_kwargs)
         if override_tier is not None:
-            routed_model = self.get_model_for_tier(override_tier)
+            routed_model = await self._pick_model_for_tier(override_tier, messages, resolved_messages, request_kwargs)
             cause = "semantic_keyword_match" if self.config.semantic_keyword_matching else "literal_keyword_match"
             verbose_router_logger.info(
                 f"ComplexityRouter: routing decision cause={cause}, "
@@ -882,7 +1030,7 @@ class ComplexityRouter(CustomLogger):
                 f"signals={signals}, routed_model={routed_model}"
             )
         else:
-            routed_model = self.get_model_for_tier(tier)
+            routed_model = await self._pick_model_for_tier(tier, messages, resolved_messages, request_kwargs)
             verbose_router_logger.info(
                 f"ComplexityRouter: routing decision cause=complexity_scorer, tier={tier.value}, "
                 f"score={score:.3f}, signals={signals}, routed_model={routed_model}"

litellm/router_strategy/complexity_router/config.py

@@ -10,7 +10,7 @@ from typing import Literal
 
 from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
 
-from litellm.types.router import AdaptiveRouterWeights
+from litellm.types.router import AdaptiveRouterWeights, RoutingPlugin
 
 
 class ComplexityTier(str, Enum):
@@ -361,7 +361,26 @@ class ComplexityRouterConfig(BaseModel):
         description="Minimum cosine similarity for a semantic keyword match",
     )
 
-    model_config = ConfigDict(extra="allow")  # Allow additional fields
+    # Session affinity: pin the first turn's routed model for the rest of the session
+    session_affinity: bool = Field(
+        default=False,
+        description=(
+            "When True and a session_id is resolvable on the request, pin the model chosen on the "
+            "session's first turn and reuse it for every later turn, skipping re-classification."
+        ),
+    )
+    session_affinity_ttl_seconds: int = Field(
+        default=3600,
+        gt=0,
+        description="TTL for the session affinity pin; refreshed on every cache hit",
+    )
+
+    plugins: list[RoutingPlugin] | None = Field(
+        default=None,
+        description="RoutingPlugin instances that narrow the classified tier's candidate models before selection",
+    )
+
+    model_config = ConfigDict(extra="allow", arbitrary_types_allowed=True)  # Allow additional fields
 
     @field_validator("tiers", mode="before")
     @classmethod
@@ -407,6 +426,15 @@ class ComplexityRouterConfig(BaseModel):
             raise ValueError("keyword_tier_rules must be non-empty when semantic_keyword_matching is enabled")
         return self
 
+    @model_validator(mode="after")
+    def _validate_plugins_adaptive_combo(self) -> "ComplexityRouterConfig":
+        if self.plugins and self.adaptive:
+            raise ValueError(
+                "plugins and adaptive=True cannot both be set: adaptive's bandit selection doesn't yet "
+                "consume plugin-narrowed candidate pools. Disable adaptive or remove plugins."
+            )
+        return self
+
 
 # Combined default config
 DEFAULT_COMPLEXITY_CONFIG = ComplexityRouterConfig()

litellm/router_utils/common_utils.py

@@ -1,14 +1,27 @@
 import hashlib
 import json
+from collections.abc import Mapping
 from typing import TYPE_CHECKING, Dict, List, Optional, Union
 
 if TYPE_CHECKING:
     from litellm.types.llms.openai import OpenAIFileObject
 
+from litellm.exceptions import BadRequestError
 from litellm.types.router import CredentialLiteLLMParams
 from litellm._logging import verbose_logger
 
 
+def _is_proxy_admin_request(request_kwargs: Optional[Mapping[str, object]]) -> bool:
+    if request_kwargs is None:
+        return False
+    metadata_value = request_kwargs.get("metadata")
+    litellm_metadata_value = request_kwargs.get("litellm_metadata")
+    metadata = metadata_value if isinstance(metadata_value, Mapping) else {}
+    litellm_metadata = litellm_metadata_value if isinstance(litellm_metadata_value, Mapping) else {}
+    user_api_key_auth = metadata.get("user_api_key_auth") or litellm_metadata.get("user_api_key_auth")
+    return getattr(user_api_key_auth, "user_role", None) == "proxy_admin"
+
+
 def get_litellm_params_sensitive_credential_hash(litellm_params: dict) -> str:
     """
     Hash of the credential params, used for mapping the file id to the right model
@@ -59,6 +72,40 @@ def filter_team_based_models(
     metadata = request_kwargs.get("metadata") or {}
     litellm_metadata = request_kwargs.get("litellm_metadata") or {}
     request_team_id = metadata.get("user_api_key_team_id") or litellm_metadata.get("user_api_key_team_id")
+    if request_team_id is None and _is_proxy_admin_request(request_kwargs) and isinstance(healthy_deployments, list):
+        requested_model = (
+            request_kwargs.get("model") or metadata.get("model_group") or litellm_metadata.get("model_group")
+        )
+        candidate_deployments = tuple(
+            (deployment.get("model_name"), deployment.get("model_info") or {}) for deployment in healthy_deployments
+        )
+        team_ids = frozenset(
+            team_id
+            for _, model_info in candidate_deployments
+            for team_id in [model_info.get("team_id")]
+            if team_id is not None
+        )
+        matches_requested_model = (
+            isinstance(requested_model, str)
+            and bool(candidate_deployments)
+            and all(
+                model_info.get("team_id") is not None
+                and (model_name == requested_model or model_info.get("team_public_model_name") == requested_model)
+                for model_name, model_info in candidate_deployments
+            )
+        )
+        if matches_requested_model and len(team_ids) > 1:
+            raise BadRequestError(
+                message=(
+                    f"Model name '{requested_model}' matches deployments from multiple teams. "
+                    "Specify the deployment ID directly to disambiguate."
+                ),
+                model=requested_model,
+                llm_provider="",
+            )
+        if matches_requested_model:
+            return healthy_deployments
+
     ids_to_remove = set()
     if isinstance(healthy_deployments, dict):
         return healthy_deployments

litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py

@@ -43,14 +43,21 @@ ITPM_CACHE_KEY = "_litellm_itpm_cache_key"
 OTPM_CACHE_KEY = "_litellm_otpm_cache_key"
 
 
-def set_io_token_rate_limit_request_kwargs(kwargs: Optional[dict[str, Any]]) -> None:
+def set_io_token_rate_limit_request_kwargs(kwargs: Optional[dict[str, Any]], store_in_context: bool = True) -> None:
     # The reservation sentinels are server-only, but `metadata` is caller
     # controlled on proxy requests. Strip any client-supplied copies here (this
     # runs before the router stashes its own reservation) so a forged
     # reservation can't drive the post-call reconcile/refund against an
     # arbitrary counter and bypass the configured limits.
     _clear_reservation_from_kwargs(kwargs)
-    _io_token_rate_limit_request_kwargs.set(kwargs)
+    # The context slot pins the entire request kwargs (messages included) for
+    # the lifetime of the surrounding context, which outlives the request when
+    # the context is captured by pooled resources (e.g. a redis connection
+    # created mid-request). Only ITPM/OTPM-limited deployments read it, so for
+    # every other deployment overwrite the slot with None instead of the
+    # kwargs; overwriting (rather than skipping) also releases a previous
+    # request's kwargs when a context is reused.
+    _io_token_rate_limit_request_kwargs.set(kwargs if store_in_context else None)
 
 
 def get_io_token_rate_limit_request_kwargs() -> Optional[dict[str, Any]]:

litellm/types/guardrails.py

@@ -56,6 +56,9 @@ from litellm.types.proxy.guardrails.guardrail_hooks.cisco_ai_defense import (
 from litellm.types.proxy.guardrails.guardrail_hooks.headroom import (
     HeadroomGuardrailConfigModel,
 )
+from litellm.types.proxy.guardrails.guardrail_hooks.compresr import (
+    CompresrGuardrailConfigModel,
+)
 
 """
 Pydantic object defining how to set guardrails on litellm proxy
@@ -123,6 +126,7 @@ class SupportedGuardrailIntegrations(Enum):
     VIGIL_GUARD = "vigil_guard"
     REPELLOAI = "repelloai"
     HEADROOM = "headroom"
+    COMPRESR = "compresr"
 
 
 class Role(Enum):
@@ -384,6 +388,86 @@ class PresidioConfigModel(PresidioPresidioConfigModelUserInterface):
     mock_redacted_text: Optional[dict] = Field(default=None, description="Mock redacted text for testing")
 
 
+BedrockChecksContentFilterCategory = Literal["VIOLENCE", "HATE", "SEXUAL", "MISCONDUCT", "INSULTS"]
+BedrockChecksPromptAttackCategory = Literal["JAILBREAK", "PROMPT_INJECTION", "PROMPT_LEAKAGE"]
+BedrockChecksSensitiveInformationEntity = Literal[
+    "ADDRESS",
+    "AGE",
+    "AWS_ACCESS_KEY",
+    "AWS_SECRET_KEY",
+    "CA_HEALTH_NUMBER",
+    "CA_SOCIAL_INSURANCE_NUMBER",
+    "CREDIT_DEBIT_CARD_CVV",
+    "CREDIT_DEBIT_CARD_EXPIRY",
+    "CREDIT_DEBIT_CARD_NUMBER",
+    "DRIVER_ID",
+    "EMAIL",
+    "INTERNATIONAL_BANK_ACCOUNT_NUMBER",
+    "IP_ADDRESS",
+    "LICENSE_PLATE",
+    "MAC_ADDRESS",
+    "NAME",
+    "PASSWORD",
+    "PHONE",
+    "PIN",
+    "SWIFT_CODE",
+    "UK_NATIONAL_HEALTH_SERVICE_NUMBER",
+    "UK_NATIONAL_INSURANCE_NUMBER",
+    "UK_UNIQUE_TAXPAYER_REFERENCE_NUMBER",
+    "URL",
+    "USERNAME",
+    "US_BANK_ACCOUNT_NUMBER",
+    "US_BANK_ROUTING_NUMBER",
+    "US_INDIVIDUAL_TAX_IDENTIFICATION_NUMBER",
+    "US_PASSPORT_NUMBER",
+    "US_SOCIAL_SECURITY_NUMBER",
+    "VEHICLE_IDENTIFICATION_NUMBER",
+]
+
+
+class BedrockChecksContentFilterCategoryItem(BaseModel):
+    category: BedrockChecksContentFilterCategory
+
+
+class BedrockChecksContentFilterModel(BaseModel):
+    categories: list[BedrockChecksContentFilterCategoryItem]
+
+
+class BedrockChecksPromptAttackCategoryItem(BaseModel):
+    category: BedrockChecksPromptAttackCategory
+
+
+class BedrockChecksPromptAttackModel(BaseModel):
+    categories: list[BedrockChecksPromptAttackCategoryItem]
+
+
+class BedrockChecksSensitiveInformationEntityItem(BaseModel):
+    type: BedrockChecksSensitiveInformationEntity
+
+
+class BedrockChecksSensitiveInformationModel(BaseModel):
+    entities: list[BedrockChecksSensitiveInformationEntityItem]
+
+
+class BedrockChecksConfigModel(BaseModel):
+    """Inline `checks` config for the resource-less Bedrock InvokeGuardrailChecks API.
+
+    Include only the checks you want to run; at least one must be set.
+    """
+
+    contentFilter: BedrockChecksContentFilterModel | None = None
+    promptAttack: BedrockChecksPromptAttackModel | None = None
+    sensitiveInformation: BedrockChecksSensitiveInformationModel | None = None
+
+    @model_validator(mode="after")
+    def _require_at_least_one_check(self) -> "BedrockChecksConfigModel":
+        if self.contentFilter is None and self.promptAttack is None and self.sensitiveInformation is None:
+            raise ValueError(
+                "Bedrock 'checks' must enable at least one of: contentFilter, promptAttack, sensitiveInformation."
+            )
+        return self
+
+
 class BedrockGuardrailConfigModel(BaseModel):
     """Configuration parameters for the AWS Bedrock guardrail"""
 
@@ -408,6 +492,35 @@ class BedrockGuardrailConfigModel(BaseModel):
     )
     aws_sts_endpoint: Optional[str] = Field(default=None, description="AWS STS endpoint URL")
     aws_bedrock_runtime_endpoint: Optional[str] = Field(default=None, description="AWS Bedrock runtime endpoint URL")
+    checks: BedrockChecksConfigModel | None = Field(
+        default=None,
+        description="Inline safeguards for the resource-less InvokeGuardrailChecks API "
+        "(contentFilter / promptAttack / sensitiveInformation). When set, the guardrail "
+        "calls InvokeGuardrailChecks instead of ApplyGuardrail and no guardrailIdentifier "
+        "is required. Mutually exclusive with guardrailIdentifier.",
+    )
+    content_filter_threshold: float | None = Field(
+        default=0.5,
+        ge=0.0,
+        le=1.0,
+        description="InvokeGuardrailChecks: block when any contentFilter severityScore >= "
+        "this value (scores are in [0,1]). Set to null to make the content filter "
+        "detect-only (logged, never blocks).",
+    )
+    prompt_attack_threshold: float | None = Field(
+        default=0.5,
+        ge=0.0,
+        le=1.0,
+        description="InvokeGuardrailChecks: block when any promptAttack severityScore >= "
+        "this value (scores are in [0,1]). Set to null to make prompt-attack detection detect-only.",
+    )
+    pii_confidence_threshold: float | None = Field(
+        default=0.5,
+        ge=0.0,
+        le=1.0,
+        description="InvokeGuardrailChecks: block when any sensitiveInformation confidenceScore "
+        ">= this value (scores are in [0,1]). Set to null to make PII detection detect-only.",
+    )
 
 
 class LakeraV2GuardrailConfigModel(BaseModel):
@@ -697,7 +810,7 @@ class BaseLitellmParams(ContentFilterConfigModel):  # works for new and patch up
         default="fail_closed",
         description=(
             "Behavior when a guardrail endpoint is unreachable due to network errors. "
-            "Implemented by guardrail='generic_guardrail_api', 'akto', 'vigil_guard', 'repelloai', and 'headroom'. "
+            "Implemented by guardrail='generic_guardrail_api', 'akto', 'vigil_guard', 'repelloai', 'headroom', and 'compresr'. "
             "'fail_closed' raises an error (default). 'fail_open' logs a critical error and allows the request to proceed."
         ),
     )
@@ -790,6 +903,7 @@ class LitellmParams(
     BedrockGuardrailConfigModel,
     LakeraV2GuardrailConfigModel,
     HeadroomGuardrailConfigModel,
+    CompresrGuardrailConfigModel,
     RepelloAIGuardrailConfigModel,
     LassoGuardrailConfigModel,
     PillarGuardrailConfigModel,
@@ -925,6 +1039,7 @@ class ApplyGuardrailRequest(BaseModel):
     entities: Optional[List[PiiEntityType]] = None
     input_type: str = "request"
     messages: Optional[List[Dict[str, Any]]] = None
+    metadata: Dict[str, Any] | None = None
 
 
 class ApplyGuardrailResponse(BaseModel):

litellm/types/integrations/custom_logger.py

@@ -3,6 +3,7 @@ from typing import Any, Dict, List, Optional
 from pydantic import BaseModel, Field
 
 CHAT_COMPLETION_AGENTIC_SURFACE = "chat_completions"
+RESPONSES_AGENTIC_SURFACE = "responses"
 CODE_INTERPRETER_INTERCEPTION_PREFIX = "_code_interpreter_interception"
 NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES = frozenset(
     ("_websearch_interception", "_compression_interception")

litellm/types/integrations/prometheus.py

@@ -213,6 +213,8 @@ DEFINED_PROMETHEUS_METRICS = Literal[
     "litellm_input_audio_tokens_metric",
     "litellm_output_reasoning_tokens_metric",
     "litellm_output_audio_tokens_metric",
+    "litellm_video_duration_seconds_metric",
+    "litellm_images_generated_metric",
     "litellm_deployment_successful_fallbacks",
     "litellm_deployment_failed_fallbacks",
     "litellm_remaining_team_budget_metric",
@@ -506,6 +508,9 @@ class PrometheusMetricLabels:
     litellm_output_reasoning_tokens_metric = litellm_output_tokens_metric
     litellm_output_audio_tokens_metric = litellm_output_tokens_metric
 
+    litellm_video_duration_seconds_metric = litellm_output_tokens_metric
+    litellm_images_generated_metric = litellm_output_tokens_metric
+
     litellm_deployment_state = [
         UserAPIKeyLabelNames.v2_LITELLM_MODEL_NAME.value,
         UserAPIKeyLabelNames.MODEL_ID.value,
@@ -717,6 +722,8 @@ class PrometheusMetricLabels:
             "litellm_input_tokens_metric",
             "litellm_total_tokens_metric",
             "litellm_output_tokens_metric",
+            "litellm_video_duration_seconds_metric",
+            "litellm_images_generated_metric",
         }
     )
     # Managed batch metrics

litellm/types/llms/anthropic.py

@@ -439,6 +439,9 @@ class ContentThinkingSignatureBlockDelta(TypedDict):
     signature: str
 
 
+StreamingContentBlockDeltaType = Literal["text_delta", "input_json_delta", "thinking_delta", "signature_delta"]
+
+
 class ContentBlockDelta(TypedDict):
     type: Literal["content_block_delta"]
     index: int