merge(litellm_internal_staging): resolve claude_code registration conflicts

Keep staging session fixture (resolve_proxy + build_gateway) and config,
retain typed load_all_deployments guards in _compat_models

.github/CODEOWNERS

@@ -1,2 +1,3 @@
 /ui/ @yuneng-jiang @ryan-crabbe-berri
 /litellm/proxy/_experimental/out/ @yuneng-jiang @ryan-crabbe-berri
+/ui/litellm-dashboard/src/lib/http/schema.d.ts

.github/workflows/test-unit-proxy-db.yml

@@ -5,6 +5,8 @@ on:
     branches:
       - main
       - litellm_internal_staging
+      - litellm_oss_staging
+      - "litellm_**"
 
 permissions:
   contents: read

.github/workflows/zizmor.yml

@@ -4,7 +4,11 @@ on:
   push:
     branches: [main, litellm_internal_staging]
   pull_request:
-    branches: [main, litellm_internal_staging]
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_staging
+      - "litellm_**"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

docker/build_from_pip/Dockerfile.build_from_pip

@@ -36,7 +36,7 @@ RUN uv venv --python python && \
       "opentelemetry-api==1.28.0" \
       "opentelemetry-sdk==1.28.0" \
       "opentelemetry-exporter-otlp==1.28.0" \
-      "ddtrace==2.19.0" \
+      "ddtrace==4.11.0" \
       "sentry-sdk==2.21.0" \
       "mangum==0.17.0" \
       "azure-ai-contentsafety==1.0.0" \

enterprise/litellm_enterprise/enterprise_callbacks/llm_guard.py

@@ -7,7 +7,8 @@
 #  Thank you users! We ❤️ you! - Krrish & Ishaan
 ## This provides an LLM Guard Integration for content moderation on the proxy
 
-from typing import Literal, Optional
+import asyncio
+from typing import Optional
 
 import aiohttp
 from fastapi import HTTPException
@@ -18,7 +19,6 @@ from litellm.integrations.custom_logger import CustomLogger
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.utils import CallTypesLiteral
-from litellm.utils import get_formatted_prompt
 
 
 class _ENTERPRISE_LLMGuard(CustomLogger):
@@ -46,45 +46,44 @@ class _ENTERPRISE_LLMGuard(CustomLogger):
         except Exception:
             pass
 
-    async def moderation_check(self, text: str):
+    async def moderation_check(self, text: str) -> str:
         """
+        Runs the LLM Guard moderation check on ``text``.
+
+        Raises an HTTPException when the content violates the safety policy;
+        otherwise returns the sanitized prompt from LLM Guard, falling back to
+        the original text when the API does not provide one.
+
         [TODO] make this more performant for high-throughput scenario
         """
         try:
-            async with aiohttp.ClientSession() as session:
-                if self.mock_redacted_text is not None:
-                    redacted_text = self.mock_redacted_text
-                else:
-                    # Make the first request to /analyze
-                    analyze_url = f"{self.llm_guard_api_base}analyze/prompt"
-                    verbose_proxy_logger.debug("Making request to: %s", analyze_url)
-                    analyze_payload = {"prompt": text}
-                    redacted_text = None
+            if self.mock_redacted_text is not None:
+                redacted_text = self.mock_redacted_text
+            else:
+                analyze_url = f"{self.llm_guard_api_base}analyze/prompt"
+                verbose_proxy_logger.debug("Making request to: %s", analyze_url)
+                async with aiohttp.ClientSession() as session:
                     async with session.post(
-                        analyze_url, json=analyze_payload
+                        analyze_url, json={"prompt": text}
                     ) as response:
                         redacted_text = await response.json()
-                verbose_proxy_logger.debug(
-                    f"LLM Guard: Received response - {redacted_text}"
+            verbose_proxy_logger.debug(
+                f"LLM Guard: Received response - {redacted_text}"
+            )
+            if redacted_text is None:
+                raise HTTPException(
+                    status_code=500,
+                    detail={
+                        "error": f"Invalid content moderation response: {redacted_text}"
+                    },
                 )
-                if redacted_text is not None:
-                    if (
-                        redacted_text.get("is_valid", None) is not None
-                        and redacted_text["is_valid"] is False
-                    ):
-                        raise HTTPException(
-                            status_code=400,
-                            detail={"error": "Violated content safety policy"},
-                        )
-                    else:
-                        pass
-                else:
-                    raise HTTPException(
-                        status_code=500,
-                        detail={
-                            "error": f"Invalid content moderation response: {redacted_text}"
-                        },
-                    )
+            if redacted_text.get("is_valid", None) is False:
+                raise HTTPException(
+                    status_code=400,
+                    detail={"error": "Violated content safety policy"},
+                )
+            sanitized_prompt = redacted_text.get("sanitized_prompt")
+            return sanitized_prompt if isinstance(sanitized_prompt, str) else text
         except Exception as e:
             verbose_proxy_logger.exception(
                 "litellm.enterprise.enterprise_hooks.llm_guard::moderation_check - Exception occurred - {}".format(
@@ -138,23 +137,75 @@ class _ENTERPRISE_LLMGuard(CustomLogger):
             return
 
         self.print_verbose("Makes LLM Guard Check")
-        try:
-            assert call_type in [
-                "completion",
-                "embeddings",
-                "image_generation",
-                "moderation",
-                "audio_transcription",
-            ]
-        except Exception:
+        if call_type not in [
+            "completion",
+            "embeddings",
+            "image_generation",
+            "moderation",
+            "audio_transcription",
+        ]:
             self.print_verbose(
                 f"Call Type - {call_type}, not in accepted list - ['completion','embeddings','image_generation','moderation','audio_transcription']"
             )
             return data
 
-        formatted_prompt = get_formatted_prompt(data=data, call_type=call_type)  # type: ignore
-        self.print_verbose(f"LLM Guard, formatted_prompt: {formatted_prompt}")
-        return await self.moderation_check(text=formatted_prompt)
+        return await self._moderate_request(data=data)
+
+    async def _moderate_request(self, data: dict) -> dict:
+        """
+        Sanitizes the request in place using the prompt returned by LLM Guard so
+        the provider-bound request carries the redacted content, then returns it.
+        """
+        messages = data.get("messages")
+        if messages is not None:
+            data["messages"] = list(
+                await asyncio.gather(
+                    *(self._moderate_message(message) for message in messages)
+                )
+            )
+            return data
+
+        input_ = data.get("input")
+        if input_ is not None:
+            data["input"] = await self._moderate_input(input_)
+            return data
+
+        prompt = data.get("prompt")
+        if isinstance(prompt, str):
+            data["prompt"] = await self.moderation_check(text=prompt)
+        return data
+
+    async def _moderate_message(self, message: dict) -> dict:
+        content = message.get("content")
+        if isinstance(content, str):
+            return {**message, "content": await self.moderation_check(text=content)}
+        if isinstance(content, list):
+            return {
+                **message,
+                "content": list(
+                    await asyncio.gather(
+                        *(self._moderate_content_part(part) for part in content)
+                    )
+                ),
+            }
+        return message
+
+    async def _moderate_content_part(self, part: dict) -> dict:
+        if part.get("type") == "text" and isinstance(part.get("text"), str):
+            return {**part, "text": await self.moderation_check(text=part["text"])}
+        return part
+
+    async def _moderate_input(self, input_: object) -> object:
+        if isinstance(input_, str):
+            return await self.moderation_check(text=input_)
+        if isinstance(input_, list):
+            return [
+                await self.moderation_check(text=item)
+                if isinstance(item, str)
+                else item
+                for item in input_
+            ]
+        return input_
 
     async def async_post_call_streaming_hook(
         self, user_api_key_dict: UserAPIKeyAuth, response: str

enterprise/litellm_enterprise/enterprise_callbacks/pagerduty/pagerduty.py

@@ -113,6 +113,10 @@ class PagerDutyAlerting(SlackAlerting):
                 user_api_key_spend=_meta.get("user_api_key_spend"),
                 user_api_key_max_budget=_meta.get("user_api_key_max_budget"),
                 user_api_key_budget_reset_at=_meta.get("user_api_key_budget_reset_at"),
+                user_api_key_user_spend=_meta.get("user_api_key_user_spend"),
+                user_api_key_user_max_budget=_meta.get("user_api_key_user_max_budget"),
+                user_api_key_team_spend=_meta.get("user_api_key_team_spend"),
+                user_api_key_team_max_budget=_meta.get("user_api_key_team_max_budget"),
                 user_api_key_org_id=_meta.get("user_api_key_org_id"),
                 user_api_key_org_alias=_meta.get("user_api_key_org_alias"),
                 user_api_key_team_id=_meta.get("user_api_key_team_id"),
@@ -196,6 +200,10 @@ class PagerDutyAlerting(SlackAlerting):
                     if user_api_key_dict.budget_reset_at
                     else None
                 ),
+                user_api_key_user_spend=user_api_key_dict.user_spend,
+                user_api_key_user_max_budget=user_api_key_dict.user_max_budget,
+                user_api_key_team_spend=user_api_key_dict.team_spend,
+                user_api_key_team_max_budget=user_api_key_dict.team_max_budget,
                 user_api_key_org_id=user_api_key_dict.org_id,
                 user_api_key_org_alias=user_api_key_dict.organization_alias,
                 user_api_key_team_id=user_api_key_dict.team_id,

helm/litellm-helm/README.md

@@ -54,6 +54,12 @@ If `db.useStackgresOperator` is used (not yet implemented):
 | `pdb.annotations`           | Extra metadata annotations to add to the PDB                                                                                                                                                                                                  | `{}`                      |
 | `pdb.labels`                | Extra metadata labels to add to the PDB                                                                                                                                                                                                       | `{}`                      |
 
+| `billingMetrics.enabled`    | Enable enterprise billable-request metering. Requires an enterprise license.                                                                                                                                                                  | `false`                   |
+| `billingMetrics.endpoint`   | Collector that the billable-request counter is pushed to.                                                                                                                                                                                    | `https://telemetry.litellm.ai` |
+| `billingMetrics.secretName` | Name of an existing Secret holding the mTLS client certificate, under the keys `tls.crt` and `tls.key`.                                                                                                                                       | `litellm-billing-metrics-mtls` |
+| `billingMetrics.caSecretName` | Name of an existing Secret holding a CA bundle under the key `ca.crt`. Only needed for a private or test collector whose server certificate is not on the public web PKI.                                                                    | `""`                      |
+| `billingMetrics.exportIntervalMs` | How often the counter is pushed, in milliseconds. The proxy defaults to `60000` when unset.                                                                                                                                             | `""`                      |
+
 #### Example `proxy_config` ConfigMap from values (default):
 
 ```
@@ -94,6 +100,21 @@ data:
 type: Opaque
 ```
 
+#### Enterprise billable-request metering
+
+Enterprise licenses meter billable requests by pushing a counter to LiteLLM's collector over mutual TLS. The chart does not create the client certificate; it mounts one you already hold, read-only, so the private key is never exposed through the environment. Create the Secret under the name the chart expects, then turn the block on:
+
+```
+kubectl create secret tls litellm-billing-metrics-mtls --cert=client.crt --key=client.key
+```
+
+```
+billingMetrics:
+  enabled: true
+```
+
+Set `billingMetrics.caSecretName` only when the collector is a private or test one whose server certificate is not on the public web PKI; the production collector needs no CA override. The chart fails the render rather than deploying a proxy that silently never exports, so a missing `secretName` or an emptied `endpoint` surfaces at `helm install` time.
+
 ### Database Settings
 
 | Name                      | Description                                                                                                                                                                                                                                                                               | Value                                                                                      |

helm/litellm-helm/templates/_helpers.tpl

@@ -50,6 +50,53 @@ app.kubernetes.io/name: {{ include "litellm.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
 
+{{/*
+Enterprise billable-request metering. The client certificate identifies the
+deployment to LiteLLM's collector, so it is mounted read-only from an existing
+Secret rather than passed through the environment.
+*/}}
+{{- define "litellm.billingMetrics.certDir" -}}/etc/litellm/billing-mtls{{- end -}}
+{{- define "litellm.billingMetrics.caDir" -}}/etc/litellm/billing-mtls-ca{{- end -}}
+
+{{- define "litellm.billingMetricsEnv" -}}
+- name: LITELLM_BILLING_METRICS_ENDPOINT
+  value: {{ required "billingMetrics.endpoint is required when billingMetrics.enabled is true" .Values.billingMetrics.endpoint | quote }}
+- name: LITELLM_BILLING_METRICS_CLIENT_CERT
+  value: {{ printf "%s/tls.crt" (include "litellm.billingMetrics.certDir" .) | quote }}
+- name: LITELLM_BILLING_METRICS_CLIENT_KEY
+  value: {{ printf "%s/tls.key" (include "litellm.billingMetrics.certDir" .) | quote }}
+{{- if .Values.billingMetrics.caSecretName }}
+- name: LITELLM_BILLING_METRICS_CA_CERT
+  value: {{ printf "%s/ca.crt" (include "litellm.billingMetrics.caDir" .) | quote }}
+{{- end }}
+{{- with .Values.billingMetrics.exportIntervalMs }}
+- name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS
+  value: {{ . | quote }}
+{{- end }}
+{{- end -}}
+
+{{- define "litellm.billingMetricsVolumes" -}}
+- name: billing-metrics-mtls
+  secret:
+    secretName: {{ required "billingMetrics.secretName is required when billingMetrics.enabled is true (an existing Secret with tls.crt and tls.key)" .Values.billingMetrics.secretName }}
+{{- if .Values.billingMetrics.caSecretName }}
+- name: billing-metrics-mtls-ca
+  secret:
+    secretName: {{ .Values.billingMetrics.caSecretName }}
+{{- end }}
+{{- end -}}
+
+{{- define "litellm.billingMetricsVolumeMounts" -}}
+- name: billing-metrics-mtls
+  mountPath: {{ include "litellm.billingMetrics.certDir" . }}
+  readOnly: true
+{{- if .Values.billingMetrics.caSecretName }}
+- name: billing-metrics-mtls-ca
+  mountPath: {{ include "litellm.billingMetrics.caDir" . }}
+  readOnly: true
+{{- end }}
+{{- end -}}
+
 {{/*
 Create the name of the service account to use
 */}}

helm/litellm-helm/templates/deployment.yaml

@@ -142,6 +142,9 @@ spec:
             {{- with .Values.extraEnvVars }}
               {{- toYaml . | nindent 12 }}
             {{- end }}
+            {{- if .Values.billingMetrics.enabled }}
+            {{- include "litellm.billingMetricsEnv" . | nindent 12 }}
+            {{- end }}
             {{- if .Values.migrationJob.enabled }}
             # Schema updates are owned by the dedicated migrations Job; skip
             # the proxy's startup `prisma db push` so N replicas don't race
@@ -220,6 +223,9 @@ spec:
             - name: npm
               mountPath: /.npm
           {{- end }}
+          {{- if .Values.billingMetrics.enabled }}
+            {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }}
+          {{- end }}
           {{- with .Values.volumeMounts }}
             {{- toYaml . | nindent 12 }}
           {{- end }}
@@ -252,6 +258,9 @@ spec:
             items:
               - key: {{ .Values.proxyConfigMap.key | default "config.yaml" }}
                 path: "config.yaml"
+        {{- if .Values.billingMetrics.enabled }}
+        {{- include "litellm.billingMetricsVolumes" . | nindent 8 }}
+        {{- end }}
       {{- with .Values.volumes }}
         {{- toYaml . | nindent 8 }}
       {{- end }}

helm/litellm-helm/tests/billing_metrics_tests.yaml

@@ -0,0 +1,297 @@
+suite: test billingMetrics wiring on the proxy deployment
+templates:
+  - deployment.yaml
+  - configmap-litellm.yaml
+  - migrations-job.yaml
+tests:
+  - it: is off by default, adding no env, volume, or mount
+    template: deployment.yaml
+    asserts:
+      - notContains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls
+            secret:
+              secretName: litellm-billing-metrics-mtls
+      - notContains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: billing-metrics-mtls
+            mountPath: /etc/litellm/billing-mtls
+            readOnly: true
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+            value: https://telemetry.litellm.ai
+
+  - it: renders the endpoint and the mounted cert paths when enabled
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+            value: https://telemetry.litellm.ai
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_CLIENT_CERT
+            value: /etc/litellm/billing-mtls/tls.crt
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_CLIENT_KEY
+            value: /etc/litellm/billing-mtls/tls.key
+
+  # The conventional Secret name is the default, so enabling the block is enough.
+  - it: mounts the default cert secret read-only alongside the config volume
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+    asserts:
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls
+            secret:
+              secretName: litellm-billing-metrics-mtls
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: billing-metrics-mtls
+            mountPath: /etc/litellm/billing-mtls
+            readOnly: true
+
+  - it: honours a secretName override
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        secretName: my-billing-mtls
+    asserts:
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls
+            secret:
+              secretName: my-billing-mtls
+      - notContains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls
+            secret:
+              secretName: litellm-billing-metrics-mtls
+
+  - it: honours an endpoint override
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        endpoint: https://collector.internal:4318
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+            value: https://collector.internal:4318
+
+  # The production collector presents a public web-PKI certificate, so the CA
+  # override must stay absent unless a private collector is configured.
+  - it: omits the CA env, volume, and mount when no caSecretName is set
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+    asserts:
+      - notContains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls-ca
+            secret:
+              secretName: billing-ca
+      - notContains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: billing-metrics-mtls-ca
+            mountPath: /etc/litellm/billing-mtls-ca
+            readOnly: true
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_CA_CERT
+            value: /etc/litellm/billing-mtls-ca/ca.crt
+
+  - it: mounts the CA secret when caSecretName is set
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        caSecretName: billing-ca
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_CA_CERT
+            value: /etc/litellm/billing-mtls-ca/ca.crt
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls-ca
+            secret:
+              secretName: billing-ca
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: billing-metrics-mtls-ca
+            mountPath: /etc/litellm/billing-mtls-ca
+            readOnly: true
+
+  - it: passes the export interval through only when set
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        exportIntervalMs: 5000
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS
+            value: "5000"
+
+  - it: omits the export interval when unset
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+    asserts:
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS
+            value: "60000"
+
+  # Kubernetes resolves duplicate env names last-wins, so the chart-owned billing
+  # entries must render after .Values.envVars or a user could silently redirect
+  # the metering export. The three billing entries are the last ones emitted here
+  # (migrationJob, which appends DISABLE_SCHEMA_UPDATE, is off for this case).
+  - it: renders the billing endpoint after envVars so it cannot be shadowed
+    template: deployment.yaml
+    set:
+      migrationJob:
+        enabled: false
+      billingMetrics:
+        enabled: true
+      envVars:
+        LITELLM_BILLING_METRICS_ENDPOINT: https://shadowed.example
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+            value: https://shadowed.example
+      - equal:
+          path: spec.template.spec.containers[0].env[-3]
+          value:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+            value: https://telemetry.litellm.ai
+      - equal:
+          path: spec.template.spec.containers[0].env[-2].name
+          value: LITELLM_BILLING_METRICS_CLIENT_CERT
+      - equal:
+          path: spec.template.spec.containers[0].env[-1].name
+          value: LITELLM_BILLING_METRICS_CLIENT_KEY
+
+  - it: keeps user-supplied volumes and mounts alongside the billing secret
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+      volumes:
+        - name: custom-callbacks
+          configMap:
+            name: my-callbacks
+      volumeMounts:
+        - name: custom-callbacks
+          mountPath: /app/callbacks
+    asserts:
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: custom-callbacks
+            configMap:
+              name: my-callbacks
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls
+            secret:
+              secretName: litellm-billing-metrics-mtls
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: custom-callbacks
+            mountPath: /app/callbacks
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: billing-metrics-mtls
+            mountPath: /etc/litellm/billing-mtls
+            readOnly: true
+
+  - it: still mounts the proxy config when enabled
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: litellm-config
+            mountPath: /etc/litellm/config.yaml
+            subPath: config.yaml
+
+  # Only the proxy serves billable traffic. The migrations Job must never mount
+  # the client certificate, and it renders its own env and volumes, so nothing
+  # stops a future edit from wiring the billing include into it by mistake.
+  - it: does not touch the migrations job when enabled
+    template: migrations-job.yaml
+    set:
+      billingMetrics:
+        enabled: true
+    asserts:
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+            value: https://telemetry.litellm.ai
+      - notExists:
+          path: spec.template.spec.containers[0].volumeMounts
+      - notExists:
+          path: spec.template.spec.volumes
+
+  - it: fails loudly when enabled with an emptied secretName
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        secretName: ""
+    asserts:
+      - failedTemplate:
+          errorMessage: billingMetrics.secretName is required when billingMetrics.enabled is true (an existing Secret with tls.crt and tls.key)
+
+  - it: fails loudly when enabled without an endpoint
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        endpoint: ""
+    asserts:
+      - failedTemplate:
+          errorMessage: billingMetrics.endpoint is required when billingMetrics.enabled is true

helm/litellm-helm/values.yaml

@@ -139,6 +139,20 @@ masterkeySecretName: ""
 # if set, use this secret key for the master key; otherwise, use the default key
 masterkeySecretKey: ""
 
+# Optional: enterprise billable-request metering. When enabled, the proxy counts
+# successful requests to inference, MCP, and A2A endpoints and pushes them to
+# LiteLLM's collector over mutual TLS. Requires an enterprise license.
+# The client certificate identifies the deployment, so it is mounted read-only
+# from an existing Secret and never passed through the environment.
+billingMetrics:
+  enabled: false
+  endpoint: https://telemetry.litellm.ai   # collector to push the counter to
+  secretName: litellm-billing-metrics-mtls # existing Secret holding tls.crt and tls.key
+  # Only for private or test collectors whose server certificate is not on the
+  # public web PKI. The production collector needs no CA override.
+  caSecretName: ""                         # existing Secret holding ca.crt
+  exportIntervalMs: ""                     # push cadence; the proxy defaults to 60000
+
 proxyConfigMap:
   # when true, creates a new configmap
   create: true

helm/litellm/templates/NOTES.txt

@@ -46,4 +46,9 @@ Reminders:
       - gateway.config.proxy_config                            (rendered into a ConfigMap and mounted at
                                                                 /app/config/config.yaml; gateway reads it via
                                                                 CONFIG_FILE_PATH)
+      - {component}.pdb.{enabled,minAvailable,maxUnavailable}  (per-component PodDisruptionBudget; disabled by
+                                                                default — with hpa.minReplicas of 1, minAvailable: 1
+                                                                would block node drains)
+      - {component}.topologySpreadConstraints                  (standard k8s list, e.g. spread replicas across
+                                                                topology.kubernetes.io/zone)
   - Enable ingress.enabled=true to dispatch / → ui, gateway data-plane prefixes → gateway, and the catch-all → backend.

helm/litellm/templates/_helpers.tpl

@@ -34,6 +34,57 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
 helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }}
 {{- end -}}
 
+{{/*
+Enterprise billable-request metering. Wired into gateway and backend, not the
+migrations job. The gateway serves nearly all billable traffic, but the backend
+keeps the named-server MCP transport (/{mcp_server_name}/mcp), which writes a
+SpendLogs row, so metering only the gateway would silently drop that traffic.
+The client certificate identifies the deployment to LiteLLM's collector, so it is
+mounted read-only from an existing Secret rather than passed through the
+environment.
+*/}}
+{{- define "litellm.billingMetrics.certDir" -}}/etc/litellm/billing-mtls{{- end -}}
+{{- define "litellm.billingMetrics.caDir" -}}/etc/litellm/billing-mtls-ca{{- end -}}
+
+{{- define "litellm.billingMetricsEnv" -}}
+- name: LITELLM_BILLING_METRICS_ENDPOINT
+  value: {{ required "billingMetrics.endpoint is required when billingMetrics.enabled is true" .Values.billingMetrics.endpoint | quote }}
+- name: LITELLM_BILLING_METRICS_CLIENT_CERT
+  value: {{ printf "%s/tls.crt" (include "litellm.billingMetrics.certDir" .) | quote }}
+- name: LITELLM_BILLING_METRICS_CLIENT_KEY
+  value: {{ printf "%s/tls.key" (include "litellm.billingMetrics.certDir" .) | quote }}
+{{- if .Values.billingMetrics.caSecretName }}
+- name: LITELLM_BILLING_METRICS_CA_CERT
+  value: {{ printf "%s/ca.crt" (include "litellm.billingMetrics.caDir" .) | quote }}
+{{- end }}
+{{- with .Values.billingMetrics.exportIntervalMs }}
+- name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS
+  value: {{ . | quote }}
+{{- end }}
+{{- end -}}
+
+{{- define "litellm.billingMetricsVolumes" -}}
+- name: billing-metrics-mtls
+  secret:
+    secretName: {{ required "billingMetrics.secretName is required when billingMetrics.enabled is true (an existing Secret with tls.crt and tls.key)" .Values.billingMetrics.secretName }}
+{{- if .Values.billingMetrics.caSecretName }}
+- name: billing-metrics-mtls-ca
+  secret:
+    secretName: {{ .Values.billingMetrics.caSecretName }}
+{{- end }}
+{{- end -}}
+
+{{- define "litellm.billingMetricsVolumeMounts" -}}
+- name: billing-metrics-mtls
+  mountPath: {{ include "litellm.billingMetrics.certDir" . }}
+  readOnly: true
+{{- if .Values.billingMetrics.caSecretName }}
+- name: billing-metrics-mtls-ca
+  mountPath: {{ include "litellm.billingMetrics.caDir" . }}
+  readOnly: true
+{{- end }}
+{{- end -}}
+
 {{/*
 Per-component selector labels — used in both Service selectors and Deployment matchLabels.
 */}}
@@ -244,6 +295,52 @@ harmless no-op for the Job and authoritative for the app pods.
 {{- end }}
 {{- end -}}
 
+{{/*
+PodDisruptionBudget shared by gateway, backend, and ui.
+
+Invoke with a dict:
+  (dict "root" $ "component" .Values.gateway "componentName" "gateway"
+        "fullname" (include "litellm.gateway.fullname" .)
+        "selectorLabels" (include "litellm.gateway.selectorLabels" .))
+
+Renders nothing unless both the component and its `pdb.enabled` are on.
+Only one of minAvailable / maxUnavailable should be set; if both are,
+minAvailable wins. If neither is set, falls back to `maxUnavailable: 1` so
+an enabled-but-unconfigured PDB still permits node drains.
+
+"Set" means non-nil and non-empty-string, so an explicit 0 (e.g.
+`maxUnavailable: 0` to forbid all voluntary disruptions) is honored rather
+than silently replaced by the fallback.
+*/}}
+{{- define "litellm.pdb" -}}
+{{- $root := .root -}}
+{{- $component := .component -}}
+{{- $min := $component.pdb.minAvailable -}}
+{{- $max := $component.pdb.maxUnavailable -}}
+{{- $minSet := not (or (kindIs "invalid" $min) (eq (printf "%v" $min) "")) -}}
+{{- $maxSet := not (or (kindIs "invalid" $max) (eq (printf "%v" $max) "")) -}}
+{{- if and $component.enabled $component.pdb $component.pdb.enabled }}
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ .fullname }}
+  labels:
+    {{- include "litellm.commonLabels" $root | nindent 4 }}
+    app.kubernetes.io/component: {{ .componentName }}
+spec:
+  selector:
+    matchLabels:
+      {{- .selectorLabels | nindent 6 }}
+  {{- if $minSet }}
+  minAvailable: {{ $min }}
+  {{- else if $maxSet }}
+  maxUnavailable: {{ $max }}
+  {{- else }}
+  maxUnavailable: 1
+  {{- end }}
+{{- end }}
+{{- end -}}
+
 {{/*
 Renders `envFrom:` block for a component's `envConfigMaps` / `envSecrets`
 lists. Each entry is a resource name; the chart wires the whole ConfigMap /

helm/litellm/templates/backend/deployment.yaml

@@ -44,14 +44,20 @@ spec:
             - name: CONFIG_FILE_PATH
               value: /app/config/config.yaml
             {{- end }}
+            {{- if .Values.billingMetrics.enabled }}
+            {{- include "litellm.billingMetricsEnv" . | nindent 12 }}
+            {{- end }}
           {{- include "litellm.envFrom" .Values.backend | nindent 10 }}
-          {{- if or .Values.gateway.config.create .Values.backend.volumeMounts }}
+          {{- if or .Values.gateway.config.create .Values.backend.volumeMounts .Values.billingMetrics.enabled }}
           volumeMounts:
             {{- if .Values.gateway.config.create }}
             - name: gateway-config
               mountPath: /app/config/config.yaml
               subPath: config.yaml
             {{- end }}
+            {{- if .Values.billingMetrics.enabled }}
+            {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }}
+            {{- end }}
             {{- with .Values.backend.volumeMounts }}
             {{- toYaml . | nindent 12 }}
             {{- end }}
@@ -66,13 +72,16 @@ spec:
           {{- end }}
           resources:
             {{- toYaml .Values.backend.resources | nindent 12 }}
-      {{- if or .Values.gateway.config.create .Values.backend.volumes }}
+      {{- if or .Values.gateway.config.create .Values.backend.volumes .Values.billingMetrics.enabled }}
       volumes:
         {{- if .Values.gateway.config.create }}
         - name: gateway-config
           configMap:
             name: {{ include "litellm.gateway.fullname" . }}-config
         {{- end }}
+        {{- if .Values.billingMetrics.enabled }}
+        {{- include "litellm.billingMetricsVolumes" . | nindent 8 }}
+        {{- end }}
         {{- with .Values.backend.volumes }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
@@ -89,4 +98,8 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.backend.topologySpreadConstraints }}
+      topologySpreadConstraints:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
 {{- end }}

helm/litellm/templates/backend/poddisruptionbudget.yaml

@@ -0,0 +1,6 @@
+{{- include "litellm.pdb" (dict
+  "root" $
+  "component" .Values.backend
+  "componentName" "backend"
+  "fullname" (include "litellm.backend.fullname" .)
+  "selectorLabels" (include "litellm.backend.selectorLabels" .)) }}

helm/litellm/templates/gateway/deployment.yaml

@@ -46,14 +46,20 @@ spec:
             - name: NUM_WORKERS
               value: {{ .Values.gateway.numWorkers | quote }}
             {{- end }}
+            {{- if .Values.billingMetrics.enabled }}
+            {{- include "litellm.billingMetricsEnv" . | nindent 12 }}
+            {{- end }}
           {{- include "litellm.envFrom" .Values.gateway | nindent 10 }}
-          {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts }}
+          {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.billingMetrics.enabled }}
           volumeMounts:
             {{- if .Values.gateway.config.create }}
             - name: gateway-config
               mountPath: /app/config/config.yaml
               subPath: config.yaml
             {{- end }}
+            {{- if .Values.billingMetrics.enabled }}
+            {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }}
+            {{- end }}
             {{- with .Values.gateway.volumeMounts }}
             {{- toYaml . | nindent 12 }}
             {{- end }}
@@ -68,13 +74,16 @@ spec:
           {{- end }}
           resources:
             {{- toYaml .Values.gateway.resources | nindent 12 }}
-      {{- if or .Values.gateway.config.create .Values.gateway.volumes }}
+      {{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.billingMetrics.enabled }}
       volumes:
         {{- if .Values.gateway.config.create }}
         - name: gateway-config
           configMap:
             name: {{ include "litellm.gateway.fullname" . }}-config
         {{- end }}
+        {{- if .Values.billingMetrics.enabled }}
+        {{- include "litellm.billingMetricsVolumes" . | nindent 8 }}
+        {{- end }}
         {{- with .Values.gateway.volumes }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
@@ -91,4 +100,8 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.gateway.topologySpreadConstraints }}
+      topologySpreadConstraints:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
 {{- end }}

helm/litellm/templates/gateway/poddisruptionbudget.yaml

@@ -0,0 +1,6 @@
+{{- include "litellm.pdb" (dict
+  "root" $
+  "component" .Values.gateway
+  "componentName" "gateway"
+  "fullname" (include "litellm.gateway.fullname" .)
+  "selectorLabels" (include "litellm.gateway.selectorLabels" .)) }}

helm/litellm/templates/ui/deployment.yaml

@@ -76,4 +76,8 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.ui.topologySpreadConstraints }}
+      topologySpreadConstraints:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
 {{- end }}

helm/litellm/templates/ui/poddisruptionbudget.yaml

@@ -0,0 +1,6 @@
+{{- include "litellm.pdb" (dict
+  "root" $
+  "component" .Values.ui
+  "componentName" "ui"
+  "fullname" (include "litellm.ui.fullname" .)
+  "selectorLabels" (include "litellm.ui.selectorLabels" .)) }}

helm/litellm/tests/billing_metrics_tests.yaml

@@ -0,0 +1,249 @@
+suite: test billingMetrics wiring on gateway and backend
+templates:
+  - gateway/deployment.yaml
+  - gateway/configmap.yaml
+  - backend/deployment.yaml
+  - migrations-job.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: is off by default, adding no env, volume, or mount
+    template: gateway/deployment.yaml
+    asserts:
+      - notContains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls
+            secret:
+              secretName: billing-mtls
+      - equal:
+          path: spec.template.spec.containers[0].volumeMounts
+          value:
+            - name: gateway-config
+              mountPath: /app/config/config.yaml
+              subPath: config.yaml
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+            value: https://telemetry.litellm.ai
+
+  - it: renders the endpoint and the mounted cert paths when enabled
+    template: gateway/deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        secretName: billing-mtls
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+            value: https://telemetry.litellm.ai
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_CLIENT_CERT
+            value: /etc/litellm/billing-mtls/tls.crt
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_CLIENT_KEY
+            value: /etc/litellm/billing-mtls/tls.key
+
+  - it: mounts the cert secret read-only alongside the config volume
+    template: gateway/deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        secretName: billing-mtls
+    asserts:
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls
+            secret:
+              secretName: billing-mtls
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: billing-metrics-mtls
+            mountPath: /etc/litellm/billing-mtls
+            readOnly: true
+
+  # The production collector presents a public web-PKI certificate, so the CA
+  # override must stay absent unless a private collector is configured.
+  - it: omits the CA env, volume, and mount when no caSecretName is set
+    template: gateway/deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        secretName: billing-mtls
+    asserts:
+      - notContains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls-ca
+            secret:
+              secretName: billing-ca
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_CA_CERT
+            value: /etc/litellm/billing-mtls-ca/ca.crt
+
+  - it: mounts the CA secret when caSecretName is set
+    template: gateway/deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        secretName: billing-mtls
+        caSecretName: billing-ca
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_CA_CERT
+            value: /etc/litellm/billing-mtls-ca/ca.crt
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls-ca
+            secret:
+              secretName: billing-ca
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: billing-metrics-mtls-ca
+            mountPath: /etc/litellm/billing-mtls-ca
+            readOnly: true
+
+  - it: passes the export interval through only when set
+    template: gateway/deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        secretName: billing-mtls
+        exportIntervalMs: 5000
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS
+            value: "5000"
+
+  - it: keeps user-supplied gateway volumes alongside the billing secret
+    template: gateway/deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        secretName: billing-mtls
+      gateway.volumes:
+        - name: custom-callbacks
+          configMap:
+            name: my-callbacks
+      gateway.volumeMounts:
+        - name: custom-callbacks
+          mountPath: /app/callbacks
+    asserts:
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: custom-callbacks
+            configMap:
+              name: my-callbacks
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls
+            secret:
+              secretName: billing-mtls
+
+  # The backend keeps the named-server MCP transport (/{mcp_server_name}/mcp),
+  # which writes a SpendLogs row, so it must meter too or that traffic is lost.
+  - it: meters the backend as well, since it serves the MCP transport
+    template: backend/deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        secretName: billing-mtls
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+            value: https://telemetry.litellm.ai
+      - contains:
+          path: spec.template.spec.containers[0].volumeMounts
+          content:
+            name: billing-metrics-mtls
+            mountPath: /etc/litellm/billing-mtls
+            readOnly: true
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls
+            secret:
+              secretName: billing-mtls
+
+  - it: leaves the backend alone when metering is off
+    template: backend/deployment.yaml
+    asserts:
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+            value: https://telemetry.litellm.ai
+
+  # The migrations job runs prisma and serves no traffic; it must never receive
+  # the client key.
+  - it: never mounts the billing cert on the migrations job
+    template: migrations-job.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        secretName: billing-mtls
+    asserts:
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+            value: https://telemetry.litellm.ai
+      - isNull:
+          path: spec.template.spec.volumes
+
+  # The conventional Secret name is the default, so enabling metering needs no
+  # secretName at all; the guard below only fires on an explicitly blanked one.
+  - it: uses the conventional secret name by default
+    template: gateway/deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+    asserts:
+      - contains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics-mtls
+            secret:
+              secretName: litellm-billing-metrics-mtls
+
+  - it: fails loudly when the secretName is explicitly blanked
+    template: gateway/deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        secretName: ""
+    asserts:
+      - failedTemplate:
+          errorMessage: billingMetrics.secretName is required when billingMetrics.enabled is true (an existing Secret with tls.crt and tls.key)
+
+  - it: fails loudly when enabled without an endpoint
+    template: gateway/deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        endpoint: ""
+        secretName: billing-mtls
+    asserts:
+      - failedTemplate:
+          errorMessage: billingMetrics.endpoint is required when billingMetrics.enabled is true

helm/litellm/tests/pdb_topology_spread_tests.yaml

@@ -0,0 +1,188 @@
+suite: test pod disruption budgets and topology spread constraints
+templates:
+  - gateway/poddisruptionbudget.yaml
+  - backend/poddisruptionbudget.yaml
+  - ui/poddisruptionbudget.yaml
+  - gateway/deployment.yaml
+  - gateway/configmap.yaml
+  - backend/deployment.yaml
+  - ui/deployment.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: renders no PDB by default
+    templates:
+      - gateway/poddisruptionbudget.yaml
+      - backend/poddisruptionbudget.yaml
+      - ui/poddisruptionbudget.yaml
+    asserts:
+      - hasDocuments:
+          count: 0
+
+  - it: gateway PDB uses minAvailable and matches the gateway selector labels
+    template: gateway/poddisruptionbudget.yaml
+    set:
+      gateway.pdb.enabled: true
+      gateway.pdb.minAvailable: 1
+    asserts:
+      - isKind:
+          of: PodDisruptionBudget
+      - equal:
+          path: apiVersion
+          value: policy/v1
+      - equal:
+          path: metadata.name
+          value: RELEASE-NAME-litellm-gateway
+      - equal:
+          path: spec.minAvailable
+          value: 1
+      - notExists:
+          path: spec.maxUnavailable
+      - equal:
+          path: spec.selector.matchLabels
+          value:
+            app.kubernetes.io/name: litellm
+            app.kubernetes.io/instance: RELEASE-NAME
+            app.kubernetes.io/component: gateway
+
+  - it: backend PDB uses maxUnavailable when minAvailable is unset
+    template: backend/poddisruptionbudget.yaml
+    set:
+      backend.pdb.enabled: true
+      backend.pdb.maxUnavailable: 25%
+    asserts:
+      - equal:
+          path: spec.maxUnavailable
+          value: 25%
+      - notExists:
+          path: spec.minAvailable
+      - equal:
+          path: spec.selector.matchLabels
+          value:
+            app.kubernetes.io/name: litellm
+            app.kubernetes.io/instance: RELEASE-NAME
+            app.kubernetes.io/component: backend
+
+  - it: minAvailable wins when both minAvailable and maxUnavailable are set
+    template: gateway/poddisruptionbudget.yaml
+    set:
+      gateway.pdb.enabled: true
+      gateway.pdb.minAvailable: 2
+      gateway.pdb.maxUnavailable: 1
+    asserts:
+      - equal:
+          path: spec.minAvailable
+          value: 2
+      - notExists:
+          path: spec.maxUnavailable
+
+  - it: an explicit maxUnavailable 0 is honored instead of the fallback
+    template: backend/poddisruptionbudget.yaml
+    set:
+      backend.pdb.enabled: true
+      backend.pdb.maxUnavailable: 0
+    asserts:
+      - equal:
+          path: spec.maxUnavailable
+          value: 0
+      - notExists:
+          path: spec.minAvailable
+
+  - it: an explicit minAvailable 0 is honored and beats a set maxUnavailable
+    template: gateway/poddisruptionbudget.yaml
+    set:
+      gateway.pdb.enabled: true
+      gateway.pdb.minAvailable: 0
+      gateway.pdb.maxUnavailable: 1
+    asserts:
+      - equal:
+          path: spec.minAvailable
+          value: 0
+      - notExists:
+          path: spec.maxUnavailable
+
+  - it: enabled PDB with neither knob set falls back to maxUnavailable 1
+    template: ui/poddisruptionbudget.yaml
+    set:
+      ui.pdb.enabled: true
+    asserts:
+      - equal:
+          path: spec.maxUnavailable
+          value: 1
+      - notExists:
+          path: spec.minAvailable
+      - equal:
+          path: spec.selector.matchLabels
+          value:
+            app.kubernetes.io/name: litellm
+            app.kubernetes.io/instance: RELEASE-NAME
+            app.kubernetes.io/component: ui
+
+  - it: renders no PDB for a disabled component even when its pdb is enabled
+    template: gateway/poddisruptionbudget.yaml
+    set:
+      gateway.enabled: false
+      gateway.pdb.enabled: true
+    asserts:
+      - hasDocuments:
+          count: 0
+
+  - it: deployments omit topologySpreadConstraints by default
+    templates:
+      - gateway/deployment.yaml
+      - backend/deployment.yaml
+      - ui/deployment.yaml
+    asserts:
+      - notExists:
+          path: spec.template.spec.topologySpreadConstraints
+
+  - it: gateway deployment renders configured topologySpreadConstraints
+    template: gateway/deployment.yaml
+    set:
+      gateway.topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: topology.kubernetes.io/zone
+          whenUnsatisfiable: ScheduleAnyway
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/component: gateway
+    asserts:
+      - equal:
+          path: spec.template.spec.topologySpreadConstraints
+          value:
+            - maxSkew: 1
+              topologyKey: topology.kubernetes.io/zone
+              whenUnsatisfiable: ScheduleAnyway
+              labelSelector:
+                matchLabels:
+                  app.kubernetes.io/component: gateway
+
+  - it: backend deployment renders configured topologySpreadConstraints
+    template: backend/deployment.yaml
+    set:
+      backend.topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: kubernetes.io/hostname
+          whenUnsatisfiable: DoNotSchedule
+          labelSelector:
+            matchLabels:
+              app.kubernetes.io/component: backend
+    asserts:
+      - equal:
+          path: spec.template.spec.topologySpreadConstraints[0].topologyKey
+          value: kubernetes.io/hostname
+      - equal:
+          path: spec.template.spec.topologySpreadConstraints[0].whenUnsatisfiable
+          value: DoNotSchedule
+
+  - it: ui deployment renders configured topologySpreadConstraints
+    template: ui/deployment.yaml
+    set:
+      ui.topologySpreadConstraints:
+        - maxSkew: 1
+          topologyKey: topology.kubernetes.io/zone
+          whenUnsatisfiable: ScheduleAnyway
+    asserts:
+      - equal:
+          path: spec.template.spec.topologySpreadConstraints[0].topologyKey
+          value: topology.kubernetes.io/zone

helm/litellm/values.yaml

@@ -73,6 +73,25 @@ masterKey:
   secretName: litellm-master-key-secret              # name of a Secret containing the master key
   secretKey: master-key
 
+# Optional: enterprise billable-request metering. When enabled, the gateway and
+# backend count successful requests to inference, MCP, and A2A endpoints and push
+# them to LiteLLM's collector over mutual TLS. Both components serve billable
+# routes: the backend keeps the named-server MCP transport. Requires an
+# enterprise license. The client certificate identifies the deployment, so it is
+# mounted read-only from an existing Secret and never passed through the env.
+billingMetrics:
+  enabled: false
+  endpoint: https://telemetry.litellm.ai              # collector to push the counter to
+  # An existing Secret holding the client certificate under tls.crt and its key
+  # under tls.key, usually created from the onboarding artifact. The default is
+  # the conventional name, so the common path is to create that Secret and set
+  # enabled: true. Override only if yours is named differently.
+  secretName: litellm-billing-metrics-mtls
+  # Only for private or test collectors whose server certificate is not on the
+  # public web PKI. The production collector needs no CA override.
+  caSecretName: ""                                    # existing Secret holding ca.crt
+  exportIntervalMs: ""                                # push cadence; the proxy defaults to 60000
+
 # External Postgres connection. 
 database:
   writer:
@@ -171,10 +190,28 @@ gateway:
     maxReplicas: 10
     targetCPUUtilizationPercentage: 70
     targetMemoryUtilizationPercentage: 80
+  # PodDisruptionBudget for the gateway pods. Set exactly one of
+  # `minAvailable` / `maxUnavailable` (minAvailable wins if both are set;
+  # enabling without either falls back to `maxUnavailable: 1`). Disabled by
+  # default: with the default hpa.minReplicas of 1, a `minAvailable: 1` PDB
+  # would block node drains entirely.
+  pdb:
+    enabled: false
+    minAvailable: ""
+    maxUnavailable: ""
   podAnnotations: {}
   nodeSelector: {}
   tolerations: []
   affinity: {}
+  # Standard k8s topologySpreadConstraints for the gateway pods, e.g. to
+  # spread replicas across zones:
+  #   - maxSkew: 1
+  #     topologyKey: topology.kubernetes.io/zone
+  #     whenUnsatisfiable: ScheduleAnyway
+  #     labelSelector:
+  #       matchLabels:
+  #         app.kubernetes.io/component: gateway
+  topologySpreadConstraints: []
 
 # ---------- backend (UI / management API) ----------
 backend:
@@ -214,10 +251,17 @@ backend:
     minReplicas: 1
     maxReplicas: 4
     targetCPUUtilizationPercentage: 70
+  # Same shape as gateway.pdb.
+  pdb:
+    enabled: false
+    minAvailable: ""
+    maxUnavailable: ""
   podAnnotations: {}
   nodeSelector: {}
   tolerations: []
   affinity: {}
+  # Same shape as gateway.topologySpreadConstraints.
+  topologySpreadConstraints: []
 
 # ---------- ui (Next.js static dashboard) ----------
 ui:
@@ -260,7 +304,14 @@ ui:
     minReplicas: 1
     maxReplicas: 3
     targetCPUUtilizationPercentage: 80
+  # Same shape as gateway.pdb.
+  pdb:
+    enabled: false
+    minAvailable: ""
+    maxUnavailable: ""
   podAnnotations: {}
   nodeSelector: {}
   tolerations: []
   affinity: {}
+  # Same shape as gateway.topologySpreadConstraints.
+  topologySpreadConstraints: []

litellm-proxy-extras/litellm_proxy_extras/migrations/20260715000000_add_issuer_to_mcp_server_table/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "issuer" TEXT;

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -325,6 +325,7 @@ model LiteLLM_MCPServerTable {
   command      String?
   args         String[]       @default([])
   env          Json?          @default("{}")
+  issuer             String?
   authorization_url  String?
   token_url          String?
   registration_url   String?

litellm/_redis.py

@@ -688,10 +688,8 @@ def get_redis_connection_pool(
     elif redis_connect_func and hasattr(redis_connect_func, "_gcp_service_account"):
         redis_kwargs["credential_provider"] = GCPIAMCredentialProvider(redis_connect_func._gcp_service_account)
 
-    connection_class = async_redis.Connection
-    if redis_kwargs.pop("ssl", False):
-        connection_class = async_redis.SSLConnection
-        redis_kwargs["connection_class"] = connection_class
+    if redis_kwargs.pop("ssl", None):
+        redis_kwargs["connection_class"] = async_redis.SSLConnection
     return async_redis.BlockingConnectionPool(timeout=REDIS_CONNECTION_POOL_TIMEOUT, **redis_kwargs)
 
 

litellm/caching/caching_handler.py

@@ -85,6 +85,22 @@ class CachingHandlerResponse(BaseModel):
 in_memory_cache_obj = InMemoryCache()
 
 
+def _drop_logging_obj_from_kwargs(request_kwargs: dict[str, object]) -> dict[str, object]:
+    """
+    The caching handler is stored on the Logging object
+    (``logging_obj._llm_caching_handler``), so keeping ``litellm_logging_obj``
+    inside ``request_kwargs`` closes a reference cycle
+    (Logging -> LLMCachingHandler -> kwargs -> Logging) that keeps the full
+    request payload (messages included) alive until a generational GC pass
+    instead of being freed by refcount when the request ends. Nothing in the
+    caching layer reads the logging object from these kwargs; cache-key
+    generation ignores litellm-internal params.
+    """
+    if "litellm_logging_obj" not in request_kwargs:
+        return request_kwargs
+    return {k: v for k, v in request_kwargs.items() if k != "litellm_logging_obj"}
+
+
 def _is_chat_completion_cached_dict(cached_result: dict) -> bool:
     cached_id = cached_result.get("id")
     if isinstance(cached_id, str) and cached_id.startswith("chatcmpl"):
@@ -118,7 +134,7 @@ class LLMCachingHandler:
 
         self.async_streaming_chunks: List[ModelResponse] = []
         self.sync_streaming_chunks: List[ModelResponse] = []
-        self.request_kwargs = request_kwargs
+        self.request_kwargs = _drop_logging_obj_from_kwargs(request_kwargs)
         self.preset_cache_key: Optional[str] = None
         self.original_function = original_function
         self.start_time = start_time
@@ -297,7 +313,7 @@ class LLMCachingHandler:
                 new_kwargs.pop("metadata", None)
             if new_kwargs.get("stream") is True and "cache_key" not in new_kwargs:
                 new_kwargs["cache_key"] = litellm.cache.get_cache_key(**new_kwargs)
-            self.request_kwargs = new_kwargs
+            self.request_kwargs = _drop_logging_obj_from_kwargs(new_kwargs)
             print_verbose("Checking Sync Cache")
             cached_result = litellm.cache.get_cache(**new_kwargs)
             if cached_result is not None:
@@ -693,7 +709,7 @@ class LLMCachingHandler:
             new_kwargs.pop("metadata", None)
         if new_kwargs.get("stream") is True and "cache_key" not in new_kwargs:
             new_kwargs["cache_key"] = litellm.cache.get_cache_key(**new_kwargs)
-        self.request_kwargs = new_kwargs
+        self.request_kwargs = _drop_logging_obj_from_kwargs(new_kwargs)
         cached_result: Optional[Any] = None
         if call_type == CallTypes.aembedding.value:
             if isinstance(new_kwargs["input"], str):

litellm/constants.py

@@ -1496,6 +1496,7 @@ MAX_TEAM_LIST_LIMIT = int(os.getenv("MAX_TEAM_LIST_LIMIT", 20))
 MAX_POLICY_ESTIMATE_IMPACT_ROWS = int(os.getenv("MAX_POLICY_ESTIMATE_IMPACT_ROWS", 1000))
 DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD = float(os.getenv("DEFAULT_PROMPT_INJECTION_SIMILARITY_THRESHOLD", 0.7))
 LENGTH_OF_LITELLM_GENERATED_KEY = int(os.getenv("LENGTH_OF_LITELLM_GENERATED_KEY", 16))
+MINIMUM_CUSTOM_KEY_LENGTH = int(os.getenv("MINIMUM_CUSTOM_KEY_LENGTH", 16))
 SECRET_MANAGER_REFRESH_INTERVAL = int(os.getenv("SECRET_MANAGER_REFRESH_INTERVAL", 86400))
 LITELLM_SETTINGS_SAFE_DB_OVERRIDES = [
     "default_internal_user_params",

litellm/exceptions.py

@@ -966,11 +966,15 @@ class BudgetExceededError(Exception):
         max_budget: float,
         message: Optional[str] = None,
         llm_provider: Optional[str] = None,
+        entity_type: Optional[str] = None,
+        entity_id: Optional[str] = None,
     ):
         self.current_cost = current_cost
         self.max_budget = max_budget
         self.status_code = 429
         self.llm_provider = llm_provider or ""
+        self.entity_type = entity_type
+        self.entity_id = entity_id
         # Surface unified rate-limit fields without joining the RateLimitError
         # hierarchy so existing `except BudgetExceededError:` handlers keep
         # working; custom callbacks reading StandardLoggingPayload pick these

litellm/integrations/langfuse/langfuse_otel.py

@@ -267,29 +267,10 @@ class LangfuseOtelLogger(OpenTelemetry):
             # If no keys, return default from env (likely logging to console or something else)
             return OpenTelemetryConfig.from_env()
 
-        # Determine endpoint - default to US cloud
-        langfuse_host = LangfuseOtelLogger._get_langfuse_otel_host()
-
-        if langfuse_host:
-            # If LANGFUSE_HOST is provided, construct OTEL endpoint from it
-            if not langfuse_host.startswith("http"):
-                langfuse_host = "https://" + langfuse_host
-            endpoint = f"{langfuse_host.rstrip('/')}/api/public/otel"
-            verbose_logger.debug(f"Using Langfuse OTEL endpoint from host: {endpoint}")
-        else:
-            # Default to US cloud endpoint
-            endpoint = LANGFUSE_CLOUD_US_ENDPOINT
-            verbose_logger.debug(f"Using Langfuse US cloud endpoint: {endpoint}")
-
-        auth_header = LangfuseOtelLogger._get_langfuse_authorization_header(
-            public_key=public_key, secret_key=secret_key
-        )
-        otlp_auth_headers = f"Authorization={auth_header}"
-
-        return OpenTelemetryConfig(
-            exporter="otlp_http",
-            endpoint=endpoint,
-            headers=otlp_auth_headers,
+        return LangfuseOtelLogger._build_langfuse_otel_config(
+            public_key=public_key,
+            secret_key=secret_key,
+            langfuse_host=LangfuseOtelLogger._get_langfuse_otel_host(),
         )
 
     @staticmethod
@@ -316,33 +297,36 @@ class LangfuseOtelLogger(OpenTelemetry):
                 "LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY must be set for Langfuse OpenTelemetry integration."
             )
 
-        # Determine endpoint - default to US cloud
-        langfuse_host = LangfuseOtelLogger._get_langfuse_otel_host()
+        return LangfuseOtelLogger._build_langfuse_otel_config(
+            public_key=public_key,
+            secret_key=secret_key,
+            langfuse_host=LangfuseOtelLogger._get_langfuse_otel_host(),
+        )
 
+    @staticmethod
+    def _build_langfuse_otel_config(
+        public_key: str, secret_key: str, langfuse_host: Optional[str]
+    ) -> "OpenTelemetryConfig":
+        """
+        Builds an OTLP HTTP config pointing at the Langfuse OTEL endpoint for the
+        given host (US cloud when no host is provided), authorized with the given keys.
+        """
         if langfuse_host:
-            # If LANGFUSE_HOST is provided, construct OTEL endpoint from it
-            if not langfuse_host.startswith("http"):
-                langfuse_host = "https://" + langfuse_host
-            endpoint = f"{langfuse_host.rstrip('/')}/api/public/otel"
+            normalized_host = langfuse_host if langfuse_host.startswith("http") else f"https://{langfuse_host}"
+            endpoint = f"{normalized_host.rstrip('/')}/api/public/otel"
             verbose_logger.debug(f"Using Langfuse OTEL endpoint from host: {endpoint}")
         else:
-            # Default to US cloud endpoint
             endpoint = LANGFUSE_CLOUD_US_ENDPOINT
             verbose_logger.debug(f"Using Langfuse US cloud endpoint: {endpoint}")
 
         auth_header = LangfuseOtelLogger._get_langfuse_authorization_header(
             public_key=public_key, secret_key=secret_key
         )
-        otlp_auth_headers = f"Authorization={auth_header}"
-
-        # Prevent modification of global env vars which causes leakage
-        # os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = endpoint
-        # os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = otlp_auth_headers
 
         return OpenTelemetryConfig(
             exporter="otlp_http",
             endpoint=endpoint,
-            headers=otlp_auth_headers,
+            headers=f"Authorization={auth_header}",
         )
 
     @staticmethod
@@ -378,6 +362,29 @@ class LangfuseOtelLogger(OpenTelemetry):
 
         return dynamic_headers
 
+    def construct_dynamic_otel_config(
+        self, standard_callback_dynamic_params: StandardCallbackDynamicParams
+    ) -> Optional["OpenTelemetryConfig"]:
+        """
+        Build a full per-request OTLP config from team/key dynamic Langfuse credentials.
+
+        Key-scoped credentials must define the export target, not just the auth
+        headers: without this, a proxy with no global LANGFUSE_* env vars keeps its
+        init-time fallback exporter (console), so key-level langfuse_otel silently
+        never reaches Langfuse.
+        """
+        public_key = standard_callback_dynamic_params.get("langfuse_public_key")
+        secret_key = standard_callback_dynamic_params.get("langfuse_secret_key")
+        if not public_key or not secret_key:
+            return None
+
+        langfuse_host = standard_callback_dynamic_params.get("langfuse_host") or self._get_langfuse_otel_host()
+        return LangfuseOtelLogger._build_langfuse_otel_config(
+            public_key=public_key,
+            secret_key=secret_key,
+            langfuse_host=langfuse_host,
+        )
+
     def create_litellm_proxy_request_started_span(
         self,
         start_time: datetime,

litellm/integrations/opentelemetry.py

@@ -28,6 +28,7 @@ from litellm.integrations.opentelemetry_utils.gen_ai_semconv import (
     parse_semconv_opt_in,
 )
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
+from litellm.litellm_core_utils.secret_redaction import redact_string
 from litellm.secret_managers.main import get_secret_bool, str_to_bool
 from litellm.types.services import ServiceLoggerPayload
 from litellm.types.utils import (
@@ -948,12 +949,22 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
         Returns:
             Tracer: The tracer to use for this request
         """
+        dynamic_config = self._get_dynamic_otel_config_from_kwargs(kwargs)
+        if dynamic_config is not None:
+            verbose_logger.debug(
+                "[OTEL DEBUG] Using DYNAMIC config tracer with endpoint: %s",
+                dynamic_config.endpoint,
+            )
+            return self._get_tracer_with_dynamic_config(dynamic_config)
+
         dynamic_headers = self._get_dynamic_otel_headers_from_kwargs(kwargs)
 
         if dynamic_headers is not None:
             # Create spans using a temporary tracer with dynamic headers
             tracer_to_use = self._get_tracer_with_dynamic_headers(dynamic_headers)
-            verbose_logger.debug("[OTEL DEBUG] Using DYNAMIC tracer with headers: %s", dynamic_headers)
+            verbose_logger.debug(
+                "[OTEL DEBUG] Using DYNAMIC tracer with headers: %s", redact_string(str(dynamic_headers))
+            )
         else:
             # For langfuse_otel without dynamic headers, create a provider with env var credentials
             if hasattr(self, "callback_name") and self.callback_name == "langfuse_otel":
@@ -989,6 +1000,32 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
 
         return dynamic_headers if dynamic_headers else None
 
+    def _get_dynamic_otel_config_from_kwargs(self, kwargs: dict) -> Optional[OpenTelemetryConfig]:
+        """Extract a full dynamic exporter config from kwargs if available."""
+        standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = kwargs.get(
+            "standard_callback_dynamic_params"
+        )
+
+        if not standard_callback_dynamic_params:
+            return None
+
+        return self.construct_dynamic_otel_config(standard_callback_dynamic_params=standard_callback_dynamic_params)
+
+    def _get_tracer_with_dynamic_config(self, dynamic_config: OpenTelemetryConfig):
+        """Create (or reuse) a tracer whose exporter target comes from a per-request config."""
+        from opentelemetry.sdk.trace import TracerProvider
+
+        cache_key = f"dynamic_config:{dynamic_config.exporter}:{dynamic_config.endpoint}:{dynamic_config.headers}"
+        if cache_key in self._tracer_provider_cache:
+            return self._tracer_provider_cache[cache_key].get_tracer(LITELLM_TRACER_NAME)
+
+        temp_provider = TracerProvider(resource=self._get_litellm_resource(self.config))
+        temp_provider.add_span_processor(self._get_span_processor(config_override=dynamic_config))
+
+        self._tracer_provider_cache[cache_key] = temp_provider
+
+        return temp_provider.get_tracer(LITELLM_TRACER_NAME)
+
     def _get_tracer_with_dynamic_headers(self, dynamic_headers: dict):
         """Create a temporary tracer with dynamic headers for this request only."""
         from opentelemetry.sdk.trace import TracerProvider
@@ -1020,6 +1057,19 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
         """
         return None
 
+    def construct_dynamic_otel_config(
+        self, standard_callback_dynamic_params: StandardCallbackDynamicParams
+    ) -> Optional[OpenTelemetryConfig]:
+        """
+        Construct a full exporter config from standard callback dynamic params.
+
+        Override this when team/key dynamic params must control the export
+        target (exporter kind + endpoint), not just the request headers. When
+        this returns a config, it takes precedence over
+        construct_dynamic_otel_headers for the request.
+        """
+        return None
+
     #########################################################
     # End of Team/Key Based Logging Control Flow
     #########################################################
@@ -2747,7 +2797,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
         verbose_logger.debug("OpenTelemetry: No parent context found, creating root span")
         return None, None
 
-    def _get_span_processor(self, dynamic_headers: Optional[dict] = None):
+    def _get_span_processor(
+        self,
+        dynamic_headers: Optional[dict] = None,
+        config_override: Optional[OpenTelemetryConfig] = None,
+    ):
         from opentelemetry.sdk.trace.export import (
             BatchSpanProcessor,
             ConsoleSpanExporter,
@@ -2755,40 +2809,45 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
             SpanExporter,
         )
 
+        otel_exporter = config_override.exporter if config_override else self.OTEL_EXPORTER
+        otel_endpoint = config_override.endpoint if config_override else self.OTEL_ENDPOINT
+        otel_headers = config_override.headers if config_override else self.OTEL_HEADERS
+
         verbose_logger.debug(
-            "OpenTelemetry Logger, initializing span processor \nself.OTEL_EXPORTER: %s\nself.OTEL_ENDPOINT: %s\nself.OTEL_HEADERS: %s",
-            self.OTEL_EXPORTER,
-            self.OTEL_ENDPOINT,
-            self.OTEL_HEADERS,
+            "OpenTelemetry Logger, initializing span processor \nexporter: %s\nendpoint: %s\nheaders: %s",
+            otel_exporter,
+            otel_endpoint,
+            redact_string(str(otel_headers)),
         )
-        _split_otel_headers = OpenTelemetry._get_headers_dictionary(headers=dynamic_headers or self.OTEL_HEADERS)
+        _split_otel_headers = OpenTelemetry._get_headers_dictionary(headers=dynamic_headers or otel_headers)
 
         if dynamic_headers:
             verbose_logger.debug(
                 "[OTEL DEBUG] Creating span processor with DYNAMIC headers: %s",
-                {k: v[:20] + "..." if len(str(v)) > 20 else v for k, v in _split_otel_headers.items()},
+                redact_string(str(_split_otel_headers)),
+            )
+        elif config_override:
+            verbose_logger.debug(
+                "[OTEL DEBUG] Creating span processor with DYNAMIC config, endpoint: %s",
+                otel_endpoint,
             )
         else:
             verbose_logger.debug("[OTEL DEBUG] Creating span processor with GLOBAL headers")
 
-        if hasattr(self.OTEL_EXPORTER, "export"):  # Check if it has the export method that SpanExporter requires
+        if hasattr(otel_exporter, "export"):  # Check if it has the export method that SpanExporter requires
             verbose_logger.debug(
                 "OpenTelemetry: intiializing SpanExporter. Value of OTEL_EXPORTER: %s",
-                self.OTEL_EXPORTER,
+                otel_exporter,
             )
-            return SimpleSpanProcessor(cast(SpanExporter, self.OTEL_EXPORTER))
+            return SimpleSpanProcessor(cast(SpanExporter, otel_exporter))
 
-        if self.OTEL_EXPORTER == "console":
+        if otel_exporter == "console":
             verbose_logger.debug(
                 "OpenTelemetry: intiializing console exporter. Value of OTEL_EXPORTER: %s",
-                self.OTEL_EXPORTER,
+                otel_exporter,
             )
             return BatchSpanProcessor(ConsoleSpanExporter())
-        elif (
-            self.OTEL_EXPORTER == "otlp_http"
-            or self.OTEL_EXPORTER == "http/protobuf"
-            or self.OTEL_EXPORTER == "http/json"
-        ):
+        elif otel_exporter == "otlp_http" or otel_exporter == "http/protobuf" or otel_exporter == "http/json":
             try:
                 from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
                     OTLPSpanExporter as OTLPSpanExporterHTTP,
@@ -2801,13 +2860,13 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
 
             verbose_logger.debug(
                 "OpenTelemetry: intiializing http exporter. Value of OTEL_EXPORTER: %s",
-                self.OTEL_EXPORTER,
+                otel_exporter,
             )
-            normalized_endpoint = self._normalize_otel_endpoint(self.OTEL_ENDPOINT, "traces")
+            normalized_endpoint = self._normalize_otel_endpoint(otel_endpoint, "traces")
             return BatchSpanProcessor(
                 OTLPSpanExporterHTTP(endpoint=normalized_endpoint, headers=_split_otel_headers),
             )
-        elif self.OTEL_EXPORTER == "otlp_grpc" or self.OTEL_EXPORTER == "grpc":
+        elif otel_exporter == "otlp_grpc" or otel_exporter == "grpc":
             try:
                 from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
                     OTLPSpanExporter as OTLPSpanExporterGRPC,
@@ -2820,16 +2879,16 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
 
             verbose_logger.debug(
                 "OpenTelemetry: intiializing grpc exporter. Value of OTEL_EXPORTER: %s",
-                self.OTEL_EXPORTER,
+                otel_exporter,
             )
-            normalized_endpoint = self._normalize_otel_endpoint(self.OTEL_ENDPOINT, "traces")
+            normalized_endpoint = self._normalize_otel_endpoint(otel_endpoint, "traces")
             return BatchSpanProcessor(
                 OTLPSpanExporterGRPC(endpoint=normalized_endpoint, headers=_split_otel_headers),
             )
         else:
             verbose_logger.debug(
                 "OpenTelemetry: intiializing console exporter. Value of OTEL_EXPORTER: %s",
-                self.OTEL_EXPORTER,
+                otel_exporter,
             )
             return BatchSpanProcessor(ConsoleSpanExporter())
 
@@ -2841,7 +2900,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
             "OpenTelemetry Logger, initializing log exporter \nself.OTEL_EXPORTER: %s\nself.OTEL_ENDPOINT: %s\nself.OTEL_HEADERS: %s",
             self.OTEL_EXPORTER,
             self.OTEL_ENDPOINT,
-            self.OTEL_HEADERS,
+            redact_string(str(self.OTEL_HEADERS)),
         )
 
         _split_otel_headers = OpenTelemetry._get_headers_dictionary(self.OTEL_HEADERS)
@@ -2928,7 +2987,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
             "OpenTelemetry Logger, initializing metric reader\nself.OTEL_EXPORTER: %s\nself.OTEL_ENDPOINT: %s\nself.OTEL_HEADERS: %s",
             self.OTEL_EXPORTER,
             self.OTEL_ENDPOINT,
-            self.OTEL_HEADERS,
+            redact_string(str(self.OTEL_HEADERS)),
         )
 
         _split_otel_headers = OpenTelemetry._get_headers_dictionary(self.OTEL_HEADERS)

litellm/litellm_core_utils/dd_tracing.py

@@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, Any, Optional, Union
 from litellm.secret_managers.main import get_secret_bool
 
 if TYPE_CHECKING:
-    from ddtrace.tracer import Tracer as DD_TRACER
+    from ddtrace.trace import Tracer as DD_TRACER
 else:
     DD_TRACER = Any
 

litellm/litellm_core_utils/litellm_logging.py

@@ -38,6 +38,7 @@ from litellm import (
 )
 from litellm._logging import _is_debugging_on, _redact_string, verbose_logger
 from litellm.exceptions import (
+    BudgetExceededError,
     validate_rate_limit_category,
     validate_rate_limit_type,
 )
@@ -925,7 +926,6 @@ class Logging(LiteLLMLoggingBaseClass):
 
     def pre_call(self, input, api_key, model=None, additional_args={}):
         # Log the exact input to the LLM API
-        litellm.error_logs["PRE_CALL"] = locals()
         try:
             self._pre_call(
                 input=input,
@@ -1135,7 +1135,6 @@ class Logging(LiteLLMLoggingBaseClass):
 
     def post_call(self, original_response, input=None, api_key=None, additional_args={}):
         # Log the exact result from the LLM API, for streaming - log the type of response received
-        litellm.error_logs["POST_CALL"] = locals()
         if isinstance(original_response, dict):
             original_response = json.dumps(original_response, default=str)
         try:
@@ -3074,7 +3073,7 @@ class Logging(LiteLLMLoggingBaseClass):
     def get_combined_callback_list(self, dynamic_success_callbacks: Optional[List], global_callbacks: List) -> List:
         if dynamic_success_callbacks is None:
             return list(global_callbacks)
-        return list(set(dynamic_success_callbacks + global_callbacks))
+        return list(dict.fromkeys(dynamic_success_callbacks + global_callbacks))
 
     def _remove_internal_litellm_callbacks(self, callbacks: List) -> List:
         """
@@ -4599,6 +4598,10 @@ class StandardLoggingPayloadSetup:
             user_api_key_spend=None,
             user_api_key_max_budget=None,
             user_api_key_budget_reset_at=None,
+            user_api_key_user_spend=None,
+            user_api_key_user_max_budget=None,
+            user_api_key_team_spend=None,
+            user_api_key_team_max_budget=None,
             user_api_key_team_id=None,
             user_api_key_org_id=None,
             user_api_key_org_alias=None,
@@ -4945,6 +4948,7 @@ class StandardLoggingPayloadSetup:
 
         rate_limit_category = validate_rate_limit_category(getattr(original_exception, "category", None))
         rate_limit_type = validate_rate_limit_type(getattr(original_exception, "rate_limit_type", None))
+        budget_error = original_exception if isinstance(original_exception, BudgetExceededError) else None
 
         return StandardLoggingPayloadErrorInformation(
             error_code=error_status,
@@ -4954,6 +4958,10 @@ class StandardLoggingPayloadSetup:
             error_message=error_message,
             error_rate_limit_category=rate_limit_category,
             error_rate_limit_type=rate_limit_type,
+            error_budget_entity_type=budget_error.entity_type if budget_error else None,
+            error_budget_entity_id=budget_error.entity_id if budget_error else None,
+            error_budget_limit=budget_error.max_budget if budget_error else None,
+            error_budget_spend=budget_error.current_cost if budget_error else None,
         )
 
     @staticmethod
@@ -5430,6 +5438,10 @@ def get_standard_logging_metadata(
         user_api_key_spend=None,
         user_api_key_max_budget=None,
         user_api_key_budget_reset_at=None,
+        user_api_key_user_spend=None,
+        user_api_key_user_max_budget=None,
+        user_api_key_team_spend=None,
+        user_api_key_team_max_budget=None,
         user_api_key_team_id=None,
         user_api_key_org_id=None,
         user_api_key_org_alias=None,
@@ -5529,6 +5541,10 @@ def create_dummy_standard_logging_payload() -> StandardLoggingPayload:
         user_api_key_team_id=str("test_team"),
         user_api_key_user_id=str("test_user"),
         user_api_key_team_alias=str("test_team_alias"),
+        user_api_key_user_spend=None,
+        user_api_key_user_max_budget=None,
+        user_api_key_team_spend=None,
+        user_api_key_team_max_budget=None,
         user_api_key_org_id=None,
         spend_logs_metadata=None,
         requester_ip_address=str("127.0.0.1"),

litellm/litellm_core_utils/redact_messages.py

@@ -77,6 +77,22 @@ def _redact_streaming_response(streaming_response):
             streaming_response.reasoning = None
 
 
+def _redact_tool_calls(tool_calls) -> None:
+    """Redact tool call arguments (assistant tool calls carry prompt-derived data)."""
+    if not tool_calls:
+        return
+    for tool_call in tool_calls:
+        function = getattr(tool_call, "function", None)
+        if function is not None and hasattr(function, "arguments"):
+            function.arguments = "redacted-by-litellm"
+
+
+def _redact_function_call(function_call) -> None:
+    """Redact legacy assistant function_call arguments."""
+    if function_call is not None and hasattr(function_call, "arguments"):
+        function_call.arguments = "redacted-by-litellm"
+
+
 def _redact_choice_content(choice):
     """Helper to redact content in a choice (message or delta)."""
     if isinstance(choice, litellm.Choices):
@@ -85,12 +101,16 @@ def _redact_choice_content(choice):
             choice.message.reasoning_content = "redacted-by-litellm"
         if hasattr(choice.message, "thinking_blocks"):
             choice.message.thinking_blocks = None
+        _redact_tool_calls(getattr(choice.message, "tool_calls", None))
+        _redact_function_call(getattr(choice.message, "function_call", None))
     elif isinstance(choice, litellm.utils.StreamingChoices):
         choice.delta.content = "redacted-by-litellm"
         if hasattr(choice.delta, "reasoning_content"):
             choice.delta.reasoning_content = "redacted-by-litellm"
         if hasattr(choice.delta, "thinking_blocks"):
             choice.delta.thinking_blocks = None
+        _redact_tool_calls(getattr(choice.delta, "tool_calls", None))
+        _redact_function_call(getattr(choice.delta, "function_call", None))
 
 
 def _redact_responses_api_output(output_items):
@@ -111,6 +131,9 @@ def _redact_responses_api_output(output_items):
                     if hasattr(summary_item, "text"):
                         summary_item.text = "redacted-by-litellm"
 
+        if hasattr(output_item, "type") and output_item.type == "function_call" and hasattr(output_item, "arguments"):
+            output_item.arguments = "redacted-by-litellm"
+
 
 def _redact_responses_api_output_dict(output_items, redacted_str: str):
     """Helper to redact ResponsesAPIResponse output items in dict form."""
@@ -131,6 +154,9 @@ def _redact_responses_api_output_dict(output_items, redacted_str: str):
                 if isinstance(summary_item, dict) and "text" in summary_item:
                     summary_item["text"] = redacted_str
 
+        if output_item.get("type") == "function_call" and "arguments" in output_item:
+            output_item["arguments"] = redacted_str
+
 
 def _redact_standard_logging_object(model_call_details: dict):
     """Redact messages and response inside standard_logging_object if present."""
@@ -162,6 +188,19 @@ def _redact_standard_logging_object(model_call_details: dict):
             standard_logging_object["response"] = {"text": redacted_str}
 
 
+def _redact_tool_calls_dict(message: dict, redacted_str: str) -> None:
+    """Redact tool call / function_call arguments in a dict-form message or delta."""
+    tool_calls = message.get("tool_calls")
+    if isinstance(tool_calls, list):
+        for tool_call in tool_calls:
+            if isinstance(tool_call, dict) and isinstance(tool_call.get("function"), dict):
+                tool_call["function"]["arguments"] = redacted_str
+
+    function_call = message.get("function_call")
+    if isinstance(function_call, dict) and "arguments" in function_call:
+        function_call["arguments"] = redacted_str
+
+
 def _redact_model_response_dict_choices(choices, redacted_str: str):
     for choice in choices:
         if isinstance(choice, dict):
@@ -173,6 +212,7 @@ def _redact_model_response_dict_choices(choices, redacted_str: str):
                     choice["message"]["thinking_blocks"] = None
                 if "audio" in choice["message"]:
                     choice["message"]["audio"] = None
+                _redact_tool_calls_dict(choice["message"], redacted_str)
             elif "delta" in choice and isinstance(choice["delta"], dict):
                 choice["delta"]["content"] = redacted_str
                 if "reasoning_content" in choice["delta"]:
@@ -181,6 +221,7 @@ def _redact_model_response_dict_choices(choices, redacted_str: str):
                     choice["delta"]["thinking_blocks"] = None
                 if "audio" in choice["delta"]:
                     choice["delta"]["audio"] = None
+                _redact_tool_calls_dict(choice["delta"], redacted_str)
         else:
             _redact_choice_content(choice)
 

litellm/litellm_core_utils/secret_redaction.py

@@ -9,6 +9,8 @@ secrets from strings without depending on the logging-configuration module.
 import re
 from typing import List
 
+from litellm.constants import MINIMUM_CUSTOM_KEY_LENGTH
+
 _REDACTED = "REDACTED"
 
 
@@ -30,7 +32,7 @@ def _build_secret_patterns() -> "re.Pattern[str]":
         # Basic auth headers
         r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}",
         # OpenAI / Anthropic sk- prefixed keys
-        r"sk-[A-Za-z0-9\-_]{20,}",
+        rf"sk-[A-Za-z0-9\-_]{{{MINIMUM_CUSTOM_KEY_LENGTH - len('sk-')},}}",
         # Generic api_key / api-key / apikey (handles 'key': 'value' dict repr)
         r"(?:api[_-]?key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]{8,}",
         # x-api-key / api-key header values (handles 'key': 'value' dict repr)

litellm/litellm_core_utils/streaming_chunk_builder_utils.py

@@ -467,6 +467,7 @@ class ChunkProcessor:
         cache_read_input_tokens: Optional[int] = None
         completion_tokens_details: Optional[CompletionTokensDetails] = None
         prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None
+        cost: Optional[float] = None
 
         if "prompt_tokens" in usage_chunk:
             prompt_tokens = usage_chunk.get("prompt_tokens", 0) or 0
@@ -476,6 +477,8 @@ class ChunkProcessor:
             cache_creation_input_tokens = usage_chunk.get("cache_creation_input_tokens")
         if "cache_read_input_tokens" in usage_chunk:
             cache_read_input_tokens = usage_chunk.get("cache_read_input_tokens")
+        if "cost" in usage_chunk:
+            cost = usage_chunk.get("cost")
         if hasattr(usage_chunk, "completion_tokens_details"):
             if isinstance(usage_chunk.completion_tokens_details, dict):
                 completion_tokens_details = CompletionTokensDetails(**usage_chunk.completion_tokens_details)
@@ -494,6 +497,7 @@ class ChunkProcessor:
             "cache_read_input_tokens": cache_read_input_tokens,
             "completion_tokens_details": completion_tokens_details,
             "prompt_tokens_details": prompt_tokens_details,
+            "cost": cost,
         }
 
     def count_reasoning_tokens(self, response: ModelResponse) -> Optional[int]:
@@ -512,6 +516,22 @@ class ChunkProcessor:
 
         return reasoning_tokens
 
+    @staticmethod
+    def _extract_usage_chunk(chunk: dict[str, Any] | ModelResponse | ModelResponseStream) -> Usage | None:
+        usage_chunk: Usage | dict[str, Any] | None = None
+        if hasattr(chunk, "usage") and chunk.usage is not None:
+            usage_chunk = chunk.usage
+        elif "usage" in chunk:
+            usage_chunk = chunk["usage"]
+        elif (isinstance(chunk, ModelResponse) or isinstance(chunk, ModelResponseStream)) and hasattr(
+            chunk, "_hidden_params"
+        ):
+            usage_chunk = chunk._hidden_params.get("usage", None)
+
+        if isinstance(usage_chunk, dict):
+            return Usage(**usage_chunk)
+        return usage_chunk
+
     def _calculate_usage_per_chunk(
         self,
         chunks: List[Union[Dict[str, Any], ModelResponse]],
@@ -548,18 +568,12 @@ class ChunkProcessor:
         # is last-wins, so without preserving this separately the 1h breakdown is
         # lost and 1h cache writes get billed at the 5m rate.
         cache_creation_token_details: Optional[CacheCreationTokenDetails] = None
+        cost: Optional[float] = None
+
         for chunk in chunks:
-            usage_chunk: Optional[Usage] = None
-            if "usage" in chunk:
-                usage_chunk = chunk["usage"]
-            elif (isinstance(chunk, ModelResponse) or isinstance(chunk, ModelResponseStream)) and hasattr(
-                chunk, "_hidden_params"
-            ):
-                usage_chunk = chunk._hidden_params.get("usage", None)
+            usage_chunk = self._extract_usage_chunk(chunk)
 
             if usage_chunk is not None:
-                if isinstance(usage_chunk, dict):
-                    usage_chunk = Usage(**usage_chunk)
                 usage_chunk_dict = self._usage_chunk_calculation_helper(usage_chunk)
                 if usage_chunk_dict["prompt_tokens"] is not None and usage_chunk_dict["prompt_tokens"] > 0:
                     prompt_tokens = usage_chunk_dict["prompt_tokens"]
@@ -610,6 +624,9 @@ class ChunkProcessor:
                     prompt_tokens_details, cache_creation_token_details
                 )
 
+                if usage_chunk_dict["cost"] is not None:
+                    cost = usage_chunk_dict["cost"]
+
         prompt_tokens_details = self._attach_cache_creation_token_details(
             prompt_tokens_details, cache_creation_token_details
         )
@@ -629,6 +646,7 @@ class ChunkProcessor:
             web_search_requests=web_search_requests,
             completion_tokens_details=completion_tokens_details,
             prompt_tokens_details=prompt_tokens_details,
+            cost=cost,
         )
 
     @staticmethod
@@ -727,6 +745,7 @@ class ChunkProcessor:
         prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = calculated_usage_per_chunk[
             "prompt_tokens_details"
         ]
+        cost: Optional[float] = calculated_usage_per_chunk["cost"]
 
         try:
             returned_usage.prompt_tokens = prompt_tokens or token_counter(model=model, messages=messages)
@@ -784,6 +803,9 @@ class ChunkProcessor:
             else:
                 returned_usage.prompt_tokens_details.web_search_requests = web_search_requests
 
+        if cost is not None:
+            setattr(returned_usage, "cost", cost)
+
         # Return a new usage object with the new values
 
         returned_usage = Usage(**returned_usage.model_dump())

litellm/litellm_core_utils/streaming_handler.py

@@ -962,10 +962,11 @@ class CustomStreamWrapper:
                 if self.custom_llm_provider == "bedrock" and "trace" in model_response:
                     return model_response
 
-                # Default - return StopIteration
-                if hasattr(model_response, "usage"):
-                    self.chunks.append(model_response)
-                raise StopIteration
+                # Don't raise StopIteration here - some providers (like OpenRouter)
+                # send usage/cost data in chunks after the finish_reason chunk
+                if hasattr(model_response, "usage") and model_response.usage is not None:
+                    return model_response
+                return
             # flush any remaining holding chunk
             if len(self.holding_chunk) > 0:
                 if model_response.choices[0].delta.content is None:
@@ -1474,12 +1475,16 @@ class CustomStreamWrapper:
 
                 self.tool_call = True
 
+            if hasattr(chunk, "usage") and chunk.usage is not None:
+                model_response.usage = chunk.usage
+
             ## RETURN ARG
-            return self.return_processed_chunk_logic(
+            result = self.return_processed_chunk_logic(
                 completion_obj=completion_obj,
                 model_response=model_response,  # type: ignore
                 response_obj=response_obj,
             )
+            return result
 
         except StopIteration:
             raise StopIteration
@@ -1686,6 +1691,21 @@ class CustomStreamWrapper:
             model_response.choices[0].finish_reason = "tool_calls"
         return model_response
 
+    @staticmethod
+    def _propagate_usage_cost_to_hidden_params(
+        response: "ModelResponse",
+    ) -> None:
+        """
+        If the assembled response carries a provider-reported cost on
+        usage.cost, copy it into _hidden_params so litellm's cost
+        calculator uses it instead of a token-based estimate.
+        """
+        _usage = getattr(response, "usage", None)
+        if _usage is not None and hasattr(_usage, "cost") and _usage.cost is not None:
+            if "additional_headers" not in response._hidden_params:
+                response._hidden_params["additional_headers"] = {}
+            response._hidden_params["additional_headers"]["llm_provider-x-litellm-response-cost"] = float(_usage.cost)
+
     def __next__(self) -> "ModelResponseStream":
         cache_hit = False
         if self.custom_llm_provider is not None and self.custom_llm_provider == "cached_response":
@@ -1741,6 +1761,10 @@ class CustomStreamWrapper:
                     # hasattr(response, "usage") is always True — must check
                     # `is not None` to avoid running this path on every chunk.
                     if getattr(response, "usage", None) is not None:
+                        usage_to_preserve = response.usage
+                        if usage_to_preserve:
+                            response._hidden_params["usage"] = usage_to_preserve
+
                         obj_dict = response.model_dump()
 
                         if "usage" in obj_dict:
@@ -1789,6 +1813,8 @@ class CustomStreamWrapper:
 
                 response = self.model_response_creator()
                 if complete_streaming_response is not None:
+                    self._propagate_usage_cost_to_hidden_params(complete_streaming_response)
+
                     setattr(
                         response,
                         "usage",
@@ -1974,97 +2000,7 @@ class CustomStreamWrapper:
                         self.chunks.append(processed_chunk)
                         return processed_chunk
         except (StopAsyncIteration, StopIteration):
-            if self.sent_last_chunk is True:
-                # log the final chunk with accurate streaming values
-                try:
-                    complete_streaming_response = litellm.stream_chunk_builder(
-                        chunks=self.chunks,
-                        messages=self.messages,
-                        logging_obj=self.logging_obj,
-                    )
-                except Exception as e:
-                    # see sync __next__: a raise from stream_chunk_builder inside this
-                    # except handler escapes __anext__ and drops the request from SpendLogs.
-                    # Recover best-effort usage from the raw chunks so cost is still tracked
-                    verbose_logger.warning(
-                        "stream_chunk_builder raised at end-of-stream (%s); logging best-effort usage from chunks.",
-                        str(e),
-                    )
-                    try:
-                        complete_streaming_response = self.model_response_creator(
-                            chunk={"usage": calculate_total_usage(chunks=self.chunks)}
-                        )
-                    except Exception:
-                        complete_streaming_response = None
-
-                response = self.model_response_creator()
-                if complete_streaming_response is not None:
-                    setattr(
-                        response,
-                        "usage",
-                        getattr(complete_streaming_response, "usage"),
-                    )
-                    try:
-                        _copy = complete_streaming_response.model_copy(deep=True)
-                    except RuntimeError:
-                        _copy = complete_streaming_response.model_copy()
-                    asyncio.create_task(
-                        self.async_cache_streaming_response(
-                            processed_chunk=_copy,
-                            cache_hit=cache_hit,
-                        )
-                    )
-                # Update hidden_params with final usage from
-                # stream_chunk_builder (see sync __next__ for full comment).
-                if (
-                    self.stream_options is None
-                    and complete_streaming_response is not None
-                    and self._last_returned_hidden_params is not None
-                ):
-                    final_usage = getattr(complete_streaming_response, "usage", None)
-                    if final_usage is not None:
-                        self._last_returned_hidden_params["usage"] = final_usage
-
-                if self.sent_stream_usage is False and self.send_stream_usage is True:
-                    self.sent_stream_usage = True
-                    return response
-
-                _deferred_cb = getattr(
-                    self.logging_obj,
-                    "_on_deferred_stream_complete",
-                    None,
-                )
-                if _deferred_cb is not None:
-                    # Proxy has post-call guardrails. Store the assembled
-                    # response so the outer streaming consumer
-                    # (ProxyLogging.async_post_call_streaming_iterator_hook)
-                    # can fire the deferred callback AFTER all guardrail
-                    # end-of-stream blocks complete.  Scheduling here via
-                    # create_task would race with unified_guardrail's
-                    # end-of-stream block for short-stream providers.
-                    self.logging_obj._deferred_stream_complete_args = (  # type: ignore[attr-defined]
-                        complete_streaming_response,
-                        cache_hit,
-                    )
-                else:
-                    # prefer_async_handlers routes CustomLogger to async_success_handler
-                    # when consumers use ``async for`` on sync-SDK streams. Legacy string
-                    # callbacks still run via executor.submit inside dispatch_success_handlers.
-                    asyncio.create_task(
-                        self.logging_obj.dispatch_success_handlers(
-                            complete_streaming_response,
-                            cache_hit=cache_hit,
-                            start_time=None,
-                            end_time=None,
-                            prefer_async_handlers=True,
-                        )
-                    )
-
-                raise StopAsyncIteration  # Re-raise StopIteration
-            else:
-                self.sent_last_chunk = True
-                processed_chunk = self.finish_reason_handler()
-                return processed_chunk
+            return await self._finalize_completed_stream(cache_hit=cache_hit)
         except httpx.TimeoutException as e:  # if httpx read timeout error occues
             traceback_exception = traceback.format_exc()
             ## ADD DEBUG INFORMATION - E.G. LITELLM REQUEST TIMEOUT
@@ -2079,20 +2015,122 @@ class CustomStreamWrapper:
                 # Handle any exceptions that might occur during streaming
                 asyncio.create_task(self.logging_obj.async_failure_handler(e, traceback_exception))
             self._handle_stream_fallback_error(e)
+        except (httpx.ReadError, httpx.RemoteProtocolError) as e:
+            if self.received_finish_reason is None:
+                self._log_stream_failure_and_raise(e)
+            return await self._finalize_completed_stream(cache_hit=cache_hit)
         except Exception as e:
-            traceback_exception = traceback.format_exc()
-            if self.logging_obj is not None:
-                self._record_partial_usage_for_failure()
-                ## LOGGING
-                threading.Thread(
-                    target=self.logging_obj.failure_handler,
-                    args=(e, traceback_exception),
-                ).start()  # log response
-                # Handle any exceptions that might occur during streaming
-                asyncio.create_task(
-                    self.logging_obj.async_failure_handler(e, traceback_exception)  # type: ignore
+            self._log_stream_failure_and_raise(e)
+
+    async def _finalize_completed_stream(self, cache_hit: bool) -> "ModelResponseStream":
+        if self.sent_last_chunk is True:
+            # log the final chunk with accurate streaming values
+            try:
+                complete_streaming_response = litellm.stream_chunk_builder(
+                    chunks=self.chunks,
+                    messages=self.messages,
+                    logging_obj=self.logging_obj,
                 )
-            self._handle_stream_fallback_error(e)
+            except Exception as e:
+                # see sync __next__: a raise from stream_chunk_builder inside this
+                # except handler escapes __anext__ and drops the request from SpendLogs.
+                # Recover best-effort usage from the raw chunks so cost is still tracked
+                verbose_logger.warning(
+                    "stream_chunk_builder raised at end-of-stream (%s); logging best-effort usage from chunks.",
+                    str(e),
+                )
+                try:
+                    complete_streaming_response = self.model_response_creator(
+                        chunk={"usage": calculate_total_usage(chunks=self.chunks)}
+                    )
+                except Exception:
+                    complete_streaming_response = None
+
+            response = self.model_response_creator()
+            if complete_streaming_response is not None:
+                self._propagate_usage_cost_to_hidden_params(complete_streaming_response)
+
+                setattr(
+                    response,
+                    "usage",
+                    getattr(complete_streaming_response, "usage"),
+                )
+                try:
+                    _copy = complete_streaming_response.model_copy(deep=True)
+                except RuntimeError:
+                    _copy = complete_streaming_response.model_copy()
+                asyncio.create_task(
+                    self.async_cache_streaming_response(
+                        processed_chunk=_copy,
+                        cache_hit=cache_hit,
+                    )
+                )
+            # Update hidden_params with final usage from
+            # stream_chunk_builder (see sync __next__ for full comment).
+            if (
+                self.stream_options is None
+                and complete_streaming_response is not None
+                and self._last_returned_hidden_params is not None
+            ):
+                final_usage = getattr(complete_streaming_response, "usage", None)
+                if final_usage is not None:
+                    self._last_returned_hidden_params["usage"] = final_usage
+
+            if self.sent_stream_usage is False and self.send_stream_usage is True:
+                self.sent_stream_usage = True
+                return response
+
+            _deferred_cb = getattr(
+                self.logging_obj,
+                "_on_deferred_stream_complete",
+                None,
+            )
+            if _deferred_cb is not None:
+                # Proxy has post-call guardrails. Store the assembled
+                # response so the outer streaming consumer
+                # (ProxyLogging.async_post_call_streaming_iterator_hook)
+                # can fire the deferred callback AFTER all guardrail
+                # end-of-stream blocks complete.  Scheduling here via
+                # create_task would race with unified_guardrail's
+                # end-of-stream block for short-stream providers.
+                self.logging_obj._deferred_stream_complete_args = (  # type: ignore[attr-defined]
+                    complete_streaming_response,
+                    cache_hit,
+                )
+            else:
+                # prefer_async_handlers routes CustomLogger to async_success_handler
+                # when consumers use ``async for`` on sync-SDK streams. Legacy string
+                # callbacks still run via executor.submit inside dispatch_success_handlers.
+                asyncio.create_task(
+                    self.logging_obj.dispatch_success_handlers(
+                        complete_streaming_response,
+                        cache_hit=cache_hit,
+                        start_time=None,
+                        end_time=None,
+                        prefer_async_handlers=True,
+                    )
+                )
+
+            raise StopAsyncIteration  # Re-raise StopIteration
+        else:
+            self.sent_last_chunk = True
+            processed_chunk = self.finish_reason_handler()
+            return processed_chunk
+
+    def _log_stream_failure_and_raise(self, e: Exception) -> NoReturn:
+        traceback_exception = traceback.format_exc()
+        if self.logging_obj is not None:
+            self._record_partial_usage_for_failure()
+            ## LOGGING
+            threading.Thread(
+                target=self.logging_obj.failure_handler,
+                args=(e, traceback_exception),
+            ).start()  # log response
+            # Handle any exceptions that might occur during streaming
+            asyncio.create_task(
+                self.logging_obj.async_failure_handler(e, traceback_exception)  # type: ignore
+            )
+        self._handle_stream_fallback_error(e)
 
     def _record_partial_usage_for_failure(self) -> None:
         """
@@ -2228,12 +2266,16 @@ def calculate_total_usage(chunks: List[ModelResponse]) -> Usage:
     """Assume most recent usage chunk has total usage uptil then."""
     prompt_tokens: int = 0
     completion_tokens: int = 0
+    latest_usage_chunk = None
+
     for chunk in chunks:
         if "usage" in chunk and chunk["usage"] is not None:
-            if "prompt_tokens" in chunk["usage"]:
-                prompt_tokens = chunk["usage"].get("prompt_tokens", 0) or 0
-            if "completion_tokens" in chunk["usage"]:
-                completion_tokens = chunk["usage"].get("completion_tokens", 0) or 0
+            usage = chunk["usage"]
+            latest_usage_chunk = usage
+            if "prompt_tokens" in usage:
+                prompt_tokens = usage.get("prompt_tokens", 0) or 0
+            if "completion_tokens" in usage:
+                completion_tokens = usage.get("completion_tokens", 0) or 0
 
     returned_usage_chunk = Usage(
         prompt_tokens=prompt_tokens,
@@ -2241,6 +2283,15 @@ def calculate_total_usage(chunks: List[ModelResponse]) -> Usage:
         total_tokens=prompt_tokens + completion_tokens,
     )
 
+    if latest_usage_chunk is not None:
+        latest_cost = (
+            latest_usage_chunk.get("cost")
+            if isinstance(latest_usage_chunk, dict)
+            else getattr(latest_usage_chunk, "cost", None)
+        )
+        if latest_cost is not None:
+            returned_usage_chunk.cost = latest_cost
+
     return returned_usage_chunk
 
 

litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py

@@ -1403,11 +1403,6 @@ class LiteLLMAnthropicMessagesAdapter:
                         assert isinstance(thinking, str)
                         assert isinstance(signature, str)
 
-                        if thinking and signature:
-                            raise ValueError(
-                                "Both `thinking` and `signature` in a single streaming chunk isn't supported."
-                            )
-
                         return "thinking", ChatCompletionThinkingBlock(
                             type="thinking", thinking=thinking, signature=signature
                         )
@@ -1463,17 +1458,14 @@ class LiteLLMAnthropicMessagesAdapter:
                 if choice.delta.reasoning_content is not None:
                     reasoning_content += choice.delta.reasoning_content
 
-        if reasoning_content and reasoning_signature:
-            raise ValueError("Both `reasoning` and `signature` in a single streaming chunk isn't supported.")
-
         if partial_json is not None:
             return "input_json_delta", ContentJsonBlockDelta(type="input_json_delta", partial_json=partial_json)
-        elif reasoning_content:
-            return "thinking_delta", ContentThinkingBlockDelta(type="thinking_delta", thinking=reasoning_content)
         elif reasoning_signature:
             return "signature_delta", ContentThinkingSignatureBlockDelta(
                 type="signature_delta", signature=reasoning_signature
             )
+        elif reasoning_content:
+            return "thinking_delta", ContentThinkingBlockDelta(type="thinking_delta", thinking=reasoning_content)
         else:
             return "text_delta", ContentTextBlockDelta(type="text_delta", text=text)
 

litellm/llms/custom_httpx/aiohttp_transport.py

@@ -85,29 +85,12 @@ class AiohttpResponseStream(httpx.AsyncByteStream):
         try:
             async for chunk in self._aiohttp_response.content.iter_chunked(self.CHUNK_SIZE):
                 yield chunk
-        except (
-            aiohttp.ClientPayloadError,
-            aiohttp.client_exceptions.ClientPayloadError,
-        ) as e:
-            # Handle incomplete transfers more gracefully
-            # Log the error but don't re-raise if we've already yielded some data
-            verbose_logger.debug(f"Transfer incomplete, but continuing: {e}")
-            # If the error is due to incomplete transfer encoding, we can still
-            # return what we've received so far, similar to how httpx handles it
-            return
         except RuntimeError as e:
-            # Some providers (e.g., SSE streams) may close the connection
-            # causing aiohttp StreamReader to raise a generic RuntimeError
-            # with message "Connection closed.". Treat this as a graceful
-            # end-of-stream so downstream consumers don't error.
-            if "Connection closed" in str(e):
-                verbose_logger.debug("Upstream closed streaming connection; ending iterator gracefully")
-                return
-            raise
+            if "Connection closed" not in str(e):
+                raise
+            raise httpx.ReadError(str(e)) from e
         except aiohttp.http_exceptions.TransferEncodingError as e:
-            # Handle transfer encoding errors gracefully
-            verbose_logger.debug(f"Transfer encoding error, but continuing: {e}")
-            return
+            raise httpx.ReadError(str(e)) from e
         except Exception:
             # For other exceptions, use the normal mapping
             with map_aiohttp_exceptions():

litellm/llms/custom_httpx/llm_http_handler.py

@@ -1879,6 +1879,7 @@ class BaseLLMHTTPHandler:
         litellm_params: GenericLiteLLMParams,
         api_key: Optional[str],
         model: str,
+        timeout: Optional[Union[float, httpx.Timeout]] = None,
     ) -> httpx.Response:
         max_attempts = max(provider_config.max_retry_on_anthropic_messages_http_error, 1)
         litellm_params_dict = dict(litellm_params)
@@ -1891,6 +1892,7 @@ class BaseLLMHTTPHandler:
                     data=signed_json_body or json.dumps(request_body),
                     stream=stream or False,
                     logging_obj=logging_obj,
+                    timeout=timeout,
                 )
                 response.raise_for_status()
                 return response
@@ -1925,6 +1927,32 @@ class BaseLLMHTTPHandler:
 
         raise RuntimeError("unreachable: anthropic messages HTTP retry loop exited without return")
 
+    @staticmethod
+    def _resolve_anthropic_messages_timeout(
+        litellm_params: GenericLiteLLMParams,
+        stream: bool,
+        custom_llm_provider: str,
+    ) -> Optional[Union[float, httpx.Timeout]]:
+        from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
+        from litellm.litellm_core_utils.request_timeout_resolver import (
+            get_configured_request_timeout,
+        )
+        from litellm.utils import supports_httpx_timeout
+
+        stream_timeout = litellm_params.get("stream_timeout") if stream else None
+        model_timeout = stream_timeout if stream_timeout is not None else litellm_params.get("timeout")
+        request_timeout = litellm_params.get("request_timeout")
+        global_timeout = get_configured_request_timeout()
+        if model_timeout is None and request_timeout is None and global_timeout is None:
+            return None
+        return CompletionTimeout.resolve(
+            model_timeout,
+            {"request_timeout": request_timeout},
+            custom_llm_provider,
+            global_timeout=global_timeout,
+            supports_httpx_timeout=supports_httpx_timeout,
+        )
+
     async def async_anthropic_messages_handler(
         self,
         model: str,
@@ -2075,6 +2103,11 @@ class BaseLLMHTTPHandler:
             litellm_params=litellm_params,
             api_key=api_key,
             model=model,
+            timeout=self._resolve_anthropic_messages_timeout(
+                litellm_params=litellm_params,
+                stream=stream or False,
+                custom_llm_provider=custom_llm_provider,
+            ),
         )
 
         # used for logging + cost tracking

litellm/llms/openai/responses/guardrail_translation/handler.py

@@ -247,6 +247,8 @@ class OpenAIResponsesHandler(BaseTranslation):
         """
         Merge remapped guardrailed tools with original tools that were not sent
         to the guardrail (e.g. web_search, web_search_preview), preserving order.
+        Tools a guardrail appended (``remapped`` longer than ``original_tools``)
+        have no original slot and are kept so an injected tool is not dropped.
         """
         if not original_tools:
             return remapped
@@ -262,6 +264,8 @@ class OpenAIResponsesHandler(BaseTranslation):
                 if j < len(remapped):
                     result.append(remapped[j])
                     j += 1
+        # Keep guardrail-appended tools that matched no original slot above.
+        result.extend(remapped[j:])
         return result
 
     def _apply_guardrailed_tools_to_data(

litellm/model_prices_and_context_window_backup.json

@@ -44261,6 +44261,90 @@
         "supports_response_schema": true,
         "supports_tool_choice": true
     },
+    "bedrock_mantle/openai.gpt-5.6-sol": {
+        "input_cost_per_token": 5.5e-06,
+        "cache_creation_input_token_cost": 6.875e-06,
+        "cache_read_input_token_cost": 5.5e-07,
+        "output_cost_per_token": 3.3e-05,
+        "litellm_provider": "bedrock_mantle",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "responses",
+        "use_openai_responses_path": true,
+        "supported_endpoints": [
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "bedrock_mantle/openai.gpt-5.6-terra": {
+        "input_cost_per_token": 2.75e-06,
+        "cache_creation_input_token_cost": 3.4375e-06,
+        "cache_read_input_token_cost": 2.75e-07,
+        "output_cost_per_token": 1.65e-05,
+        "litellm_provider": "bedrock_mantle",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "responses",
+        "use_openai_responses_path": true,
+        "supported_endpoints": [
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "bedrock_mantle/openai.gpt-5.6-luna": {
+        "input_cost_per_token": 1.1e-06,
+        "cache_creation_input_token_cost": 1.375e-06,
+        "cache_read_input_token_cost": 1.1e-07,
+        "output_cost_per_token": 6.6e-06,
+        "litellm_provider": "bedrock_mantle",
+        "max_input_tokens": 272000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "responses",
+        "use_openai_responses_path": true,
+        "supported_endpoints": [
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
     "bedrock_mantle/openai.gpt-5.5": {
         "input_cost_per_token": 5.5e-06,
         "cache_read_input_token_cost": 5.5e-07,
@@ -44373,6 +44457,7 @@
         "supports_vision": true
     },
     "bedrock_mantle/xai.grok-4.3": {
+        "use_openai_responses_path": true,
         "input_cost_per_token": 1.25e-06,
         "output_cost_per_token": 2.5e-06,
         "cache_read_input_token_cost": 2e-07,

litellm/models/mcp_server.py

@@ -79,6 +79,7 @@ class LiteLLM_MCPServerTable(LiteLLMPydanticObjectBase):
     command: Optional[str] = None
     args: List[str] = Field(default_factory=list)
     env: Dict[str, str] = Field(default_factory=dict)
+    issuer: Optional[str] = None
     authorization_url: Optional[str] = None
     token_url: Optional[str] = None
     registration_url: Optional[str] = None

litellm/proxy/_experimental/mcp_server/db.py

@@ -48,6 +48,7 @@ if TYPE_CHECKING:
 
 _AUTH_FLOW_SCOPED_FIELDS: frozenset = frozenset(
     {
+        "issuer",
         "authorization_url",
         "token_url",
         "registration_url",
@@ -60,6 +61,13 @@ _AUTH_FLOW_SCOPED_FIELDS: frozenset = frozenset(
     }
 )
 
+
+def _blank_to_none(value: Optional[str]) -> Optional[str]:
+    if not isinstance(value, str):
+        return None
+    return value.strip() or None
+
+
 # Token-exchange settings with dedicated columns that also exist on
 # ``MCPCredentials`` as a legacy shape (rows and REST callers that predate the
 # columns). Every write lifts blob values into the columns and strips them from
@@ -697,13 +705,15 @@ async def update_mcp_server(
     # of being reset to a schema default (transport=sse, allow_all_keys=False...).
     data_dict = _prepare_mcp_server_data(data, exclude_unset=True, fields_set=fields_set)
 
-    # Pre-fetch existing record once if we need it for auth_type or credential logic
+    # Pre-fetch existing record once if we need it for auth_type, url, or credential logic
     existing = None
     has_credentials = "credentials" in data_dict and data_dict["credentials"] is not None
     # An explicit token-exchange column write (set or clear) also migrates the
     # legacy blob copies below, so the existing row is needed for those updates.
     explicit_te_write = bool(_TOKEN_EXCHANGE_COLUMN_FIELDS & data_dict.keys())
-    if data.auth_type or has_credentials or explicit_te_write:
+    url_provided = "url" in data_dict and data_dict["url"] is not None
+    issuer_provided = "issuer" in data_dict
+    if data.auth_type or has_credentials or explicit_te_write or url_provided or issuer_provided:
         existing = await MCPServerRepository(prisma_client).table.find_unique(where={"server_id": data.server_id})
 
     auth_type_changed = bool(
@@ -711,13 +721,30 @@ async def update_mcp_server(
         and existing
         and _credential_auth_class(existing.auth_type) != _credential_auth_class(data.auth_type)
     )
+    # A url change re-points the server at a potentially different upstream, so any discovered or
+    # trust-on-first-use OAuth endpoints/issuer belong to the old upstream and must re-discover.
+    url_changed = bool(url_provided and existing and existing.url != data_dict["url"])
+    old_issuer = _blank_to_none(getattr(existing, "issuer", None)) if existing else None
+    issuer_changed = bool(
+        issuer_provided and old_issuer is not None and _blank_to_none(data_dict.get("issuer")) != old_issuer
+    )
 
     # Clear stale credentials when auth_type changes but no new credentials provided
     if auth_type_changed and "credentials" not in data_dict:
         data_dict["credentials"] = None
 
-    if auth_type_changed:
-        data_dict.update({field: None for field in _AUTH_FLOW_SCOPED_FIELDS if field not in data_dict})
+    if auth_type_changed or url_changed or issuer_changed:
+        # Clear each auth-flow-scoped field that the caller either omitted (partial update) or
+        # resubmitted unchanged. The edit form re-sends every field, so a stale issuer/endpoint
+        # belonging to the old upstream would otherwise survive a url/auth_type change and win in the
+        # resolution merge; only a genuinely new submitted value is kept.
+        data_dict.update(
+            {
+                field: None
+                for field in _AUTH_FLOW_SCOPED_FIELDS
+                if field not in data_dict or data_dict[field] == getattr(existing, field, None)
+            }
+        )
 
     # An explicit column write that does not touch credentials must still migrate
     # the row's legacy blob copies: lift values for columns the caller left
@@ -1181,6 +1208,7 @@ def mcp_oauth_token_identity(server: object) -> tuple[object, ...]:
         getattr(server, "spec_path", None),
         getattr(server, "auth_type", None),
         getattr(server, "oauth2_flow", None),
+        getattr(server, "issuer", None),
         getattr(server, "authorization_url", None),
         getattr(server, "token_url", None),
         getattr(server, "registration_url", None),

litellm/proxy/_experimental/mcp_server/mcp_server_manager.py

@@ -186,6 +186,104 @@ _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES: tuple[MCPAuth, ...] = (
 )
 
 
+def _blank_to_none(value: str | None) -> str | None:
+    """Collapse an absent, empty, or whitespace-only string to ``None``.
+
+    OAuth endpoint fields are consumed by truthiness-based merges (``row or discovered``) and by the
+    corroboration gate. A whitespace-only value is truthy to ``or`` but is not a usable endpoint, so
+    without this the merge would keep the blank value for redirects while the gate treats it as
+    unpinned and backfills the other fields, yielding a broken half-discovered config. Normalizing
+    the pinned fields once, at each build entry point, gives every downstream consumer a single
+    notion of "blank" so those code paths cannot disagree.
+    """
+    if not isinstance(value, str):
+        return None
+    return value.strip() or None
+
+
+def _uses_issuer_anchor(manual_issuer: str | None, is_discovery_auth_type: bool) -> bool:
+    """Whether the endpoints are authoritatively anchored to an admin-pinned issuer (RFC 8414 §3.3).
+
+    This is the trust/provenance property, distinct from whether the ``issuer`` field is merely
+    populated: a trust-on-first-use discovered issuer sets ``issuer`` for token identity but is NOT
+    anchored, so its endpoints stay resource-rooted. Anchoring holds only when the issuer was pinned
+    (present on the row/config) on a discovery auth type. Every consumer of "is this anchored" reads
+    this one definition, so the answer cannot diverge across build paths.
+    """
+    return _blank_to_none(manual_issuer) is not None and is_discovery_auth_type
+
+
+def _endpoints_yield_to_issuer(
+    issuer: str | None,
+    is_discovery_auth_type: bool,
+    authorization_url: str | None,
+    token_url: str | None,
+    registration_url: str | None,
+) -> tuple[str | None, str | None, str | None]:
+    """The single rule that makes an admin-configured ``issuer`` the sole authoritative endpoint
+    source (RFC 8414 §3.3): when it is set for a discovery auth type, the stored/manual
+    ``authorization_url``/``token_url``/``registration_url`` do not apply. They neither anchor nor
+    short-circuit discovery, never override the issuer document in the merge, and never substitute for
+    it when the issuer fetch fails (fail-closed). Returns the endpoint values that remain in force,
+    i.e. all ``None`` when issuer-anchored, else the inputs unchanged. Called at every resolution site
+    so the invariant holds in one place instead of being re-derived per merge.
+    """
+    if issuer is not None and is_discovery_auth_type:
+        return None, None, None
+    return authorization_url, token_url, registration_url
+
+
+def _normalized_authorize_endpoint(url: str) -> str:
+    """Compare authorize endpoints on scheme, host, and path only. The default port is elided and
+    the host is lowercased so ``https://IDP.example.com:443/authorize/`` and
+    ``https://idp.example.com/authorize`` are the same identity; query and trailing slash are not."""
+    parsed = urlparse(url)
+    scheme = parsed.scheme.lower()
+    host = (parsed.hostname or "").lower()
+    default_port = {"https": 443, "http": 80}.get(scheme)
+    try:
+        port = parsed.port
+    except ValueError:
+        port = None
+    authority = host if port is None or port == default_port else f"{host}:{port}"
+    return f"{scheme}://{authority}{parsed.path.rstrip('/')}"
+
+
+def _issuer_matches(claimed_issuer: object, configured_issuer: str) -> bool:
+    """RFC 8414 §3.3 issuer equality between the metadata document's self-attested ``issuer`` and the
+    admin-configured issuer, tolerant only of URL-insignificant differences (scheme/host case, the
+    default port, a trailing slash). A non-string or empty claimed issuer never matches, so a
+    document that omits ``issuer`` fails closed under issuer-anchored discovery.
+    """
+    if not isinstance(claimed_issuer, str) or not claimed_issuer:
+        return False
+    return _normalized_authorize_endpoint(claimed_issuer) == _normalized_authorize_endpoint(configured_issuer)
+
+
+def _endpoints_corroborate_authorization_url(
+    source_authorization_url: str | None,
+    trusted_authorization_url: str | None,
+) -> bool:
+    """Whether a source's ``token_url``/``registration_url`` may be paired with a trusted authorize
+    endpoint. This is the single trust rule for adopting OAuth endpoints from any non-manual source.
+
+    Discovery is rooted at the MCP resource (RFC 9728), so a compromised upstream can advertise an
+    attacker-run authorization server. When ``authorization_url`` is admin-pinned, pairing it with a
+    ``token_url`` from a different source is the RFC 9700 authorization-server mix-up: the user signs
+    in at the trusted authorize endpoint while the gateway redeems the code, with the stored client
+    secret and PKCE verifier, at the attacker's token endpoint. Endpoints are trustworthy together
+    only when they share an authorization server, so a source's endpoints are adopted only when the
+    same source advertised an ``authorization_endpoint`` matching the pinned value. With no pinned
+    value (``trusted_authorization_url is None``) there is nothing to protect: the authorize endpoint
+    comes from the same source as the token endpoint, so they corroborate each other by construction.
+    """
+    if not (trusted_authorization_url and trusted_authorization_url.strip()):
+        return True
+    return bool(source_authorization_url) and _normalized_authorize_endpoint(
+        source_authorization_url
+    ) == _normalized_authorize_endpoint(trusted_authorization_url)
+
+
 def _carry_forward_resolved_oauth_endpoints(new_server: MCPServer, previous_server: MCPServer | None) -> None:
     """Keep the last known good OAuth endpoints when a rebuild's re-discovery comes back empty.
 
@@ -193,26 +291,98 @@ def _carry_forward_resolved_oauth_endpoints(new_server: MCPServer, previous_serv
     during re-discovery downgrades a working server (``authorization_url`` set) to a broken one
     (``None``, /authorize 400s) with no configuration change. Mirrors the ``short_prefix``
     carry-forward. Skipped when the server's ``url`` or ``auth_type`` changed, since the previous
-    endpoints may then belong to a different upstream. ``registration_url`` IS carried here even
-    though ``_persist_discovered_oauth_endpoints`` refuses to write it to the row: carrying only
-    restores the same in-memory value the previous build already ran with, while persisting it
-    would flip ``_dcr_bridge_relays_client_registration`` (which keys off the stored column) for
-    dcr_bridge servers that never had one configured.
+    endpoints may then belong to a different upstream. ``registration_url`` IS carried even though
+    ``_persist_discovered_oauth_endpoints`` refuses to write it to the row: carrying only restores
+    the same in-memory value the previous build already ran with, while persisting it would flip
+    ``_dcr_bridge_relays_client_registration`` (which keys off the stored column) for dcr_bridge
+    servers that never had one configured.
+
+    Carry-forward is a non-manual endpoint source, so the same trust rule as discovery applies: the
+    previous ``token_url``/``registration_url``/``scopes`` are carried only when the previous
+    ``authorization_url`` corroborates the authorize endpoint this build will use, i.e. when the
+    incoming build has no pinned authorize endpoint (``None`` -> we adopt the previous one too, a
+    consistent group) or pins the same one. An admin re-pointing ``authorization_url`` to a different
+    server must not keep serving the old server's token endpoint or granted scopes.
+
+    When the server is issuer-anchored (``issuer_is_anchored`` -- a pinned issuer on a discovery auth
+    type), the endpoints come solely from the §3.3-validated issuer document, so carry-forward is
+    skipped entirely for its endpoints: a failed issuer fetch leaves them ``None`` and must stay
+    ``None`` (fail-closed), never resurrected from the previous registry entry. A merely discovered
+    (trust-on-first-use) issuer is NOT anchored -- ``issuer`` is set for token identity but the
+    endpoints are resource-rooted, so they still carry forward as last-known-good, gated by the
+    corroboration check below like any other resource-rooted server. Scopes stay resource-driven and
+    can carry either way.
     """
     if previous_server is None:
         return
     if previous_server.url != new_server.url or previous_server.auth_type != new_server.auth_type:
         return
+    if new_server.issuer_is_anchored:
+        # Endpoints come solely from the §3.3-validated issuer document; a failed fetch stays
+        # fail-closed and must not be resurrected from the previous entry. Only the resource-driven
+        # scopes carry as last-known-good.
+        if not new_server.scopes and previous_server.scopes:
+            new_server.scopes = previous_server.scopes
+        return
+    may_carry = _endpoints_corroborate_authorization_url(
+        previous_server.authorization_url, new_server.authorization_url
+    )
     if new_server.authorization_url is None and previous_server.authorization_url:
         new_server.authorization_url = previous_server.authorization_url
-    if new_server.token_url is None and previous_server.token_url:
+    if may_carry and new_server.token_url is None and previous_server.token_url:
         new_server.token_url = previous_server.token_url
-    if new_server.registration_url is None and previous_server.registration_url:
+    if may_carry and new_server.registration_url is None and previous_server.registration_url:
         new_server.registration_url = previous_server.registration_url
-    if not new_server.scopes and previous_server.scopes:
+    if may_carry and not new_server.scopes and previous_server.scopes:
         new_server.scopes = previous_server.scopes
 
 
+def _restrict_discovery_to_corroborated_authorization_server(
+    metadata: MCPOAuthMetadata | None,
+    manual_authorization_url: str | None,
+    server_identifier: str,
+    is_dcr_bridge: bool,
+) -> MCPOAuthMetadata | None:
+    """Reject discovered token/registration endpoints a manually pinned authorize endpoint cannot
+    vouch for (the RFC 9700 authorization-server mix-up).
+
+    Discovery is rooted at the MCP resource, so a compromised upstream can advertise an attacker
+    ``token_endpoint``: with ``authorization_url`` admin-pinned but ``token_url`` blank, the merge
+    would pair the trusted authorize endpoint with that attacker token endpoint, and the gateway would
+    post the authorization code and client secret there. So the discovered ``token_url`` and
+    ``registration_url`` are kept only if the document corroborates the pin (its
+    ``authorization_endpoint`` matches). ``scopes`` are deliberately NOT gated here: per the MCP
+    authorization spec Scope Selection Strategy and RFC 9700 §2.3, the scopes a client requests are
+    resource-driven (the WWW-Authenticate challenge or the RFC 9728 protected-resource
+    ``scopes_supported``), and scope inflation by a compromised resource is bounded by the
+    authorization server and user consent (RFC 6749 §3.3), not by the client second-guessing the
+    request. With no pin there is no trust anchor to protect, so discovery is returned as-is.
+    """
+    if metadata is None or not (manual_authorization_url and manual_authorization_url.strip()):
+        return metadata
+    if _endpoints_corroborate_authorization_url(metadata.authorization_url, manual_authorization_url):
+        return metadata
+    if not metadata.token_url and not metadata.registration_url:
+        return metadata
+    bridge_note = (
+        " The discovered registration_url is rejected with it, so this dcr_bridge server stays on the"
+        " short-circuit registration arm."
+        if is_dcr_bridge and metadata.registration_url
+        else ""
+    )
+    verbose_logger.warning(
+        "MCP OAuth discovery for server %s advertised authorization_endpoint %s, which does not match the "
+        "manually configured authorization_url %s; rejecting the discovered token_url/registration_url so "
+        "authorization codes and client credentials only follow the configured authorization server. "
+        "Configure Token URL manually if the mismatch is intentional.%s",
+        server_identifier,
+        _normalized_authorize_endpoint(metadata.authorization_url) if metadata.authorization_url else "<absent>",
+        _normalized_authorize_endpoint(manual_authorization_url),
+        bridge_note,
+    )
+    return metadata.model_copy(update={"token_url": None, "registration_url": None})
+
+
 def invalidate_user_env_vars_cache(user_id: str, server_id: str) -> None:
     """Drop a cached entry after the user stores or clears their env var values
     so the next request reads the fresh value instead of a stale one."""
@@ -1026,36 +1196,68 @@ class MCPServerManager:
             )
 
             auth_type = server_config.get("auth_type", None)
-            if server_url and (
-                auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES
+            manual_issuer = _blank_to_none(server_config.get("issuer"))
+            manual_authorization_url = _blank_to_none(server_config.get("authorization_url"))
+            manual_token_url = _blank_to_none(server_config.get("token_url"))
+            manual_registration_url = _blank_to_none(server_config.get("registration_url"))
+            is_discovery_auth_type = auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES
+            use_issuer_anchor = _uses_issuer_anchor(manual_issuer, is_discovery_auth_type)
+            manual_authorization_url, manual_token_url, manual_registration_url = _endpoints_yield_to_issuer(
+                manual_issuer,
+                is_discovery_auth_type,
+                manual_authorization_url,
+                manual_token_url,
+                manual_registration_url,
+            )
+            should_discover = bool(server_url) and (
+                is_discovery_auth_type
                 or self._obo_needs_endpoint_discovery(
                     auth_type,
                     server_config.get("token_exchange_endpoint"),
-                    server_config.get("token_url"),
+                    manual_token_url,
                 )
-            ):
+            )
+            if not should_discover:
+                mcp_oauth_metadata = None
+            elif manual_issuer is not None and is_discovery_auth_type:
+                mcp_oauth_metadata = await self._fetch_issuer_anchored_oauth_metadata(manual_issuer, server_url)
+            else:
                 mcp_oauth_metadata = await self._descovery_metadata(
                     server_url=server_url,
-                    allow_origin_fallback=auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES,
+                    allow_origin_fallback=is_discovery_auth_type,
+                )
+
+            if use_issuer_anchor:
+                gated_oauth_metadata = mcp_oauth_metadata
+            elif is_discovery_auth_type:
+                gated_oauth_metadata = _restrict_discovery_to_corroborated_authorization_server(
+                    mcp_oauth_metadata,
+                    manual_authorization_url,
+                    server_name or server_id,
+                    bool(server_config.get("dcr_bridge")),
                 )
             else:
-                mcp_oauth_metadata = None
+                gated_oauth_metadata = mcp_oauth_metadata
 
             # Filter blank scopes (e.g. YAML ``scopes: [""]``) the same way the DB-build path does, so
             # an all-blank list normalizes to None rather than a ``("",)`` tuple that skips the
             # entra_obo fail-closed scope precondition and POSTs an empty scope to the IdP.
             resolved_scopes = self._extract_scopes(server_config.get("scopes")) or (
-                mcp_oauth_metadata.scopes if mcp_oauth_metadata else None
+                gated_oauth_metadata.scopes if gated_oauth_metadata else None
             )
-            resolved_authorization_url = server_config.get("authorization_url") or (
-                mcp_oauth_metadata.authorization_url if mcp_oauth_metadata else None
+            resolved_authorization_url = manual_authorization_url or (
+                gated_oauth_metadata.authorization_url if gated_oauth_metadata else None
             )
-            resolved_token_url = server_config.get("token_url") or (
-                mcp_oauth_metadata.token_url if mcp_oauth_metadata else None
+            resolved_token_url = manual_token_url or (gated_oauth_metadata.token_url if gated_oauth_metadata else None)
+            resolved_registration_url = manual_registration_url or (
+                gated_oauth_metadata.registration_url if gated_oauth_metadata else None
             )
-            resolved_registration_url = server_config.get("registration_url") or (
-                mcp_oauth_metadata.registration_url if mcp_oauth_metadata else None
+            discovered_issuer = (
+                gated_oauth_metadata.discovered_issuer
+                if gated_oauth_metadata and not gated_oauth_metadata.from_origin_fallback
+                else None
             )
+            effective_issuer = manual_issuer or discovered_issuer
 
             config_oauth2_flow = server_config.get("oauth2_flow", None)
             if auth_type == MCPAuth.oauth2 and config_oauth2_flow not in (
@@ -1104,6 +1306,8 @@ class MCPServerManager:
                 client_secret=server_config.get("client_secret", None),
                 oauth2_flow=self._explicit_oauth2_flow(config_oauth2_flow),
                 scopes=resolved_scopes,
+                issuer=effective_issuer,
+                issuer_is_anchored=use_issuer_anchor,
                 authorization_url=resolved_authorization_url,
                 token_url=resolved_token_url,
                 registration_url=resolved_registration_url,
@@ -1364,6 +1568,52 @@ class MCPServerManager:
             decrypt_global_env_var_values(env_vars_list)
         return env_vars_list
 
+    async def _resolve_table_oauth_metadata(
+        self,
+        *,
+        mcp_server: LiteLLM_MCPServerTable,
+        auth_type: MCPAuthType,
+        server_url: Optional[str],
+        manual_issuer: Optional[str],
+        manual_authorization_url: Optional[str],
+        manual_token_url: Optional[str],
+        is_discovery_auth_type: bool,
+        use_issuer_anchor: bool,
+        scopes: Optional[list[str]],
+        token_exchange_endpoint: Optional[str],
+    ) -> Optional[MCPOAuthMetadata]:
+        has_all_upstream_oauth_fields = bool(manual_authorization_url and manual_token_url and scopes)
+        needs_discovery = bool(server_url) and (
+            (is_discovery_auth_type and not has_all_upstream_oauth_fields)
+            or self._obo_needs_endpoint_discovery(auth_type, token_exchange_endpoint, manual_token_url)
+        )
+        if not needs_discovery:
+            mcp_oauth_metadata: Optional[MCPOAuthMetadata] = None
+        elif use_issuer_anchor and manual_issuer is not None:
+            mcp_oauth_metadata = await self._fetch_issuer_anchored_oauth_metadata(manual_issuer, server_url)
+        else:
+            mcp_oauth_metadata = await self._descovery_metadata(
+                server_url=server_url,  # type: ignore[arg-type]
+                allow_origin_fallback=is_discovery_auth_type,
+            )
+        if needs_discovery and not use_issuer_anchor and mcp_oauth_metadata is None:
+            verbose_logger.warning(
+                "MCP OAuth discovery yielded no metadata for server %s (%s); "
+                "OAuth endpoints/scopes stay unresolved until a rebuild succeeds",
+                mcp_server.server_id,
+                server_url,
+            )
+        if use_issuer_anchor:
+            return mcp_oauth_metadata
+        if is_discovery_auth_type:
+            return _restrict_discovery_to_corroborated_authorization_server(
+                mcp_oauth_metadata,
+                manual_authorization_url,
+                mcp_server.server_id,
+                bool(getattr(mcp_server, "dcr_bridge", None)),
+            )
+        return mcp_oauth_metadata
+
     async def build_mcp_server_from_table(
         self,
         mcp_server: LiteLLM_MCPServerTable,
@@ -1447,32 +1697,38 @@ class MCPServerManager:
 
         auth_type = cast(MCPAuthType, mcp_server.auth_type)
         server_url = mcp_server.url
-        needs_discovery = bool(server_url) and (
-            (auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES and not mcp_server.authorization_url)
-            or self._obo_needs_endpoint_discovery(
-                auth_type,
-                mcp_server.token_exchange_endpoint
-                or (credentials_dict.get("token_exchange_endpoint") if credentials_dict else None),
-                mcp_server.token_url,
-            )
+        manual_issuer = _blank_to_none(mcp_server.issuer)
+        manual_authorization_url = _blank_to_none(mcp_server.authorization_url)
+        manual_token_url = _blank_to_none(mcp_server.token_url)
+        manual_registration_url = _blank_to_none(mcp_server.registration_url)
+        is_discovery_auth_type = auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES
+        use_issuer_anchor = _uses_issuer_anchor(manual_issuer, is_discovery_auth_type)
+        manual_authorization_url, manual_token_url, manual_registration_url = _endpoints_yield_to_issuer(
+            manual_issuer, is_discovery_auth_type, manual_authorization_url, manual_token_url, manual_registration_url
         )
-        mcp_oauth_metadata = (
-            await self._descovery_metadata(
-                server_url=server_url,  # type: ignore[arg-type]
-                allow_origin_fallback=auth_type in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES,
-            )
-            if needs_discovery
+        token_exchange_endpoint = mcp_server.token_exchange_endpoint or (
+            credentials_dict.get("token_exchange_endpoint") if credentials_dict else None
+        )
+        gated_oauth_metadata = await self._resolve_table_oauth_metadata(
+            mcp_server=mcp_server,
+            auth_type=auth_type,
+            server_url=server_url,
+            manual_issuer=manual_issuer,
+            manual_authorization_url=manual_authorization_url,
+            manual_token_url=manual_token_url,
+            is_discovery_auth_type=is_discovery_auth_type,
+            use_issuer_anchor=use_issuer_anchor,
+            scopes=scopes,
+            token_exchange_endpoint=token_exchange_endpoint,
+        )
+
+        resolved_scopes = scopes or (gated_oauth_metadata.scopes if gated_oauth_metadata else None)
+        discovered_issuer = (
+            gated_oauth_metadata.discovered_issuer
+            if gated_oauth_metadata and not gated_oauth_metadata.from_origin_fallback
             else None
         )
-        if needs_discovery and mcp_oauth_metadata is None:
-            verbose_logger.warning(
-                "MCP OAuth discovery yielded no metadata for server %s (%s); "
-                "OAuth endpoints stay unresolved until a rebuild succeeds",
-                mcp_server.server_id,
-                server_url,
-            )
-
-        resolved_scopes = scopes or (mcp_oauth_metadata.scopes if mcp_oauth_metadata else None)
+        effective_issuer = manual_issuer or discovered_issuer
 
         new_server = MCPServer(
             server_id=mcp_server.server_id,
@@ -1492,9 +1748,11 @@ class MCPServerManager:
             client_secret=client_secret_value or getattr(mcp_server, "client_secret", None),
             oauth2_flow=self._explicit_oauth2_flow(getattr(mcp_server, "oauth2_flow", None)),
             scopes=resolved_scopes,
-            authorization_url=mcp_server.authorization_url or getattr(mcp_oauth_metadata, "authorization_url", None),
-            token_url=mcp_server.token_url or getattr(mcp_oauth_metadata, "token_url", None),
-            registration_url=mcp_server.registration_url or getattr(mcp_oauth_metadata, "registration_url", None),
+            issuer=effective_issuer,
+            issuer_is_anchored=use_issuer_anchor,
+            authorization_url=manual_authorization_url or getattr(gated_oauth_metadata, "authorization_url", None),
+            token_url=manual_token_url or getattr(gated_oauth_metadata, "token_url", None),
+            registration_url=manual_registration_url or getattr(gated_oauth_metadata, "registration_url", None),
             token_endpoint_auth_method=(
                 credentials_dict.get("token_endpoint_auth_method") if credentials_dict else None
             ),
@@ -1545,16 +1803,18 @@ class MCPServerManager:
             await self._persist_discovered_obo_token_url(
                 server_id=mcp_server.server_id,
                 auth_type=auth_type,
-                existing_token_url=mcp_server.token_url,
+                existing_token_url=manual_token_url,
                 discovered_token_url=new_server.token_url,
             )
             await self._persist_discovered_oauth_endpoints(
                 server_id=mcp_server.server_id,
                 auth_type=auth_type,
-                existing_authorization_url=mcp_server.authorization_url,
-                existing_token_url=mcp_server.token_url,
+                existing_issuer=manual_issuer,
+                existing_authorization_url=manual_authorization_url,
+                existing_token_url=manual_token_url,
                 existing_scopes=scopes,
-                metadata=mcp_oauth_metadata,
+                metadata=gated_oauth_metadata,
+                is_issuer_anchored=use_issuer_anchor,
             )
         return new_server
 
@@ -1598,10 +1858,12 @@ class MCPServerManager:
         *,
         server_id: str,
         auth_type: MCPAuthType | None,
+        existing_issuer: str | None,
         existing_authorization_url: str | None,
         existing_token_url: str | None,
         existing_scopes: list[str] | None,
         metadata: MCPOAuthMetadata | None,
+        is_issuer_anchored: bool = False,
     ) -> None:
         """Write freshly discovered OAuth endpoints back onto the DB row.
 
@@ -1615,19 +1877,37 @@ class MCPServerManager:
         because ``_dcr_bridge_relays_client_registration`` keys off that column. Best-effort: a
         failed write re-discovers on the next build. Scopes go through ``update_mcp_server`` so
         they merge into the credentials blob without touching the stored client credentials.
+
+        For an issuer-anchored server (``is_issuer_anchored``) the endpoints are re-derived from the
+        §3.3-validated issuer document on every build, so they are NOT persisted into the endpoint
+        columns: persisting them would make the next build see populated endpoints and treat them as
+        authoritative stored values, defeating the "endpoints come solely from the issuer" invariant.
+        Only the resource-driven scopes are persisted for such servers.
         """
         if auth_type not in _UPSTREAM_OAUTH_DISCOVERY_AUTH_TYPES:
             return
         if metadata is None or metadata.from_origin_fallback:
             return
+        issuer_update = (
+            {"issuer": metadata.discovered_issuer} if metadata.discovered_issuer and not existing_issuer else {}
+        )
         authorization_url_update = (
             {"authorization_url": metadata.authorization_url}
-            if metadata.authorization_url and not existing_authorization_url
+            if metadata.authorization_url and not existing_authorization_url and not is_issuer_anchored
+            else {}
+        )
+        token_url_update = (
+            {"token_url": metadata.token_url}
+            if metadata.token_url and not existing_token_url and not is_issuer_anchored
             else {}
         )
-        token_url_update = {"token_url": metadata.token_url} if metadata.token_url and not existing_token_url else {}
         scopes_update = {"credentials": {"scopes": metadata.scopes}} if metadata.scopes and not existing_scopes else {}
-        updates: dict[str, object] = {**authorization_url_update, **token_url_update, **scopes_update}
+        updates: dict[str, object] = {
+            **issuer_update,
+            **authorization_url_update,
+            **token_url_update,
+            **scopes_update,
+        }
         if not updates:
             return
         from litellm.proxy._experimental.mcp_server.db import (  # noqa: PLC0415  # db.py imports this module at load
@@ -3200,8 +3480,41 @@ class MCPServerManager:
                 return metadata
         return None
 
+    async def _fetch_issuer_anchored_oauth_metadata(
+        self, issuer: str, server_url: Optional[str]
+    ) -> Optional[MCPOAuthMetadata]:
+        """RFC 8414 issuer-anchored discovery for the OAuth endpoints, with resource-driven scopes.
+
+        Fetch authorization-server metadata from the admin-configured issuer's own origin and adopt
+        its ``token_endpoint``/``registration_endpoint`` only when the document self-attests that same
+        issuer (RFC 8414 §3.3). Because the trust anchor is the pinned issuer rather than anything the
+        MCP resource advertises, the endpoints are authoritative for that issuer and cannot be
+        substituted by a compromised resource. Fails closed (returns None) on a §3.3 mismatch or a
+        fetch failure. The issuer is passed as its own ``server_url`` so the endpoint fetch is treated
+        as same-authority and is not subject to the resource-scoped SSRF shortcut.
+
+        Scopes are NOT taken from the issuer document. Per the MCP authorization spec Scope Selection
+        Strategy and RFC 9728, the scopes a client requests are resource-driven (the WWW-Authenticate
+        challenge or the protected-resource ``scopes_supported``), so the resource's advertised scopes
+        are fetched separately and used; the resource can influence only the requested scope, which
+        the authorization server and user consent bound (RFC 6749 §3.3), never the token endpoint.
+        """
+        metadata = await self._fetch_single_authorization_server_metadata(issuer, issuer, require_issuer=issuer)
+        if metadata is None:
+            verbose_logger.warning(
+                "MCP OAuth issuer-anchored discovery for issuer %s yielded no metadata whose issuer "
+                "matched (RFC 8414 §3.3); OAuth endpoints stay unresolved until a rebuild succeeds",
+                issuer,
+            )
+            return None
+        resource_metadata = (
+            await self._descovery_metadata(server_url, allow_origin_fallback=False) if server_url else None
+        )
+        resource_scopes = resource_metadata.scopes if resource_metadata else None
+        return metadata.model_copy(update={"scopes": resource_scopes})
+
     async def _fetch_single_authorization_server_metadata(
-        self, issuer_url: str, server_url: str
+        self, issuer_url: str, server_url: str, require_issuer: Optional[str] = None
     ) -> Optional[MCPOAuthMetadata]:
         try:
             parsed = urlparse(issuer_url)
@@ -3245,20 +3558,33 @@ class MCPServerManager:
                 )
                 continue
 
-            scopes = self._extract_scopes(data.get("scopes_supported"))
+            claimed_issuer = data.get("issuer")
             verbose_logger.debug(
                 "Authorization server metadata from %s: issuer=%s grant_types_supported=%s "
                 "token_endpoint_auth_methods_supported=%s",
                 url,
-                data.get("issuer"),
+                claimed_issuer,
                 data.get("grant_types_supported"),
                 data.get("token_endpoint_auth_methods_supported"),
             )
+            if require_issuer is not None and not _issuer_matches(claimed_issuer, require_issuer):
+                verbose_logger.warning(
+                    "MCP OAuth issuer-anchored discovery: metadata at %s self-attests issuer %r, which "
+                    "does not match the configured issuer %r (RFC 8414 §3.3); rejecting so a compromised "
+                    "resource cannot substitute an attacker authorization server",
+                    url,
+                    claimed_issuer,
+                    require_issuer,
+                )
+                continue
+
+            scopes = self._extract_scopes(data.get("scopes_supported"))
             metadata = MCPOAuthMetadata(
                 scopes=scopes,
                 authorization_url=data.get("authorization_endpoint"),
                 token_url=data.get("token_endpoint"),
                 registration_url=data.get("registration_endpoint"),
+                discovered_issuer=claimed_issuer if isinstance(claimed_issuer, str) and claimed_issuer else None,
             )
 
             if any(
@@ -4979,6 +5305,7 @@ class MCPServerManager:
             command=getattr(server, "command", None),
             args=getattr(server, "args", None) or [],
             env=getattr(server, "env", None) or {},
+            issuer=server.issuer,
             authorization_url=server.authorization_url,
             token_url=server.token_url,
             registration_url=server.registration_url,
@@ -5088,6 +5415,7 @@ class MCPServerManager:
             command=getattr(server, "command", None),
             args=getattr(server, "args", None) or [],
             env=getattr(server, "env", None) or {},
+            issuer=server.issuer,
             authorization_url=server.authorization_url,
             token_url=server.token_url,
             registration_url=server.registration_url,

litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py

@@ -175,16 +175,18 @@ mcp_oauth2_token_cache = MCPOAuth2TokenCache()
 def _compute_per_user_token_ttl(server: "MCPServer", expires_in: Optional[int]) -> int:
     """Compute Redis TTL for a per-user token.
 
-    Uses server.token_storage_ttl_seconds when configured; otherwise derives
-    TTL from expires_in minus the expiry buffer; falls back to the default TTL.
+    Uses server.token_storage_ttl_seconds when configured, capped at the token's
+    remaining lifetime (expires_in minus the expiry buffer) so a cached entry never
+    outlives the token itself; otherwise derives TTL from expires_in minus the
+    expiry buffer; falls back to the default TTL.
     """
+    lifetime_bound = expires_in - MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS if expires_in is not None else None
     if server.token_storage_ttl_seconds is not None:
-        return max(server.token_storage_ttl_seconds, 1)
-    if expires_in is not None:
-        return max(
-            expires_in - MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS,
-            1,
-        )
+        if lifetime_bound is None:
+            return max(server.token_storage_ttl_seconds, 1)
+        return max(min(server.token_storage_ttl_seconds, lifetime_bound), 1)
+    if lifetime_bound is not None:
+        return max(lifetime_bound, 1)
     return MCP_PER_USER_TOKEN_DEFAULT_TTL
 
 

litellm/proxy/_experimental/mcp_server/rest_endpoints.py

@@ -1138,6 +1138,7 @@ if MCP_AVAILABLE:
                 static_headers=request.static_headers,
                 client_id=client_id,
                 client_secret=client_secret,
+                issuer=request.issuer,
                 token_url=request.token_url,
                 scopes=scopes,
                 authorization_url=request.authorization_url,

litellm/proxy/_experimental/mcp_server/semantic_tool_filter.py

@@ -4,6 +4,7 @@ Semantic MCP Tool Filtering using semantic-router
 Filters MCP tools semantically for /chat/completions and /responses endpoints.
 """
 
+import asyncio
 from typing import TYPE_CHECKING, Any, Dict, List, Optional
 
 from litellm._logging import verbose_logger
@@ -76,6 +77,7 @@ class SemanticMCPToolFilter:
         self.tool_router: Optional["SemanticRouter"] = None
         self.context_window_error: Optional[str] = None
         self._tool_map: Dict[str, Any] = {}  # MCPTool objects or OpenAI function dicts
+        self._index_sync_lock = asyncio.Lock()
 
     async def build_router_from_mcp_registry(self) -> None:
         """Build semantic router from all MCP tools in the registry (no auth checks)."""
@@ -182,6 +184,81 @@ class SemanticMCPToolFilter:
                 return
             raise
 
+    def _has_tools_missing_from_index(self, tools: list[Any]) -> bool:
+        """Allocation-free check for any named tool not yet in the semantic index."""
+        return any(name and name not in self._tool_map for name in (self._extract_tool_info(t)[0] for t in tools))
+
+    def _tools_missing_from_index(self, tools: list[Any]) -> dict[str, Any]:
+        """Map name -> tool for every named tool not yet in the semantic index."""
+        return {
+            name: tool
+            for name, tool in ((self._extract_tool_info(t)[0], t) for t in tools)
+            if name and name not in self._tool_map
+        }
+
+    async def _ensure_tools_indexed(self, available_tools: list[Any]) -> None:
+        """
+        Index request-time tools the startup build never saw.
+
+        The startup index lists every registered MCP server WITHOUT per-user
+        credentials, so servers requiring per-user auth (interactive OAuth
+        tokens, user-scoped env vars) contribute zero routes. Tools reaching
+        the filter came through an authenticated expansion; without indexing
+        them here they can never be selected, so requests either bypass
+        filtering entirely (N->N) or lose every tool to unrelated matches.
+
+        Runs async-only (no synchronous embedding on the request path) and
+        never writes shared error state: an embedding failure here raises and
+        is scoped to the requesting call, so one request's oversized tool
+        description cannot poison the filter for other users on the worker.
+        """
+        from semantic_router.routers import SemanticRouter
+        from semantic_router.routers.base import Route
+
+        from litellm.router_strategy.auto_router.litellm_encoder import (
+            LiteLLMRouterEncoder,
+        )
+
+        if not self._has_tools_missing_from_index(available_tools):
+            return
+
+        async with self._index_sync_lock:
+            missing = self._tools_missing_from_index(available_tools)
+            if not missing:
+                return
+
+            descriptions = {name: self._extract_tool_info(tool)[1] for name, tool in missing.items()}
+            routes = [
+                Route(
+                    name=name,
+                    description=description,
+                    utterances=[description],
+                    score_threshold=self.similarity_threshold,
+                )
+                for name, description in descriptions.items()
+            ]
+
+            if self.tool_router is None:
+                router = SemanticRouter(
+                    routes=[],
+                    encoder=LiteLLMRouterEncoder(
+                        litellm_router_instance=self.router_instance,
+                        model_name=self.embedding_model,
+                        score_threshold=self.similarity_threshold,
+                    ),
+                    auto_sync="local",
+                    top_k=self.top_k,
+                )
+                await router.aadd(routes)
+                self.tool_router = router
+            else:
+                await self.tool_router.aadd(routes)
+
+            self._tool_map.update(missing)
+            verbose_logger.info(
+                f"Semantic tool filter indexed {len(routes)} request-time tools missing from the startup index"
+            )
+
     async def filter_tools(
         self,
         query: str,
@@ -216,22 +293,34 @@ class SemanticMCPToolFilter:
         if not query or not query.strip():
             return available_tools
 
-        # Router should be built on startup - if not, something went wrong
-        if self.tool_router is None:
-            verbose_logger.warning("Router not initialized - was build_router_from_mcp_registry() called on startup?")
-            return available_tools
-
         # Run semantic filtering
         try:
+            await self._ensure_tools_indexed(available_tools)
+
+            if self.tool_router is None:
+                verbose_logger.warning("Semantic router could not be built from the request's tools")
+                return available_tools
+
+            available_names = [name for name in (self._extract_tool_info(t)[0] for t in available_tools) if name]
+            if not available_names:
+                return available_tools
+
             limit = top_k or self.top_k
-            matches = self.tool_router(text=query, limit=limit)
+            if self.tool_router.top_k < limit:
+                self.tool_router.top_k = limit
+            matches = self.tool_router(text=query, limit=limit, route_filter=available_names)
             matched_tool_names = self._extract_tool_names_from_matches(matches)
 
             if not matched_tool_names:
                 return available_tools
 
-            return self._get_tools_by_names(matched_tool_names, available_tools)
+            filtered_tools = self._get_tools_by_names(matched_tool_names, available_tools)
+            if not filtered_tools:
+                return available_tools
+            return filtered_tools
 
+        except SemanticToolFilterContextWindowError:
+            raise
         except Exception as e:
             if _is_context_window_error(e):
                 verbose_logger.error(
@@ -240,7 +329,7 @@ class SemanticMCPToolFilter:
                 )
                 raise SemanticToolFilterContextWindowError(
                     embedding_model=self.embedding_model,
-                    stage="the user query",
+                    stage="the user query or the MCP tool descriptions being indexed",
                     original_error=str(e),
                 ) from e
             verbose_logger.error(f"Semantic tool filter failed: {e}", exc_info=True)

litellm/proxy/_lazy_openapi_snapshot.json

@@ -7613,6 +7613,18 @@
               ],
               "title": "Messages"
             },
+            "metadata": {
+              "anyOf": [
+                {
+                  "additionalProperties": true,
+                  "type": "object"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Metadata"
+            },
             "text": {
               "title": "Text",
               "type": "string"

litellm/proxy/_types.py

@@ -1263,6 +1263,7 @@ class NewMCPServerRequest(LiteLLMPydanticObjectBase):
     command: Optional[str] = None
     args: List[str] = Field(default_factory=list)
     env: Dict[str, str] = Field(default_factory=dict)
+    issuer: Optional[str] = None
     authorization_url: Optional[str] = None
     token_url: Optional[str] = None
     registration_url: Optional[str] = None
@@ -1368,6 +1369,7 @@ class UpdateMCPServerRequest(LiteLLMPydanticObjectBase):
     command: Optional[str] = None
     args: List[str] = Field(default_factory=list)
     env: Dict[str, str] = Field(default_factory=dict)
+    issuer: Optional[str] = None
     authorization_url: Optional[str] = None
     token_url: Optional[str] = None
     registration_url: Optional[str] = None
@@ -2367,6 +2369,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
         None,
         description="If True, stores request messages and responses in spend logs. Default is False.",
     )
+    disable_auto_add_proxy_admin_to_teams: bool | None = Field(
+        None,
+        description="By default, the user calling /team/new is automatically added to the new team as a team admin. If True, proxy admins are no longer auto-added; members explicitly listed in members_with_roles are unaffected. Default is False.",
+    )
     maximum_spend_logs_retention_period: Optional[str] = Field(
         None,
         description="Maximum retention period for spend logs (e.g., '7d' for 7 days). Logs older than this will be deleted.",

litellm/proxy/auth/auth_checks.py

@@ -361,7 +361,11 @@ def _global_proxy_budget_check(global_proxy_spend: Optional[float], skip_budget_
         and route != "/models"
     ):
         if math.isfinite(litellm.max_budget) and global_proxy_spend > litellm.max_budget:
-            raise litellm.BudgetExceededError(current_cost=global_proxy_spend, max_budget=litellm.max_budget)
+            raise litellm.BudgetExceededError(
+                current_cost=global_proxy_spend,
+                max_budget=litellm.max_budget,
+                entity_type=Litellm_EntityType.PROXY.value,
+            )
 
 
 _GUARDRAIL_MODIFICATION_KEYS: tuple = (
@@ -648,6 +652,8 @@ async def common_checks(
                     current_cost=user_spend,
                     max_budget=user_budget,
                     message=f"ExceededBudget: User={user_object.user_id} over budget. Spend={user_spend}, Budget={user_budget}",
+                    entity_type=Litellm_EntityType.USER.value,
+                    entity_id=user_object.user_id,
                 )
 
         # Each scope reads a distinct counter key with no cross-scope ordering
@@ -1093,6 +1099,8 @@ async def _check_end_user_budget(
             current_cost=end_user_spend,
             max_budget=end_user_budget,
             message=f"ExceededBudget: End User={end_user_obj.user_id} over budget. Spend={end_user_spend}, Budget={end_user_budget}",
+            entity_type=Litellm_EntityType.END_USER.value,
+            entity_id=end_user_obj.user_id,
         )
 
 
@@ -3552,6 +3560,8 @@ async def _virtual_key_max_budget_check(
                 current_cost=spend,
                 max_budget=valid_token.max_budget,
                 message=f"Budget has been exceeded! Key={key_descriptor} Current cost: {spend}, Max budget: {valid_token.max_budget}",
+                entity_type=Litellm_EntityType.KEY.value,
+                entity_id=valid_token.token,
             )
 
 
@@ -3593,6 +3603,8 @@ async def _virtual_key_multi_budget_check(
                     f"ExceededBudget: Key over {w['budget_duration']} budget. "
                     f"Spend=${window_spend:.4f}, Limit=${w['max_budget']:.2f}"
                 ),
+                entity_type=Litellm_EntityType.KEY.value,
+                entity_id=valid_token.token,
             )
 
 
@@ -3824,6 +3836,8 @@ async def _check_team_member_budget(
                     current_cost=team_member_spend,
                     max_budget=team_member_budget,
                     message=f"Budget has been exceeded! User={valid_token.user_id} in Team={team_object.team_id} Current cost: {team_member_spend}, Max budget: {team_member_budget}",
+                    entity_type=Litellm_EntityType.TEAM_MEMBER.value,
+                    entity_id=f"{valid_token.user_id}:{team_object.team_id}",
                 )
 
 
@@ -3923,6 +3937,8 @@ async def _team_max_budget_check(
                 current_cost=spend,
                 max_budget=team_object.max_budget,
                 message=f"Budget has been exceeded! Team={team_object.team_id} Current cost: {spend}, Max budget: {team_object.max_budget}",
+                entity_type=Litellm_EntityType.TEAM.value,
+                entity_id=team_object.team_id,
             )
 
 
@@ -3960,6 +3976,8 @@ async def _team_multi_budget_check(
                     f"ExceededBudget: Team={team_object.team_id} over {w['budget_duration']} budget. "
                     f"Spend=${window_spend:.4f}, Limit=${w['max_budget']:.2f}"
                 ),
+                entity_type=Litellm_EntityType.TEAM.value,
+                entity_id=team_object.team_id,
             )
 
 
@@ -4081,6 +4099,8 @@ async def _project_max_budget_check(
             current_cost=project_object.spend,
             max_budget=max_budget,
             message=f"Budget has been exceeded! Project={project_object.project_id} Current cost: {project_object.spend}, Max budget: {max_budget}",
+            entity_type=Litellm_EntityType.PROJECT.value,
+            entity_id=project_object.project_id,
         )
 
 
@@ -4269,6 +4289,8 @@ async def _organization_max_budget_check(
             current_cost=org_spend,
             max_budget=org_max_budget,
             message=f"Budget has been exceeded! Organization={org_id} Current cost: {org_spend}, Max budget: {org_max_budget}",
+            entity_type=Litellm_EntityType.ORGANIZATION.value,
+            entity_id=org_id,
         )
 
 
@@ -4326,6 +4348,8 @@ async def _tag_max_budget_check(
                 current_cost=tag_spend,
                 max_budget=tag_object.litellm_budget_table.max_budget,
                 message=f"Budget has been exceeded! Tag={tag_name} Current cost: {tag_spend}, Max budget: {tag_object.litellm_budget_table.max_budget}",
+                entity_type=Litellm_EntityType.TAG.value,
+                entity_id=tag_name,
             )
 
 

litellm/proxy/auth/auth_utils.py

@@ -10,7 +10,7 @@ from fastapi import HTTPException, Request, status
 import litellm
 from litellm import Router, provider_list
 from litellm._logging import verbose_proxy_logger
-from litellm.constants import STANDARD_CUSTOMER_ID_HEADERS
+from litellm.constants import MINIMUM_CUSTOM_KEY_LENGTH, STANDARD_CUSTOMER_ID_HEADERS
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
 from litellm.litellm_core_utils.url_utils import SSRFError, validate_url
 from litellm.proxy._types import *
@@ -1533,4 +1533,6 @@ def get_model_from_request(
 
 
 def abbreviate_api_key(api_key: str) -> str:
+    if len(api_key) < MINIMUM_CUSTOM_KEY_LENGTH:
+        return "sk-..."
     return f"sk-...{api_key[-4:]}"

litellm/proxy/auth/user_api_key_auth.py

@@ -1797,6 +1797,8 @@ async def _user_api_key_auth_builder(
                                 raise litellm.BudgetExceededError(
                                     current_cost=team_member_spend,
                                     max_budget=team_member_budget,
+                                    entity_type=Litellm_EntityType.TEAM_MEMBER.value,
+                                    entity_id=f"{valid_token.user_id}:{valid_token.team_id}",
                                 )
 
             # Check 3. If token is expired

litellm/proxy/client/cli/README.md

@@ -471,6 +471,109 @@ The token minted by `lite login` is a short-lived, per-session agent credential,
 
 The credential is short-lived by design (default 24h, configurable via `LITELLM_CLI_JWT_EXPIRATION_HOURS`); run `lite login` again to refresh it, which also re-reads your latest team and user settings. It does not appear in the Keys UI and cannot be rotated or revoked mid-session. `lite auth print-token` (usable as Claude Code's `apiKeyHelper`) prints it while it's still fresh and fails once it expires -- there is no silent renewal, so a long-running session needs a fresh `lite login` once a day. `lite claude`, `lite codex`, and `lite opencode` work with it on a default deployment; `EXPERIMENTAL_UI_LOGIN` is not required. If you need a long-lived, rotatable key that shows up in the Keys UI, create a dedicated virtual key in the dashboard and pass it via `--api-key` or `LITELLM_PROXY_API_KEY` instead.
 
+### Route Every Claude Code Session Through the Proxy
+
+`lite claude` wraps a single invocation, but `lite up` goes further: it patches `~/.claude/settings.json`, Claude Code's own config file, so that every Claude Code session started afterward -- from any terminal, launched normally with just `claude`, no wrapper needed -- routes through your LiteLLM proxy. It sets `env.ANTHROPIC_BASE_URL` to the proxy URL and `apiKeyHelper` to a `lite auth print-token` invocation, drops any stray static `ANTHROPIC_API_KEY` so the helper-issued token wins, and leaves every other setting in the file untouched. It backs up the original file before patching it.
+
+Two things need to already be true: you've run `lite login`, since the apiKeyHelper depends on that stored token, and the proxy is already reachable, since `lite up` does not start one for you.
+
+```bash
+lite login
+litellm --config litellm/proxy/dev_config.yaml &
+lite up
+```
+
+`lite up` runs in the foreground and blocks. Press Ctrl-C to stop it, which restores the original settings file and exits. If the process is ever killed uncleanly instead -- `kill -9`, a crash -- the settings file is left patched, and `lite down` is the manual recovery path: run it at any later point to restore from the same backup.
+
+This is a one-time file patch and restore, not a live traffic interceptor. A Claude Code session already running before `lite up` started keeps whatever `ANTHROPIC_BASE_URL` and token it loaded at its own startup, and a session still running when `lite up` stops keeps routing through the proxy until it exits; only sessions *started* while the patch is in effect are affected, and only *new* sessions after a restore go back to Anthropic directly.
+
+Cursor is not supported: it has no equivalent file-based config to hot-patch this way, since its model routing lives in its own app storage and is configured through its GUI.
+
+### QA Complexity-Based Auto-Routing Against Your Real Proxy
+
+`lite autoroute` lets you try LiteLLM's complexity-based auto-routing -- picking a cheaper or more expensive model depending on how complex a prompt looks -- against models your key already has access to on your real, running proxy, without editing that proxy's `config.yaml` and without any real request ever bypassing it. It builds a second, throwaway proxy locally that forwards every request back to your real proxy, and points Claude Code at that local proxy for the duration of the session.
+
+#### Install the CLI
+
+`lite autoroute up` builds and runs a throwaway litellm proxy locally, so unlike the rest of this CLI it needs the proxy server runtime, not just the thin `litellm[cli]` client. Install `litellm[proxy]` (which ships the `lite` command too) with a single curl command -- no existing Python tooling required, `uv` is bootstrapped automatically if missing:
+
+```bash
+curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm/main/scripts/install.sh | sh
+```
+
+To QA an unreleased branch or commit instead of the latest PyPI release, set `LITELLM_CLI_REF`:
+
+```bash
+curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm/<branch-or-commit>/scripts/install.sh | \
+  LITELLM_CLI_REF=<branch-or-commit> sh
+```
+
+The thin `scripts/install-cli.sh` installs only `litellm[cli]`, which is enough for `lite login`, `lite claude`, and `lite up`, but not for `lite autoroute up`; running it against a `litellm[cli]` install fails fast with a message telling you to install the proxy runtime.
+
+Point the CLI at your real proxy and key before running any `lite model-groups` or `lite autoroute` command -- like every other command in this CLI, they read `LITELLM_PROXY_URL`/`LITELLM_PROXY_API_KEY` (or `--base-url`/`--api-key`), no `lite login` required:
+
+```bash
+export LITELLM_PROXY_URL=http://localhost:4000
+export LITELLM_PROXY_API_KEY=sk-...
+```
+
+#### List Your Accessible Model Groups
+
+```bash
+lite model-groups list [--format table|json]
+```
+
+Lists the model groups your key can reach on the proxy, via `/model_group/info`, along with each group's mode (`chat`, `embedding`, etc.) and per-token pricing. This is also what `lite autoroute configure` uses internally to discover what it can offer you.
+
+#### Configure the Auto-Router
+
+```bash
+lite autoroute configure
+```
+
+An interactive wizard. It runs the same model-group discovery as above, splits the results into chat-capable and embedding-capable pools, and asks you to assign one or more models from the chat pool to each of the four complexity tiers -- SIMPLE, MEDIUM, COMPLEX, REASONING. Each tier's picker is a type-to-filter fuzzy search (fzf-style) rather than a scrollable numbered list, so it stays usable even with hundreds of model groups: type a substring to narrow the list, tab to toggle a model into the selection, enter to confirm (assigning more than one model to a tier is exactly when this matters -- complexity_router picks randomly among a tier's pool per request, and adaptive mode specifically depends on having more than one candidate to choose from). From there it optionally offers: classifying prompt complexity with an LLM (again picked from your discovered pool) instead of the free built-in heuristic scorer, semantic keyword matching for tier assignment (needs an embedding model from the pool), and adaptive (bandit-based) selection layered on top of tiering.
+
+The wizard writes the result to `~/.litellm/autorouter/config.yaml` with `0600` permissions, since the file embeds your real proxy API key. Every model referenced anywhere in that config -- tier targets, the classifier model, the embedding model -- becomes its own `litellm_proxy/<model-name>` deployment whose `api_base` and `api_key` point back at your real proxy. That is the trick that keeps your real proxy's config untouched: every actual network call this generates, whether it is the routed completion, an LLM-classifier call, or an embedding call, forwards transparently through your real, already-running proxy with your real key.
+
+You do not need to tell Claude Code to request `autorouter` by name yourself: `lite autoroute up` also sets `ANTHROPIC_DEFAULT_SONNET_MODEL`, `ANTHROPIC_DEFAULT_HAIKU_MODEL`, and `ANTHROPIC_DEFAULT_OPUS_MODEL` to `autorouter` in `~/.claude/settings.json`, so every one of Claude Code's own model tiers requests it directly regardless of `/model` or whatever it defaults to otherwise. (A bare `model_name: "*"` deployment looks like the obvious way to catch any request instead, but litellm's Router looks up auto-router deployments by the literal requested model string with no wildcard resolution, so a `"*"` entry would never actually match real traffic -- these env var overrides are what makes it work.)
+
+You must run `configure` at least once before `up`; running `up` first fails with a clear error telling you to configure first.
+
+#### Launch the Ephemeral Auto-Router Proxy
+
+```bash
+lite autoroute up
+```
+
+Starts a local, throwaway litellm proxy on a random free port, running the config `configure` generated, with a freshly-minted random API key baked in for this session only (your real proxy key never leaves the generated config -- it only appears there, forwarding to your real proxy). It waits for the ephemeral proxy to report healthy, then patches `~/.claude/settings.json` the same way `lite up` does, except with a static `ANTHROPIC_AUTH_TOKEN` env var instead of an `apiKeyHelper`, since this key is short-lived and self-issued rather than something needing SSO refresh. Any `claude` session started afterward, from any terminal, routes through the ephemeral proxy.
+
+`lite autoroute up` runs in the foreground and streams the ephemeral proxy's own log file into your terminal, so you can watch its routing decisions -- which tier and model got picked for each request -- as you use Claude Code normally. Press Ctrl-C (or send SIGTERM) to stop it; this kills the child proxy process and restores your original Claude Code settings, in that order.
+
+#### Recover From an Unclean Shutdown
+
+```bash
+lite autoroute down
+```
+
+If the `lite autoroute up` process dies uncleanly -- `kill -9`, a crash -- rather than being stopped with Ctrl-C, `down` is the manual recovery path: it kills any leftover ephemeral proxy process found via a recorded pid file and restores Claude Code's settings from whatever backup is on disk.
+
+#### Example
+
+```bash
+lite autoroute configure
+lite autoroute up
+# use Claude Code as normal in another terminal; routing decisions stream live
+lite autoroute down   # only needed if `up` was killed uncleanly instead of Ctrl-C'd
+```
+
+#### Caveats
+
+Adaptive mode's learned state does not persist across `lite autoroute up` sessions -- there is no local database, so every session starts adaptive selection cold. A Claude Code session already running before `up` started, or still running when it stops, keeps whatever settings it loaded at its own startup; like `lite up`, this is a one-time file patch and restore, not a live traffic interceptor. Only Claude Code is supported, for the same reason as `lite up`: no other supported agent (for example Cursor) has an equivalent hot-patchable config file.
+
+A session that outlives `up` (or is still running the moment you stop it) keeps sending requests, master key included, to that now-freed loopback port until you restart it. Once the ephemeral proxy process exits, nothing stops another local account on the same machine from binding that same port and receiving those requests instead -- unlike `lite up`'s `apiKeyHelper`, which is re-resolved per request, `autoroute`'s master key is a static value, so whoever receives them gets a live-looking token along with the prompt content. Restart any Claude Code session before you consider the machine clean, run `lite autoroute down` promptly rather than leaving a stopped session's settings patched, and do not run `lite autoroute up` on a shared or multi-tenant host.
+
+Do not run `lite up` and `lite autoroute up` at the same time. Each patches `~/.claude/settings.json` and keeps its own separate backup, with no coordination between them: whichever one you stop or crash out of last is the one whose backup gets restored, which can silently leave the *other* mode's settings (a static master key and a now-dead loopback URL, or a stale `apiKeyHelper`) active. Run `lite down` or `lite autoroute down` (whichever applies) before switching to the other mode.
+
 ## Environment Variables
 
 The CLI respects the following environment variables:

litellm/proxy/client/cli/commands/agents.py

@@ -212,7 +212,7 @@ def _is_interactive() -> bool:
     return sys.stdin.isatty()
 
 
-def _resolve_api_key(ctx: click.Context) -> str:
+def resolve_api_key(ctx: click.Context) -> str:
     base_url = ctx.obj["base_url"]
     api_key = ctx.obj.get("api_key")
     if api_key:
@@ -238,7 +238,7 @@ _SKIP_VERIFY_HELP = "Skip the pre-launch key check against the proxy."
 def _launch(ctx: click.Context, binary: str, args: Sequence[str], *, skip_verify: bool) -> None:
     base_url = ctx.obj["base_url"]
     started_interactive = _is_interactive()
-    api_key = _resolve_api_key(ctx)
+    api_key = resolve_api_key(ctx)
 
     display_name, _ = agent_profile(binary)
     click.echo(f"litellm: routing {display_name} through proxy at {base_url.rstrip('/')}")
@@ -288,5 +288,6 @@ __all__ = [
     "agent_launch_args",
     "verify_proxy_key",
     "agent_profile",
+    "resolve_api_key",
     "AgentRunError",
 ]

litellm/proxy/client/cli/commands/auth.py

@@ -72,7 +72,7 @@ def display_teams_table(teams: List[Dict[str, Any]]) -> None:
     console = Console()
 
     if not teams:
-        console.print("❌ No teams found for your user.")
+        console.print("No teams found for your user.")
         return
 
     table = Table(title="Available Teams")
@@ -162,7 +162,7 @@ def display_interactive_team_selection(teams: List[Dict[str, Any]], selected_ind
     # Clear the screen using Rich's method
     console.clear()
 
-    console.print("🎯 Select a Team (Use ↑↓ arrows, Enter to select, 'q' to skip):\n")
+    console.print("Select a Team (Use up/down arrows, Enter to select, 'q' to skip):\n")
 
     for i, team in enumerate(teams):
         team_alias = team.get("team_alias") or "N/A"
@@ -184,7 +184,7 @@ def display_interactive_team_selection(teams: List[Dict[str, Any]], selected_ind
 
         # Highlight the selected item
         if i == selected_index:
-            console.print(f"➤ [bold cyan]{team_alias}[/bold cyan] ({team_id})")
+            console.print(f"> [bold cyan]{team_alias}[/bold cyan] ({team_id})")
             console.print(f"   Models: [yellow]{models_str}[/yellow]")
             console.print(f"   Budget: [blue]{budget_str}[/blue]\n")
         else:
@@ -220,15 +220,13 @@ def prompt_team_selection(teams: List[Dict[str, Any]]) -> Optional[Dict[str, Any
                 # Clear screen and show selection
                 console = Console()
                 console.clear()
-                click.echo(
-                    f"✅ Selected team: {selected_team.get('team_alias', 'N/A')} ({selected_team.get('team_id')})"
-                )
+                click.echo(f"Selected team: {selected_team.get('team_alias', 'N/A')} ({selected_team.get('team_id')})")
                 return selected_team
             elif key == "quit" or key == "escape":
                 # Clear screen
                 console = Console()
                 console.clear()
-                click.echo("ℹ️ Team selection skipped.")
+                click.echo("Team selection skipped.")
                 return None
             elif key is None:
                 # If we can't get key input, fall back to simple selection
@@ -237,7 +235,7 @@ def prompt_team_selection(teams: List[Dict[str, Any]]) -> Optional[Dict[str, Any
     except KeyboardInterrupt:
         console = Console()
         console.clear()
-        click.echo("\n❌ Team selection cancelled.")
+        click.echo("\nTeam selection cancelled.")
         return None
     except Exception:
         # If interactive mode fails, fall back to simple selection
@@ -265,15 +263,15 @@ def prompt_team_selection_fallback(
             if 0 <= index < len(teams):
                 selected_team = teams[index]
                 click.echo(
-                    f"\n✅ Selected team: {selected_team.get('team_alias', 'N/A')} ({selected_team.get('team_id')})"
+                    f"\nSelected team: {selected_team.get('team_alias', 'N/A')} ({selected_team.get('team_id')})"
                 )
                 return selected_team
             else:
-                click.echo(f"❌ Invalid selection. Please enter a number between 1 and {len(teams)}")
+                click.echo(f"Invalid selection. Please enter a number between 1 and {len(teams)}")
         except ValueError:
-            click.echo("❌ Invalid input. Please enter a number or 'skip'")
+            click.echo("Invalid input. Please enter a number or 'skip'")
         except KeyboardInterrupt:
-            click.echo("\n❌ Team selection cancelled.")
+            click.echo("\nTeam selection cancelled.")
             return None
 
 
@@ -437,7 +435,7 @@ def _poll_for_authentication(base_url: str, key_id: str, poll_secret: str) -> Op
         user_id = data.get("user_id")
         normalized_teams: List[Dict[str, Any]] = _normalize_teams(teams, team_details)
         if not normalized_teams:
-            click.echo("⚠️ No teams available for selection.")
+            click.echo("Warning: No teams available for selection.")
             return None
 
         # User has multiple teams - let them select
@@ -457,7 +455,7 @@ def _poll_for_authentication(base_url: str, key_id: str, poll_secret: str) -> Op
                 "team_id": None,  # Set by server in JWT
             }
 
-        click.echo("❌ Team selection cancelled or JWT generation failed.")
+        click.echo("Team selection cancelled or JWT generation failed.")
         return None
 
     # JWT is ready (single team or team already selected)
@@ -468,7 +466,7 @@ def _poll_for_authentication(base_url: str, key_id: str, poll_secret: str) -> Op
 
     # Show which team was assigned
     if team_id and len(teams) == 1:
-        click.echo(f"\n✅ Automatically assigned to team: {team_id}")
+        click.echo(f"\nAutomatically assigned to team: {team_id}")
 
     if api_key:
         return {
@@ -494,19 +492,19 @@ def _handle_team_selection_during_polling(
         The JWT token with the selected team, or None if selection was skipped
     """
     if not teams:
-        click.echo("ℹ️ No teams found. You can create or join teams using the web interface.")
+        click.echo("No teams found. You can create or join teams using the web interface.")
         return None
 
     click.echo("\n" + "=" * 60)
-    click.echo("📋 Select a team for your CLI session...")
+    click.echo("Select a team for your CLI session...")
 
     team_id = _render_and_prompt_for_team_selection(teams)
 
     if not team_id:
-        click.echo("ℹ️ No team selected.")
+        click.echo("No team selected.")
         return None
 
-    click.echo(f"\n🔄 Generating JWT for team: {team_id}")
+    click.echo(f"\nGenerating JWT for team: {team_id}")
 
     poll_url = f"{base_url}/sso/cli/poll/{key_id}?team_id={team_id}"
     data = _poll_for_ready_data(
@@ -520,7 +518,7 @@ def _handle_team_selection_during_polling(
         return None
     jwt_token = data.get("key")
     if jwt_token:
-        click.echo(f"✅ Successfully generated JWT for team: {team_id}")
+        click.echo(f"Successfully generated JWT for team: {team_id}")
         return jwt_token
 
     return None
@@ -568,14 +566,14 @@ def _render_and_prompt_for_team_selection(teams: List[Dict[str, Any]]) -> Option
                 selected_team = teams[index]
                 team_id = str(selected_team.get("team_id"))
                 team_alias = selected_team.get("team_alias") or team_id
-                click.echo(f"\n✅ Selected team: {team_alias} ({team_id})")
+                click.echo(f"\nSelected team: {team_alias} ({team_id})")
                 return team_id
 
-            click.echo(f"❌ Invalid selection. Please enter a number between 1 and {len(teams)}")
+            click.echo(f"Invalid selection. Please enter a number between 1 and {len(teams)}")
         except ValueError:
-            click.echo("❌ Invalid input. Please enter a number or 'skip'")
+            click.echo("Invalid input. Please enter a number or 'skip'")
         except KeyboardInterrupt:
-            click.echo("\n❌ Team selection cancelled.")
+            click.echo("\nTeam selection cancelled.")
             return None
 
 
@@ -628,7 +626,7 @@ def login(ctx: click.Context):
                 }
             )
 
-            click.echo("\n✅ Login successful!")
+            click.echo("\nLogin successful!")
             click.echo(f"JWT Token: {api_key[:20]}...")
             click.echo("You can now use the CLI without specifying --api-key")
 
@@ -637,7 +635,7 @@ def login(ctx: click.Context):
             show_commands()
             return
         else:
-            click.echo("❌ Authentication timed out. Please try again.")
+            click.echo("Authentication timed out. Please try again.")
             click.echo(
                 "The proxy never reported the browser sign-in as finished. If you did complete it, "
                 "check the proxy logs for /sso/callback errors and confirm SSO is configured on the proxy."
@@ -645,10 +643,10 @@ def login(ctx: click.Context):
             return
 
     except KeyboardInterrupt:
-        click.echo("\n❌ Authentication cancelled by user.")
+        click.echo("\nAuthentication cancelled by user.")
         return
     except Exception as e:
-        click.echo(f"❌ Authentication failed: {e}")
+        click.echo(f"Authentication failed: {e}")
         return
 
 
@@ -656,7 +654,7 @@ def login(ctx: click.Context):
 def logout():
     """Logout and clear stored authentication"""
     clear_token()
-    click.echo("✅ Logged out successfully. Authentication token cleared.")
+    click.echo("Logged out successfully. Authentication token cleared.")
 
 
 @click.command(name="print-token")
@@ -703,10 +701,10 @@ def whoami():
     token_data = load_token()
 
     if not token_data:
-        click.echo("❌ Not authenticated. Run 'lite login' to authenticate.")
+        click.echo("Not authenticated. Run 'lite login' to authenticate.")
         return
 
-    click.echo("✅ Authenticated")
+    click.echo("Authenticated")
     click.echo(f"User Email: {token_data.get('user_email', 'Unknown')}")
     click.echo(f"User ID: {token_data.get('user_id', 'Unknown')}")
     click.echo(f"User Role: {token_data.get('user_role', 'Unknown')}")
@@ -717,7 +715,7 @@ def whoami():
     click.echo(f"Token age: {age_hours:.1f} hours")
 
     if age_hours > CLI_JWT_EXPIRATION_HOURS:
-        click.echo(f"⚠️ Warning: Token is more than {CLI_JWT_EXPIRATION_HOURS} hours old and may have expired.")
+        click.echo(f"Warning: Token is more than {CLI_JWT_EXPIRATION_HOURS} hours old and may have expired.")
 
 
 @click.group(name="auth")

litellm/proxy/client/cli/commands/autoroute/commands.py

@@ -0,0 +1,207 @@
+import atexit
+import json
+import secrets
+import signal
+import threading
+from types import FrameType
+
+import click
+import yaml
+from pydantic import JsonValue, TypeAdapter, ValidationError
+
+from ..up import CLAUDE_SETTINGS_PATH, UpError, load_json_or_empty, restore_claude_settings, write_backup
+from ..up import BackupRecord as ClaudeBackupRecord
+from .process import (
+    AUTOROUTE_DIR,
+    CONFIG_PATH,
+    LOG_PATH,
+    PidRecord,
+    ProcessLaunchError,
+    allocate_free_port,
+    clear_pid_record,
+    is_running,
+    launch_proxy,
+    missing_proxy_runtime_modules,
+    poll_liveliness,
+    read_pid_record,
+    secure_create,
+    stream_log,
+    terminate,
+    write_pid_record,
+)
+from .settings import merge_claude_settings_static_token
+from .wizard import run_configure_wizard
+
+AUTOROUTE_BACKUP_PATH = AUTOROUTE_DIR / "claude_settings_backup.json"
+
+_GENERATED_CONFIG_ADAPTER = TypeAdapter(dict[str, JsonValue])
+
+
+def _mint_and_embed_master_key() -> str:
+    """Generate a fresh key for this session and write it into the generated config.yaml.
+
+    Must go under general_settings, not litellm_settings -- the proxy server only ever
+    reads general_settings.master_key (proxy_server.py:4530) to authenticate requests. A
+    key placed under litellm_settings is silently ignored, leaving the ephemeral proxy with
+    no real auth: any request reaches it regardless of the token Claude Code sends.
+    """
+    master_key = secrets.token_urlsafe(32)
+    with open(CONFIG_PATH, "r") as f:
+        try:
+            generated = _GENERATED_CONFIG_ADAPTER.validate_python(yaml.safe_load(f))
+        except (yaml.YAMLError, ValidationError):
+            raise click.ClickException(
+                f"{CONFIG_PATH} is empty or corrupt. Run `lite autoroute configure` again to regenerate it."
+            )
+    general_settings = generated.get("general_settings")
+    updated_settings: dict[str, JsonValue] = {
+        **(general_settings if isinstance(general_settings, dict) else {}),
+        "master_key": master_key,
+    }
+    updated: dict[str, JsonValue] = {**generated, "general_settings": updated_settings}
+    with secure_create(CONFIG_PATH) as f:
+        yaml.safe_dump(updated, f, sort_keys=False)
+    return master_key
+
+
+@click.group(name="autoroute")
+def autoroute_group() -> None:
+    """QA complexity-based auto-routing against models your key can already use"""
+
+
+@autoroute_group.command("configure")
+@click.pass_context
+def configure(ctx: click.Context) -> None:
+    """Discover accessible models and generate an ephemeral auto-router config"""
+    run_configure_wizard(ctx)
+
+
+@autoroute_group.command("up")
+def up() -> None:
+    """Launch the ephemeral auto-router proxy and route Claude Code through it"""
+    if not CONFIG_PATH.exists():
+        raise click.ClickException("No config found. Run `lite autoroute configure` first.")
+
+    missing = missing_proxy_runtime_modules()
+    if missing:
+        raise click.ClickException(
+            "lite autoroute up launches a local litellm proxy, which needs the proxy runtime that the "
+            f"thin `litellm[cli]` install does not include (missing: {', '.join(missing)}). Install the "
+            "proxy runtime with `uv tool install --force 'litellm[proxy]'`, or to QA a branch, "
+            "`curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm/<branch>/scripts/install.sh | "
+            "LITELLM_CLI_REF=<branch> sh`."
+        )
+
+    try:
+        existing_pid = read_pid_record()
+    except UpError as e:
+        raise click.ClickException(str(e))
+    if existing_pid is not None and is_running(existing_pid.pid):
+        raise click.ClickException(
+            "An ephemeral proxy is already running (lite autoroute up looks already active). "
+            "Run `lite autoroute down` first."
+        )
+
+    if AUTOROUTE_BACKUP_PATH.exists():
+        raise click.ClickException(
+            f"{AUTOROUTE_BACKUP_PATH} already exists -- `lite autoroute up` looks like it's already "
+            "running (or crashed without cleanup). Run `lite autoroute down` first."
+        )
+
+    master_key = _mint_and_embed_master_key()
+    port = allocate_free_port()
+    base_url = f"http://127.0.0.1:{port}"
+    process = launch_proxy(CONFIG_PATH, port, LOG_PATH)
+    write_pid_record(PidRecord(pid=process.pid, port=port, config_path=str(CONFIG_PATH), log_path=str(LOG_PATH)))
+
+    try:
+        poll_liveliness(base_url, LOG_PATH, process)
+    except ProcessLaunchError as e:
+        terminate(process.pid)
+        clear_pid_record()
+        raise click.ClickException(str(e))
+
+    try:
+        original_existed = CLAUDE_SETTINGS_PATH.exists()
+        original_settings = load_json_or_empty(CLAUDE_SETTINGS_PATH)
+        write_backup(
+            ClaudeBackupRecord(existed=original_existed, content=original_settings if original_existed else None),
+            AUTOROUTE_BACKUP_PATH,
+        )
+        merged = merge_claude_settings_static_token(original_settings, base_url, master_key)
+        CLAUDE_SETTINGS_PATH.parent.mkdir(parents=True, exist_ok=True)
+        with secure_create(CLAUDE_SETTINGS_PATH) as f:
+            json.dump(merged, f, indent=2)
+    except UpError as e:
+        terminate(process.pid)
+        clear_pid_record()
+        raise click.ClickException(str(e))
+
+    click.echo(f"litellm: ephemeral auto-router proxy up at {base_url} (pid {process.pid})")
+    click.echo("Claude Code sessions started now will route through it. Press Ctrl-C to stop and restore.")
+
+    stop_event = threading.Event()
+    restored = threading.Lock()
+
+    def _teardown() -> None:
+        if not restored.acquire(blocking=False):
+            return
+        terminate(process.pid)
+        clear_pid_record()
+        try:
+            restore_claude_settings(CLAUDE_SETTINGS_PATH, AUTOROUTE_BACKUP_PATH)
+        except UpError as e:
+            # Runs from atexit/a signal handler too, outside Click's own exception
+            # handling -- raising here would only produce an unhandled-exception
+            # warning on stderr, not a clean message.
+            click.echo(str(e), err=True)
+            return
+        click.echo("\nStopped ephemeral proxy and restored Claude Code settings.")
+        click.echo(
+            f"Restart any Claude Code session still open from this session, or another local account could "
+            f"bind the now-free port {port} and receive its requests. Do not use `lite autoroute up` on a "
+            f"shared or multi-tenant host."
+        )
+
+    def _handle_signal(_signum: int, _frame: FrameType | None) -> None:
+        stop_event.set()
+
+    signal.signal(signal.SIGINT, _handle_signal)
+    signal.signal(signal.SIGTERM, _handle_signal)
+    atexit.register(_teardown)
+
+    log_thread = threading.Thread(target=stream_log, args=(LOG_PATH, stop_event), daemon=True)
+    log_thread.start()
+
+    stop_event.wait()
+    _teardown()
+
+
+@autoroute_group.command("down")
+def down() -> None:
+    """Restore Claude Code settings and stop a leftover ephemeral proxy, if any"""
+    try:
+        record: PidRecord | None = read_pid_record()
+    except UpError as e:
+        # down is the crash-recovery path -- a corrupt pid record must not block it; clear the
+        # unusable record and keep going rather than leaving the user with no way to clean up.
+        click.echo(f"{e} Clearing it and continuing cleanup.", err=True)
+        record = None
+    if record is not None and is_running(record.pid):
+        terminate(record.pid)
+        click.echo(f"Stopped leftover ephemeral proxy (pid {record.pid}).")
+    clear_pid_record()
+
+    try:
+        restored = restore_claude_settings(CLAUDE_SETTINGS_PATH, AUTOROUTE_BACKUP_PATH)
+    except UpError as e:
+        raise click.ClickException(str(e))
+    if restored is None:
+        click.echo("Nothing to restore.")
+    elif restored.existed:
+        click.echo(f"Restored {CLAUDE_SETTINGS_PATH} to its original contents.")
+    else:
+        click.echo(f"Removed {CLAUDE_SETTINGS_PATH} (it did not exist before `lite autoroute up`).")
+
+
+__all__ = ["autoroute_group"]

litellm/proxy/client/cli/commands/autoroute/config.py

@@ -0,0 +1,249 @@
+from typing import Literal, Union
+
+from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter
+
+TIER_NAMES: tuple[str, ...] = ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING")
+AUTOROUTER_MODEL_NAME = "autorouter"
+
+
+class ConfigGenerationError(Exception):
+    """Raised when an AutorouteConfig references a model the discovery step didn't find."""
+
+
+class DiscoveredModel(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    name: str
+    mode: str = "chat"
+    input_cost_per_token: float | None = None
+    output_cost_per_token: float | None = None
+
+
+class _RawModelGroup(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+
+    model_group: str
+    # Optional: some real deployments return an explicit `"mode": null` for models that
+    # were registered without a mode (seen for embedding models like voyage-4-large).
+    # ModelGroupInfo's own "chat" default (litellm/types/router.py) only applies when the
+    # key is missing entirely, not when it's present as null, so this must tolerate None.
+    mode: str | None = "chat"
+    input_cost_per_token: float | None = None
+    output_cost_per_token: float | None = None
+
+
+_RAW_MODEL_GROUPS_ADAPTER = TypeAdapter(list[_RawModelGroup])
+
+
+def parse_discovered_models(raw: list[JsonValue]) -> tuple[DiscoveredModel, ...]:
+    """Validate a raw `/model_group/info` response into typed models."""
+    parsed = _RAW_MODEL_GROUPS_ADAPTER.validate_python(raw)
+    return tuple(
+        DiscoveredModel(
+            name=group.model_group,
+            # A null mode means the server genuinely doesn't know what this model does;
+            # "unknown" (rather than guessing "chat") keeps it out of both chat_models()
+            # and embedding_models() instead of risking a wrong-mode deployment.
+            mode=group.mode or "unknown",
+            input_cost_per_token=group.input_cost_per_token,
+            output_cost_per_token=group.output_cost_per_token,
+        )
+        for group in parsed
+    )
+
+
+def chat_models(models: tuple[DiscoveredModel, ...]) -> tuple[DiscoveredModel, ...]:
+    return tuple(m for m in models if m.mode == "chat")
+
+
+def embedding_models(models: tuple[DiscoveredModel, ...]) -> tuple[DiscoveredModel, ...]:
+    return tuple(m for m in models if m.mode == "embedding")
+
+
+class HeuristicClassifier(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    kind: Literal["heuristic"] = "heuristic"
+
+
+class LLMClassifier(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    kind: Literal["llm"] = "llm"
+    model: str
+    timeout_ms: int = 3000
+
+
+ClassifierChoice = Union[HeuristicClassifier, LLMClassifier]
+
+
+class NoSemanticMatching(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    kind: Literal["none"] = "none"
+
+
+class KeywordTierRule(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    keywords: tuple[str, ...]
+    tier: str
+
+
+# Satisfies complexity_router's "semantic matching requires non-empty keyword_tier_rules"
+# invariant with a sane starting point; the wizard lets the user override these per tier.
+DEFAULT_KEYWORD_TIER_RULES: tuple[KeywordTierRule, ...] = (
+    KeywordTierRule(keywords=("hi", "hello", "thanks"), tier="SIMPLE"),
+    KeywordTierRule(keywords=("explain", "how does"), tier="MEDIUM"),
+    KeywordTierRule(keywords=("refactor", "implement", "debug"), tier="COMPLEX"),
+    KeywordTierRule(keywords=("step by step", "think through", "prove"), tier="REASONING"),
+)
+
+
+class SemanticMatching(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    kind: Literal["semantic"] = "semantic"
+    embedding_model: str
+    match_threshold: float = 0.5
+    keyword_tier_rules: tuple[KeywordTierRule, ...] = DEFAULT_KEYWORD_TIER_RULES
+
+
+SemanticMatchingChoice = Union[NoSemanticMatching, SemanticMatching]
+
+
+class AutorouteConfig(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    base_url: str
+    api_key: str
+    # Each tier maps to a pool of one or more models; complexity_router picks randomly among
+    # them per request (or, in adaptive mode, learns which to prefer within the pool).
+    tiers: dict[str, tuple[str, ...]]
+    default_model: str
+    classifier: ClassifierChoice = Field(default_factory=HeuristicClassifier)
+    semantic_matching: SemanticMatchingChoice = Field(default_factory=NoSemanticMatching)
+    adaptive: bool = False
+
+
+def validate_config(config: AutorouteConfig, discovered: tuple[DiscoveredModel, ...]) -> None:
+    """Raise ConfigGenerationError if config references a model discovery didn't return."""
+    chat_names: frozenset[str] = frozenset(m.name for m in chat_models(discovered))
+    embedding_names: frozenset[str] = frozenset(m.name for m in embedding_models(discovered))
+
+    for tier, models in config.tiers.items():
+        for model in models:
+            if model not in chat_names:
+                raise ConfigGenerationError(f"Tier {tier} references unknown chat model '{model}'")
+
+    if config.default_model not in chat_names:
+        raise ConfigGenerationError(f"default_model '{config.default_model}' is not a known chat model")
+
+    if isinstance(config.classifier, LLMClassifier) and config.classifier.model not in chat_names:
+        raise ConfigGenerationError(f"classifier model '{config.classifier.model}' is not a known chat model")
+
+    if (
+        isinstance(config.semantic_matching, SemanticMatching)
+        and config.semantic_matching.embedding_model not in embedding_names
+    ):
+        raise ConfigGenerationError(
+            f"embedding model '{config.semantic_matching.embedding_model}' is not a known embedding model"
+        )
+
+
+def _litellm_proxy_deployment(name: str, base_url: str, api_key: str) -> dict[str, JsonValue]:
+    return {
+        "model_name": name,
+        "litellm_params": {
+            "model": f"litellm_proxy/{name}",
+            "api_base": base_url,
+            "api_key": api_key,
+        },
+    }
+
+
+def build_generated_model_list(config: AutorouteConfig) -> list[JsonValue]:
+    """Build the model_list for the ephemeral proxy's config.yaml.
+
+    Every real model referenced anywhere (tier targets, classifier, embedding) is deduplicated
+    to exactly one `litellm_proxy/<name>` deployment forwarding to the customer's real proxy,
+    plus one `auto_router/complexity_router` deployment tying the tiers together.
+    """
+    referenced_names = {model for models in config.tiers.values() for model in models}
+    referenced_names.add(config.default_model)
+    if isinstance(config.classifier, LLMClassifier):
+        referenced_names.add(config.classifier.model)
+    if isinstance(config.semantic_matching, SemanticMatching):
+        referenced_names.add(config.semantic_matching.embedding_model)
+
+    proxy_deployments = [
+        _litellm_proxy_deployment(name, config.base_url, config.api_key) for name in sorted(referenced_names)
+    ]
+
+    complexity_router_config: dict[str, JsonValue] = {
+        "tiers": {tier: list(models) for tier, models in config.tiers.items()},
+        "default_model": config.default_model,
+    }
+    if isinstance(config.classifier, LLMClassifier):
+        complexity_router_config["classifier_type"] = "llm"
+        complexity_router_config["classifier_llm_config"] = {
+            "model": config.classifier.model,
+            "timeout_ms": config.classifier.timeout_ms,
+        }
+    if isinstance(config.semantic_matching, SemanticMatching):
+        complexity_router_config["semantic_keyword_matching"] = True
+        complexity_router_config["embedding_model"] = config.semantic_matching.embedding_model
+        complexity_router_config["match_threshold"] = config.semantic_matching.match_threshold
+        complexity_router_config["keyword_tier_rules"] = [
+            {"keywords": list(rule.keywords), "tier": rule.tier} for rule in config.semantic_matching.keyword_tier_rules
+        ]
+    if config.adaptive:
+        complexity_router_config["adaptive"] = True
+
+    auto_router_litellm_params: dict[str, JsonValue] = {
+        "model": "auto_router/complexity_router",
+        "complexity_router_config": complexity_router_config,
+    }
+    # A bare "*" model_name looks like the obvious way to catch every request Claude Code
+    # might send regardless of which model it thinks it's using, but Router's auto-router
+    # registry is keyed by the literal requested model string (router.py:10711-10717), not
+    # resolved through pattern/wildcard matching first -- so a "*" entry here would only ever
+    # match a client that literally sends model="*", never an actual wildcard catch-all. Callers
+    # instead need to make Claude Code request this "autorouter" name directly (see
+    # ANTHROPIC_DEFAULT_*_MODEL in settings.py's merge_claude_settings_static_token).
+    return [
+        *proxy_deployments,
+        {"model_name": AUTOROUTER_MODEL_NAME, "litellm_params": auto_router_litellm_params},
+    ]
+
+
+def build_generated_proxy_config(config: AutorouteConfig, master_key: str) -> dict[str, JsonValue]:
+    """Full config.yaml content for the ephemeral proxy, including its own auth key.
+
+    master_key must live under general_settings, not litellm_settings -- the proxy server
+    only ever reads general_settings.master_key (proxy_server.py:4530) to authenticate
+    requests; a key placed under litellm_settings is silently ignored, leaving the proxy
+    with no real auth at all.
+    """
+    return {
+        "model_list": build_generated_model_list(config),
+        "general_settings": {"master_key": master_key},
+    }
+
+
+__all__ = [
+    "AUTOROUTER_MODEL_NAME",
+    "TIER_NAMES",
+    "AutorouteConfig",
+    "ClassifierChoice",
+    "ConfigGenerationError",
+    "DEFAULT_KEYWORD_TIER_RULES",
+    "DiscoveredModel",
+    "HeuristicClassifier",
+    "KeywordTierRule",
+    "LLMClassifier",
+    "NoSemanticMatching",
+    "SemanticMatching",
+    "SemanticMatchingChoice",
+    "build_generated_model_list",
+    "build_generated_proxy_config",
+    "chat_models",
+    "embedding_models",
+    "parse_discovered_models",
+    "validate_config",
+]

litellm/proxy/client/cli/commands/autoroute/process.py

@@ -0,0 +1,190 @@
+import contextlib
+import importlib.util
+import json
+import os
+import signal
+import socket
+import subprocess
+import sys
+import threading
+import time
+from dataclasses import dataclass
+from pathlib import Path
+
+import click
+import requests
+from pydantic import TypeAdapter, ValidationError
+
+from ..up import UpError, secure_create
+
+AUTOROUTE_DIR = Path.home() / ".litellm" / "autorouter"
+CONFIG_PATH = AUTOROUTE_DIR / "config.yaml"
+LOG_PATH = AUTOROUTE_DIR / "proxy.log"
+PID_RECORD_PATH = AUTOROUTE_DIR / "proxy.pid.json"
+
+
+class ProcessLaunchError(Exception):
+    """Raised when the ephemeral proxy subprocess fails to come up healthy."""
+
+
+@dataclass(frozen=True, slots=True)
+class PidRecord:
+    pid: int
+    port: int
+    config_path: str
+    log_path: str
+
+
+_PID_RECORD_ADAPTER = TypeAdapter(PidRecord)
+
+
+_PROXY_RUNTIME_MODULES: tuple[str, ...] = ("fastapi", "uvicorn", "backoff", "orjson", "websockets", "apscheduler")
+
+
+def missing_proxy_runtime_modules() -> tuple[str, ...]:
+    """Proxy-server modules that ``lite autoroute up`` needs but the thin CLI install lacks.
+
+    ``launch_proxy`` runs the full ``litellm.proxy.proxy_cli`` server, whose dependencies live in
+    the ``proxy`` extra, not the ``cli`` extra that installs the ``lite`` command. On a thin
+    ``litellm[cli]`` install the subprocess dies with a bare ``ModuleNotFoundError``; detecting the
+    gap here lets ``up`` fail with an actionable message instead.
+    """
+    return tuple(name for name in _PROXY_RUNTIME_MODULES if importlib.util.find_spec(name) is None)
+
+
+def allocate_free_port() -> int:
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+        sock.bind(("127.0.0.1", 0))
+        return int(sock.getsockname()[1])
+
+
+def launch_proxy(config_path: Path, port: int, log_path: Path) -> "subprocess.Popen[bytes]":
+    log_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(log_path, "w") as log_file:
+        return subprocess.Popen(
+            [
+                sys.executable,
+                "-m",
+                "litellm.proxy.proxy_cli",
+                "--config",
+                str(config_path),
+                "--port",
+                str(port),
+                "--host",
+                "127.0.0.1",
+            ],
+            stdout=log_file,
+            stderr=subprocess.STDOUT,
+        )
+
+
+def _tail(log_path: Path, lines: int = 40) -> str:
+    if not log_path.exists():
+        return "(no log output captured)"
+    return "\n".join(log_path.read_text(errors="replace").splitlines()[-lines:])
+
+
+def poll_liveliness(base_url: str, log_path: Path, process: "subprocess.Popen[bytes]", timeout: float = 30.0) -> None:
+    """Poll /health/liveliness until it responds, the process dies, or timeout elapses."""
+    deadline = time.monotonic() + timeout
+    url = base_url.rstrip("/") + "/health/liveliness"
+    while time.monotonic() < deadline:
+        if process.poll() is not None:
+            raise ProcessLaunchError(
+                f"Ephemeral proxy exited early (code {process.returncode}). Last log lines:\n{_tail(log_path)}"
+            )
+        with contextlib.suppress(requests.RequestException):
+            if requests.get(url, timeout=2).status_code == 200:
+                return
+        time.sleep(0.5)
+    raise ProcessLaunchError(
+        f"Ephemeral proxy never became healthy within {timeout}s. Last log lines:\n{_tail(log_path)}"
+    )
+
+
+def write_pid_record(record: PidRecord, path: Path | None = None) -> None:
+    resolved_path = path if path is not None else PID_RECORD_PATH
+    resolved_path.parent.mkdir(parents=True, exist_ok=True)
+    with open(resolved_path, "w") as f:
+        json.dump(
+            {"pid": record.pid, "port": record.port, "config_path": record.config_path, "log_path": record.log_path},
+            f,
+            indent=2,
+        )
+
+
+def read_pid_record(path: Path | None = None) -> PidRecord | None:
+    resolved_path = path if path is not None else PID_RECORD_PATH
+    if not resolved_path.exists():
+        return None
+    with open(resolved_path, "r") as f:
+        content = f.read()
+    try:
+        return _PID_RECORD_ADAPTER.validate_json(content)
+    except ValidationError:
+        raise UpError(f"{resolved_path} contains invalid or unexpected JSON; cannot proceed safely.")
+
+
+def clear_pid_record(path: Path | None = None) -> None:
+    resolved_path = path if path is not None else PID_RECORD_PATH
+    resolved_path.unlink(missing_ok=True)
+
+
+def is_running(pid: int) -> bool:
+    try:
+        os.kill(pid, 0)
+    except ProcessLookupError:
+        return False
+    except PermissionError:
+        return True
+    return True
+
+
+def terminate(pid: int, grace_period: float = 5.0) -> None:
+    """Terminate a process by pid, escalating from SIGTERM to SIGKILL if needed."""
+    if not is_running(pid):
+        return
+    with contextlib.suppress(ProcessLookupError):
+        os.kill(pid, signal.SIGTERM)
+    deadline = time.monotonic() + grace_period
+    while time.monotonic() < deadline and is_running(pid):
+        time.sleep(0.2)
+    if is_running(pid):
+        with contextlib.suppress(ProcessLookupError):
+            os.kill(pid, signal.SIGKILL)
+
+
+def stream_log(log_path: Path, stop_event: threading.Event) -> None:
+    """Print new lines appended to log_path until stop_event is set. Blocks the calling thread."""
+    while not log_path.exists() and not stop_event.is_set():
+        time.sleep(0.1)
+    if stop_event.is_set() or not log_path.exists():
+        return
+    with open(log_path, "r") as f:
+        while not stop_event.is_set():
+            line = f.readline()
+            if line:
+                click.echo(line, nl=False)
+            else:
+                time.sleep(0.2)
+
+
+__all__ = [
+    "AUTOROUTE_DIR",
+    "CONFIG_PATH",
+    "LOG_PATH",
+    "PID_RECORD_PATH",
+    "PidRecord",
+    "ProcessLaunchError",
+    "allocate_free_port",
+    "clear_pid_record",
+    "is_running",
+    "launch_proxy",
+    "missing_proxy_runtime_modules",
+    "poll_liveliness",
+    "read_pid_record",
+    "secure_create",
+    "stream_log",
+    "terminate",
+    "write_pid_record",
+]

litellm/proxy/client/cli/commands/autoroute/settings.py

@@ -0,0 +1,46 @@
+from pydantic import JsonValue
+
+from .config import AUTOROUTER_MODEL_NAME
+
+ENV_KEY = "env"
+API_KEY_HELPER_KEY = "apiKeyHelper"
+ANTHROPIC_API_KEY_KEY = "ANTHROPIC_API_KEY"
+ANTHROPIC_AUTH_TOKEN_KEY = "ANTHROPIC_AUTH_TOKEN"
+ANTHROPIC_BASE_URL_KEY = "ANTHROPIC_BASE_URL"
+# Force every one of Claude Code's own model tiers to request the auto-router by name.
+# Router's auto-router registry is keyed by the literal requested model string
+# (litellm/router.py:10711-10717) with no wildcard/pattern resolution, so a bare "*"
+# model_name can never work as a catch-all -- these overrides are what actually makes
+# Claude Code send "autorouter" regardless of /model or its own version-specific defaults.
+ANTHROPIC_DEFAULT_MODEL_ENV_KEYS = (
+    "ANTHROPIC_DEFAULT_SONNET_MODEL",
+    "ANTHROPIC_DEFAULT_HAIKU_MODEL",
+    "ANTHROPIC_DEFAULT_OPUS_MODEL",
+)
+
+
+def merge_claude_settings_static_token(
+    settings: dict[str, JsonValue], base_url: str, auth_token: str
+) -> dict[str, JsonValue]:
+    """Return a new settings dict wired to a local ephemeral proxy with a static token.
+
+    Unlike up.py's merge_claude_settings (which sets apiKeyHelper for a long-lived, real
+    remote proxy needing refreshable SSO tokens), this proxy is ephemeral and its key was just
+    minted for this session, so a plain env var is simpler and correct. Any existing
+    apiKeyHelper is cleared so it can't fight with the static token.
+    """
+    raw_env = settings.get(ENV_KEY, {})
+    base_env = raw_env if isinstance(raw_env, dict) else {}
+    env: dict[str, JsonValue] = {
+        **base_env,
+        ANTHROPIC_BASE_URL_KEY: base_url.rstrip("/"),
+        ANTHROPIC_AUTH_TOKEN_KEY: auth_token,
+        **{key: AUTOROUTER_MODEL_NAME for key in ANTHROPIC_DEFAULT_MODEL_ENV_KEYS},
+    }
+    env.pop(ANTHROPIC_API_KEY_KEY, None)
+    merged: dict[str, JsonValue] = {**settings, ENV_KEY: env}
+    merged.pop(API_KEY_HELPER_KEY, None)
+    return merged
+
+
+__all__ = ["merge_claude_settings_static_token"]

litellm/proxy/client/cli/commands/autoroute/wizard.py

@@ -0,0 +1,150 @@
+import sys
+from pathlib import Path
+
+import click
+import yaml
+from InquirerPy import inquirer
+from InquirerPy.base.control import Choice
+
+from .... import Client
+from .config import (
+    DEFAULT_KEYWORD_TIER_RULES,
+    TIER_NAMES,
+    AutorouteConfig,
+    ConfigGenerationError,
+    DiscoveredModel,
+    HeuristicClassifier,
+    KeywordTierRule,
+    LLMClassifier,
+    NoSemanticMatching,
+    SemanticMatching,
+    build_generated_model_list,
+    chat_models,
+    embedding_models,
+    parse_discovered_models,
+    validate_config,
+)
+from .process import CONFIG_PATH, secure_create
+
+
+def _is_interactive() -> bool:
+    return sys.stdin.isatty()
+
+
+def _fuzzy_pick(models: tuple[DiscoveredModel, ...], prompt_label: str, multiselect: bool) -> list[str]:
+    """Type-to-filter picker over a (possibly huge) model pool, using InquirerPy's fzf-style fuzzy prompt.
+
+    A plain numbered table + typed index does not scale past a handful of models -- proxies with
+    hundreds of model groups made that interaction unusable. This lets the user narrow the pool by
+    typing a substring instead of scrolling/counting.
+
+    Assumes the caller already checked interactivity (run_configure_wizard does, once, up front) --
+    checking here too would check the wrong thing under test, where InquirerPy is driven through its
+    own injected input/output rather than the real process stdin.
+    """
+    choices = [Choice(value=model.name, name=model.name) for model in models]
+    toggle_hint = "tab to toggle, " if multiselect else ""
+    while True:
+        result = inquirer.fuzzy(
+            message=f"{prompt_label}: type to filter, {toggle_hint}enter to confirm",
+            choices=choices,
+            multiselect=multiselect,
+            max_height="70%",
+        ).execute()
+        selected = result if multiselect else [result]
+        if selected:
+            return selected
+        click.echo("Select at least one model.")
+
+
+def _render_and_prompt_for_model(models: tuple[DiscoveredModel, ...], prompt_label: str) -> str:
+    return _fuzzy_pick(models, prompt_label, multiselect=False)[0]
+
+
+def _render_and_prompt_for_models(models: tuple[DiscoveredModel, ...], prompt_label: str) -> tuple[str, ...]:
+    return tuple(_fuzzy_pick(models, prompt_label, multiselect=True))
+
+
+def _parse_keywords(raw: str) -> tuple[str, ...]:
+    return tuple(keyword.strip() for keyword in raw.split(",") if keyword.strip())
+
+
+def _prompt_for_keyword_tier_rules() -> tuple[KeywordTierRule, ...]:
+    """Let the user supply the semantic-matching keywords per tier, since matching those
+    keywords against the request is the whole point of enabling it. Each prompt is prefilled
+    with the built-in default, so pressing enter keeps it."""
+    click.echo("\nEnter example keywords/phrases per tier (comma-separated); press enter to keep the default:")
+    defaults = {rule.tier: rule.keywords for rule in DEFAULT_KEYWORD_TIER_RULES}
+
+    def _rule_for(tier: str) -> KeywordTierRule:
+        default_keywords = defaults.get(tier, ())
+        raw = click.prompt(f"  {tier} keywords", default=", ".join(default_keywords), show_default=True)
+        return KeywordTierRule(keywords=_parse_keywords(raw) or default_keywords, tier=tier)
+
+    return tuple(_rule_for(tier) for tier in TIER_NAMES)
+
+
+def run_configure_wizard(ctx: click.Context) -> Path:
+    """Discover the caller's accessible models, walk them through tier assignment, write config."""
+    base_url = ctx.obj["base_url"]
+    api_key = ctx.obj["api_key"]
+    client = Client(base_url=base_url, api_key=api_key)
+
+    raw_groups = client.model_groups.info()
+    if not isinstance(raw_groups, list):
+        raise click.ClickException(
+            f"Unexpected response from /model_group/info: expected a list, got {type(raw_groups).__name__}"
+        )
+    discovered = parse_discovered_models(raw_groups)
+    chat_pool = chat_models(discovered)
+    embedding_pool = embedding_models(discovered)
+
+    if not chat_pool:
+        raise click.ClickException("Your key has no chat-capable models available on this proxy.")
+
+    if not _is_interactive():
+        raise click.ClickException("`lite autoroute configure` requires an interactive terminal.")
+
+    click.echo("Assign model(s) to each complexity tier (from what your key can access):")
+    tiers = {tier: _render_and_prompt_for_models(chat_pool, tier) for tier in TIER_NAMES}
+    default_model = tiers["MEDIUM"][0]
+
+    classifier = HeuristicClassifier()
+    if click.confirm("\nUse an LLM classifier instead of the free heuristic scorer?", default=False):
+        classifier_model = _render_and_prompt_for_model(chat_pool, "LLM classifier")
+        classifier = LLMClassifier(model=classifier_model)
+
+    semantic_matching = NoSemanticMatching()
+    if embedding_pool and click.confirm("\nEnable semantic keyword matching?", default=False):
+        embedding_model = _render_and_prompt_for_model(embedding_pool, "semantic embeddings")
+        keyword_tier_rules = _prompt_for_keyword_tier_rules()
+        semantic_matching = SemanticMatching(embedding_model=embedding_model, keyword_tier_rules=keyword_tier_rules)
+
+    adaptive = click.confirm("\nEnable adaptive (bandit) selection on top of tiering?", default=False)
+
+    config = AutorouteConfig(
+        base_url=base_url,
+        api_key=api_key,
+        tiers=tiers,
+        default_model=default_model,
+        classifier=classifier,
+        semantic_matching=semantic_matching,
+        adaptive=adaptive,
+    )
+    try:
+        validate_config(config, discovered)
+    except ConfigGenerationError as e:
+        raise click.ClickException(str(e))
+
+    model_list = build_generated_model_list(config)
+    CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True)
+    with secure_create(CONFIG_PATH) as f:
+        yaml.safe_dump({"model_list": model_list}, f, sort_keys=False)
+
+    click.echo(f"\nWrote {CONFIG_PATH}")
+    for tier, models in tiers.items():
+        click.echo(f"  {tier}: {', '.join(models)}")
+    return CONFIG_PATH
+
+
+__all__ = ["run_configure_wizard"]

litellm/proxy/client/cli/commands/chat.py

@@ -150,7 +150,7 @@ def chat(
             f"Max Tokens: [yellow]{max_tokens or 'unlimited'}[/yellow]\n\n"
             f"Type your messages and press Enter. Type '/quit' or '/exit' to end the session.\n"
             f"Type '/help' for more commands.",
-            title="🤖 Chat Session",
+            title="Chat Session",
         )
     )
 

litellm/proxy/client/cli/commands/encryption.py

@@ -32,7 +32,7 @@ def migrate(ctx: click.Context, check_only: bool, dry_run: bool):
 
     Requires the proxy to be started with
     ``general_settings.encryption_algorithm: aes-256-gcm``. Idempotent and
-    resumable — safe to re-run after an interruption.
+    resumable; safe to re-run after an interruption.
 
     Examples:
         litellm-proxy encryption migrate --check   # attestation scan, no writes

litellm/proxy/client/cli/commands/keys.py

@@ -309,12 +309,12 @@ def _import_keys_to_destination(
             imported_count += 1
 
             key_alias = key.get("key_alias", "N/A")
-            click.echo(f"✓ Imported key: {key_alias}")
+            click.echo(f"Imported key: {key_alias}")
 
         except Exception as e:
             failed_count += 1
             key_alias = key.get("key_alias", "N/A")
-            click.echo(f"✗ Failed to import key {key_alias}: {str(e)}", err=True)
+            click.echo(f"Failed to import key {key_alias}: {str(e)}", err=True)
 
     return imported_count, failed_count
 

litellm/proxy/client/cli/commands/model_groups.py

@@ -0,0 +1,57 @@
+from typing import Literal
+
+import click
+import rich
+import rich.table
+
+from ... import Client
+
+
+def create_client(ctx: click.Context) -> Client:
+    return Client(base_url=ctx.obj["base_url"], api_key=ctx.obj["api_key"])
+
+
+@click.group(name="model-groups")
+def model_groups() -> None:
+    """Inspect model groups your key can access on the proxy"""
+
+
+@model_groups.command("list")
+@click.option(
+    "--format",
+    "output_format",
+    type=click.Choice(["table", "json"]),
+    default="table",
+    help="Output format (table or json)",
+)
+@click.pass_context
+def list_model_groups(ctx: click.Context, output_format: Literal["table", "json"]) -> None:
+    """List model groups accessible to your key, with mode and pricing"""
+    client = create_client(ctx)
+    groups = client.model_groups.info()
+    if not isinstance(groups, list):
+        raise click.ClickException(
+            f"Unexpected response from /model_group/info: expected a list, got {type(groups).__name__}"
+        )
+
+    if output_format == "json":
+        rich.print_json(data=groups)
+        return
+
+    table = rich.table.Table(title="Accessible Model Groups")
+    table.add_column("Model", style="cyan")
+    table.add_column("Mode", style="green")
+    table.add_column("Input $/token", style="yellow")
+    table.add_column("Output $/token", style="yellow")
+
+    for group in groups:
+        table.add_row(
+            str(group.get("model_group", "")),
+            str(group.get("mode", "chat")),
+            str(group.get("input_cost_per_token", "")),
+            str(group.get("output_cost_per_token", "")),
+        )
+    rich.print(table)
+
+
+__all__ = ["model_groups"]

litellm/proxy/client/cli/commands/teams.py

@@ -21,7 +21,7 @@ def display_teams_table(teams: List[Dict[str, Any]]) -> None:
     console = Console()
 
     if not teams:
-        console.print("❌ No teams found for your user.")
+        console.print("No teams found for your user.")
         return
 
     table = Table(title="Available Teams")
@@ -91,10 +91,10 @@ def available(ctx: click.Context):
         teams = client.teams.get_available()
         if teams:
             console = Console()
-            console.print("\n🎯 Available Teams to Join:")
+            console.print("\nAvailable Teams to Join:")
             display_teams_table(teams)
         else:
-            click.echo("ℹ️ No available teams to join.")
+            click.echo("No available teams to join.")
     except requests.exceptions.HTTPError as e:
         click.echo(f"Error: HTTP {e.response.status_code}", err=True)
         error_body = e.response.json()
@@ -113,7 +113,7 @@ def assign_key(ctx: click.Context, team_id: Optional[str]):
     api_key = ctx.obj["api_key"]
 
     if not api_key:
-        click.echo("❌ No API key found. Please login first using 'litellm login'")
+        click.echo("No API key found. Please login first using 'litellm login'")
         raise click.Abort()
 
     try:
@@ -122,7 +122,7 @@ def assign_key(ctx: click.Context, team_id: Optional[str]):
             teams = client.teams.list()
 
             if not teams:
-                click.echo("❌ No teams found for your user.")
+                click.echo("No teams found for your user.")
                 return
 
             # Use interactive selection from auth module
@@ -133,14 +133,14 @@ def assign_key(ctx: click.Context, team_id: Optional[str]):
             if selected_team:
                 team_id = selected_team.get("team_id")
             else:
-                click.echo("❌ Operation cancelled.")
+                click.echo("Operation cancelled.")
                 return
 
         # Update the key with the selected team
         if team_id:
-            click.echo(f"\n🔄 Assigning your key to team: {team_id}")
+            click.echo(f"\nAssigning your key to team: {team_id}")
             client.keys.update(key=api_key, team_id=team_id)
-            click.echo(f"✅ Successfully assigned key to team: {team_id}")
+            click.echo(f"Successfully assigned key to team: {team_id}")
 
             # Show team details if available
             teams = client.teams.list()
@@ -148,9 +148,9 @@ def assign_key(ctx: click.Context, team_id: Optional[str]):
                 if team.get("team_id") == team_id:
                     models = team.get("models", [])
                     if models:
-                        click.echo(f"🎯 You can now access models: {', '.join(models)}")
+                        click.echo(f"You can now access models: {', '.join(models)}")
                     else:
-                        click.echo("🎯 You can now access all available models")
+                        click.echo("You can now access all available models")
                     break
 
     except requests.exceptions.HTTPError as e:

litellm/proxy/client/cli/commands/up.py

@@ -0,0 +1,283 @@
+import atexit
+import contextlib
+import json
+import os
+import shlex
+import shutil
+import signal
+import sys
+import threading
+from dataclasses import dataclass
+from pathlib import Path
+from types import FrameType
+from typing import IO, Iterator, Mapping
+
+import click
+from pydantic import JsonValue, TypeAdapter, ValidationError
+
+from litellm.litellm_core_utils.cli_token_utils import is_cli_token_fresh
+
+from .agents import AgentRunError, resolve_api_key, verify_proxy_key
+from .auth import load_token, login
+
+ENV_KEY = "env"
+API_KEY_HELPER_KEY = "apiKeyHelper"
+ANTHROPIC_BASE_URL_KEY = "ANTHROPIC_BASE_URL"
+ANTHROPIC_API_KEY_KEY = "ANTHROPIC_API_KEY"
+
+CLAUDE_SETTINGS_PATH = Path.home() / ".claude" / "settings.json"
+BACKUP_PATH = Path.home() / ".litellm" / "claude_settings_backup.json"
+
+
+class UpError(Exception):
+    """Raised for any user-actionable failure while starting/stopping interception."""
+
+
+@dataclass(frozen=True, slots=True)
+class BackupRecord:
+    """Snapshot of ~/.claude/settings.json taken right before `lite up` patches it."""
+
+    existed: bool
+    content: dict[str, JsonValue] | None
+
+
+_SETTINGS_ADAPTER = TypeAdapter(dict[str, JsonValue])
+_BACKUP_RECORD_ADAPTER = TypeAdapter(BackupRecord)
+
+
+def load_json_or_empty(path: Path) -> dict[str, JsonValue]:
+    if not path.exists():
+        return {}
+    with open(path, "r") as f:
+        content = f.read()
+    if not content.strip():
+        return {}
+    try:
+        return _SETTINGS_ADAPTER.validate_json(content)
+    except ValidationError:
+        raise UpError(f"{path} contains invalid JSON (or its root is not an object); cannot proceed safely.")
+
+
+def merge_claude_settings(
+    settings: Mapping[str, JsonValue], base_url: str, api_key_helper: str
+) -> dict[str, JsonValue]:
+    """Return a new settings dict wired to route Claude Code through the proxy.
+
+    Only env.ANTHROPIC_BASE_URL and the top-level apiKeyHelper are overridden; a
+    stray env.ANTHROPIC_API_KEY is dropped so it cannot outrank the helper-issued
+    token (same reasoning as build_agent_env in agents.py). Every other key is
+    preserved untouched.
+    """
+    raw_env = settings.get(ENV_KEY, {})
+    base_env = raw_env if isinstance(raw_env, dict) else {}
+    env = {**base_env, ANTHROPIC_BASE_URL_KEY: base_url.rstrip("/")}
+    env.pop(ANTHROPIC_API_KEY_KEY, None)
+    return {**settings, ENV_KEY: env, API_KEY_HELPER_KEY: api_key_helper}
+
+
+@contextlib.contextmanager
+def secure_create(path: Path) -> Iterator[IO[str]]:
+    """Open path for writing with mode 0600 fixed up before any content is written.
+
+    A plain `open(path, "w")` creates a *new* file at the umask-derived default (commonly 0644)
+    and leaves it world- or group-readable until a later `chmod` call catches up -- a real window
+    in which a file holding a credential is readable by another local account. Passing the mode to
+    `os.open` closes that window for a brand-new file, but `O_CREAT`'s mode argument is only
+    applied on creation: if the file already exists its old, broader permissions carry over
+    untouched. `os.fchmod` right after opening -- before a single byte of the new content is
+    written -- covers both cases.
+    """
+    fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+    os.fchmod(fd, 0o600)
+    f: IO[str] = os.fdopen(fd, "w")
+    try:
+        yield f
+    finally:
+        f.close()
+
+
+def write_backup(record: BackupRecord, backup_path: Path | None = None) -> None:
+    path = backup_path if backup_path is not None else BACKUP_PATH
+    path.parent.mkdir(exist_ok=True)
+    with secure_create(path) as f:
+        json.dump({"existed": record.existed, "content": record.content}, f, indent=2)
+
+
+def read_backup(backup_path: Path | None = None) -> BackupRecord | None:
+    path = backup_path if backup_path is not None else BACKUP_PATH
+    if not path.exists():
+        return None
+    with open(path, "r") as f:
+        content = f.read()
+    try:
+        return _BACKUP_RECORD_ADAPTER.validate_json(content)
+    except ValidationError:
+        raise UpError(f"{path} contains invalid or unexpected JSON; cannot restore from it safely.")
+
+
+def restore_claude_settings(settings_path: Path | None = None, backup_path: Path | None = None) -> BackupRecord | None:
+    """Restore settings_path from the backup at backup_path, then delete the backup.
+
+    Returns the restored record, or None if there was nothing to restore.
+    """
+    resolved_settings_path = settings_path if settings_path is not None else CLAUDE_SETTINGS_PATH
+    resolved_backup_path = backup_path if backup_path is not None else BACKUP_PATH
+    record = read_backup(resolved_backup_path)
+    if record is None:
+        return None
+    if record.existed and record.content is not None:
+        resolved_settings_path.parent.mkdir(parents=True, exist_ok=True)
+        with open(resolved_settings_path, "w") as f:
+            json.dump(record.content, f, indent=2)
+    elif resolved_settings_path.exists():
+        resolved_settings_path.unlink()
+    resolved_backup_path.unlink()
+    return record
+
+
+def resolve_api_key_helper(base_url: str) -> str:
+    """Build the shell command Claude Code should run for its apiKeyHelper.
+
+    Resolves `lite` to an absolute path so the helper works regardless of the
+    PATH visible to whatever subprocess Claude Code spawns it from. Passing
+    --base-url explicitly (rather than relying on the bare invocation Claude
+    Code would otherwise use) makes `print-token` enforce that the cached
+    token was actually issued for this proxy -- without it, a token minted
+    for a different, previously-logged-into proxy would be handed to
+    whichever server `up` currently points at.
+    """
+    lite_path = shutil.which("lite")
+    if lite_path is None:
+        raise UpError(
+            "Could not find `lite` on your PATH. Claude Code's apiKeyHelper needs "
+            "an absolute path to it, so `lite up` cannot continue."
+        )
+    return f"{shlex.quote(lite_path)} auth print-token --base-url {shlex.quote(base_url)}"
+
+
+def _ensure_fresh_login(ctx: click.Context) -> None:
+    base_url = ctx.obj["base_url"].rstrip("/")
+    token_data = load_token()
+    if token_data and token_data.get("base_url") == base_url and is_cli_token_fresh(token_data):
+        return
+
+    if not sys.stdin.isatty():
+        raise UpError(
+            "No fresh LiteLLM login found for this proxy. Run `lite login` first (apiKeyHelper "
+            "reads this token on every Claude Code request)."
+        )
+
+    click.echo("No fresh LiteLLM login found for this proxy; starting login...")
+    ctx.invoke(login)
+    token_data = load_token()
+    if not token_data or token_data.get("base_url") != base_url or not is_cli_token_fresh(token_data):
+        raise UpError("Login did not produce a usable token; cannot start `lite up`.")
+
+
+def _restore_and_report() -> None:
+    record = restore_claude_settings()
+    if record is None:
+        click.echo("Nothing to restore.")
+        return
+    if record.existed:
+        click.echo(f"Restored {CLAUDE_SETTINGS_PATH} to its original contents.")
+    else:
+        click.echo(f"Removed {CLAUDE_SETTINGS_PATH} (it did not exist before `lite up`).")
+
+
+@click.command(name="up")
+@click.pass_context
+def up(ctx: click.Context) -> None:
+    """Route every Claude Code session through your LiteLLM proxy until stopped.
+
+    Patches ~/.claude/settings.json so Claude Code picks up the proxy on its own
+    next startup, from any terminal -- no need to launch it through `lite`.
+    Press Ctrl-C to stop and restore your original settings. Assumes the proxy
+    is already running (this does not start one for you). Cursor is not
+    supported: it has no equivalent file-based config to patch.
+    """
+    base_url = ctx.obj["base_url"]
+
+    try:
+        _ensure_fresh_login(ctx)
+        api_key = resolve_api_key(ctx)
+        verify_proxy_key(base_url, api_key)
+
+        if BACKUP_PATH.exists():
+            raise UpError(
+                f"{BACKUP_PATH} already exists -- `lite up` looks like it's already "
+                "running (or crashed without cleanup). Run `lite down` first."
+            )
+
+        api_key_helper = resolve_api_key_helper(base_url)
+        original_existed = CLAUDE_SETTINGS_PATH.exists()
+        original_settings = load_json_or_empty(CLAUDE_SETTINGS_PATH)
+        write_backup(
+            BackupRecord(
+                existed=original_existed,
+                content=original_settings if original_existed else None,
+            )
+        )
+
+        CLAUDE_SETTINGS_PATH.parent.mkdir(exist_ok=True)
+        merged = merge_claude_settings(original_settings, base_url, api_key_helper)
+        with open(CLAUDE_SETTINGS_PATH, "w") as f:
+            json.dump(merged, f, indent=2)
+    except (AgentRunError, UpError) as e:
+        raise click.ClickException(str(e))
+
+    click.echo(f"litellm: routing Claude Code through proxy at {base_url.rstrip('/')}")
+    click.echo("Press Ctrl-C to stop and restore your original settings.")
+
+    stop_event = threading.Event()
+    restored = threading.Lock()
+
+    def _handle_signal(_signum: int, _frame: FrameType | None) -> None:
+        stop_event.set()
+
+    def _restore_once() -> None:
+        if not restored.acquire(blocking=False):
+            return
+        try:
+            _restore_and_report()
+        except UpError as e:
+            # Runs from atexit/a signal handler, outside Click's own exception
+            # handling -- raising here would only produce an unhandled-exception
+            # warning on stderr, not a clean message.
+            click.echo(str(e), err=True)
+
+    signal.signal(signal.SIGINT, _handle_signal)
+    signal.signal(signal.SIGTERM, _handle_signal)
+    atexit.register(_restore_once)
+
+    stop_event.wait()
+    _restore_once()
+
+
+@click.command(name="down")
+def down() -> None:
+    """Restore ~/.claude/settings.json if a `lite up` session left it patched.
+
+    Use this after a `lite up` process was killed uncleanly (e.g. `kill -9`)
+    instead of stopped with Ctrl-C.
+    """
+    try:
+        _restore_and_report()
+    except UpError as e:
+        raise click.ClickException(str(e))
+
+
+__all__ = [
+    "BACKUP_PATH",
+    "CLAUDE_SETTINGS_PATH",
+    "BackupRecord",
+    "UpError",
+    "down",
+    "load_json_or_empty",
+    "merge_claude_settings",
+    "read_backup",
+    "resolve_api_key_helper",
+    "restore_claude_settings",
+    "up",
+    "write_backup",
+]

litellm/proxy/client/cli/interface.py

@@ -27,13 +27,13 @@ def styled_prompt():
         verbose_logger.debug(f"Error getting terminal size: {e}")
         click.echo("\n" * 3)
 
-    # Unicode box drawing characters
-    top_left = "┌"
-    top_right = "┐"
-    bottom_left = "└"
-    bottom_right = "┘"
-    horizontal = "─"
-    vertical = "│"
+    # ASCII box drawing characters
+    top_left = "+"
+    top_right = "+"
+    bottom_left = "+"
+    bottom_right = "+"
+    horizontal = "-"
+    vertical = "|"
 
     # Create the box with increased width
     width = 80

litellm/proxy/client/cli/main.py

@@ -9,15 +9,18 @@ from litellm.proxy.client.health import HealthManagementClient
 
 from .commands.agents import agent_commands
 from .commands.auth import auth_group, get_stored_api_key, login, logout, whoami
+from .commands.autoroute.commands import autoroute_group
 from .commands.chat import chat
 from .commands.credentials import credentials
 from .commands.encryption import encryption
 from .commands.http import http
 from .commands.keys import keys
+from .commands.model_groups import model_groups
 
 # local imports
 from .commands.models import models
 from .commands.teams import teams
+from .commands.up import down, up
 from .commands.users import users
 from .interface import interactive_shell
 
@@ -131,6 +134,13 @@ cli.add_command(users)
 # Add a top-level command per coding agent (claude, codex, opencode, ...)
 for agent_command in agent_commands():
     cli.add_command(agent_command)
+# Add the up/down commands (route Claude Code through the local LiteLLM proxy)
+cli.add_command(up)
+cli.add_command(down)
+# Add the model-groups command group (discover models your key can access)
+cli.add_command(model_groups)
+# Add the autoroute command group (QA auto-routing against your real proxy)
+cli.add_command(autoroute_group, name="autoroute")
 
 
 if __name__ == "__main__":

litellm/proxy/db/db_spend_update_writer.py

@@ -27,7 +27,7 @@ from typing import (
 
 import litellm
 from litellm._logging import verbose_proxy_logger
-from litellm.caching import DualCache, RedisCache
+from litellm.caching import RedisCache
 from litellm.constants import (
     DB_SPEND_UPDATE_JOB_NAME,
     DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME,
@@ -44,7 +44,6 @@ from litellm.proxy._types import (
     DailyUserSpendTransaction,
     DBSpendUpdateTransactions,
     Litellm_EntityType,
-    LiteLLM_UserTable,
     SpendLogsMetadata,
     SpendLogsPayload,
     SpendUpdateQueueItem,
@@ -137,7 +136,6 @@ class DBSpendUpdateWriter:
             disable_spend_logs,
             litellm_proxy_budget_name,
             prisma_client,
-            user_api_key_cache,
         )
         from litellm.proxy.utils import ProxyUpdateSpend, hash_token
 
@@ -195,7 +193,6 @@ class DBSpendUpdateWriter:
                     org_id=org_id,
                     end_user_id=end_user_id,
                     prisma_client=prisma_client,
-                    user_api_key_cache=user_api_key_cache,
                     litellm_proxy_budget_name=litellm_proxy_budget_name,
                     payload=payload,
                 )
@@ -326,7 +323,6 @@ class DBSpendUpdateWriter:
         org_id: Optional[str],
         end_user_id: Optional[str],
         prisma_client: Optional[PrismaClient],
-        user_api_key_cache: DualCache,
         litellm_proxy_budget_name: Optional[str],
         payload: SpendLogsPayload,
     ):
@@ -345,7 +341,6 @@ class DBSpendUpdateWriter:
                 response_cost=response_cost,
                 user_id=user_id,
                 prisma_client=prisma_client,
-                user_api_key_cache=user_api_key_cache,
                 litellm_proxy_budget_name=litellm_proxy_budget_name,
                 end_user_id=end_user_id,
             )
@@ -510,7 +505,6 @@ class DBSpendUpdateWriter:
         response_cost: Optional[float],
         user_id: Optional[str],
         prisma_client: Optional[PrismaClient],
-        user_api_key_cache: DualCache,
         litellm_proxy_budget_name: Optional[str],
         end_user_id: Optional[str] = None,
     ):
@@ -518,10 +512,6 @@ class DBSpendUpdateWriter:
         - Update that user's row
         - Update litellm-proxy-budget row (global proxy spend)
         """
-        ## if an end-user is passed in, do an upsert - we can't guarantee they already exist in db
-        existing_user_obj = await user_api_key_cache.async_get_cache(key=user_id)
-        if existing_user_obj is not None and isinstance(existing_user_obj, dict):
-            existing_user_obj = LiteLLM_UserTable(**existing_user_obj)
         try:
             if prisma_client is not None:  # update
                 user_ids = [user_id]

litellm/proxy/db/query_engine_reaper.py

@@ -0,0 +1,212 @@
+"""Supervisor-side reaper for orphaned Prisma query-engine processes.
+
+Each proxy worker owns a Prisma query-engine subprocess whose only cleanup
+hook is an in-process ``atexit`` handler. When a multi-worker supervisor
+(uvicorn's multiprocess manager, the gunicorn arbiter) force-kills a hung or
+crashed worker, that handler never runs: the engine reparents to the nearest
+subreaper (PID 1 in a container, which is the supervisor itself under the
+standard docker entrypoint) and keeps its database connection pool
+established forever, while the replacement worker opens a fresh pool. Over
+repeated worker deaths the active database connections grow without bound.
+
+The reaper runs only in the supervisor process, where a query-engine process
+can never be a legitimate direct child: workers own their engines, and the
+supervisor never starts one. Any direct child whose command name begins with
+``query-engine`` is therefore an adopted orphan and is terminated
+(SIGTERM, bounded grace, SIGKILL) and reaped. On Linux the supervisor also
+marks itself a child subreaper so orphans reparent to it even when it is not
+PID 1.
+
+Linux-only by construction (``/proc`` scan, ``prctl``); a no-op elsewhere.
+"""
+
+import ctypes
+import os
+import signal
+import sys
+import threading
+import time
+from typing import Optional
+
+from litellm._logging import verbose_proxy_logger
+
+QUERY_ENGINE_COMM_PREFIX = "query-engine"
+REAPER_SCAN_INTERVAL_SECONDS = 5.0
+SIGTERM_GRACE_SECONDS = 10.0
+PR_SET_CHILD_SUBREAPER = 36
+
+
+def set_child_subreaper() -> bool:
+    """Mark this process as a child subreaper so orphaned descendants
+    reparent to it instead of PID 1. Best-effort: when it fails (or on
+    non-Linux) the reaper still covers the containerized case where the
+    supervisor already is PID 1."""
+    if not sys.platform.startswith("linux"):
+        return False
+    try:
+        libc = ctypes.CDLL(None, use_errno=True)
+        result: int = libc.prctl(  # pyright: ignore[reportAny]  # ctypes types foreign calls as Any; default restype is c_int
+            PR_SET_CHILD_SUBREAPER, 1, 0, 0, 0
+        )
+        return result == 0
+    except (OSError, AttributeError):
+        return False
+
+
+def _read_comm_and_ppid(pid: int, proc_root: str) -> Optional[tuple[str, int]]:
+    try:
+        with open(f"{proc_root}/{pid}/stat", encoding="ascii", errors="replace") as stat_file:
+            data = stat_file.read()
+    except (FileNotFoundError, ProcessLookupError, PermissionError, OSError):
+        return None
+    lparen = data.find("(")
+    rparen = data.rfind(")")
+    if lparen == -1 or rparen == -1 or rparen < lparen:
+        return None
+    comm = data[lparen + 1 : rparen]
+    fields = data[rparen + 2 :].split()
+    if len(fields) < 2:
+        return None
+    try:
+        ppid = int(fields[1])
+    except ValueError:
+        return None
+    return comm, ppid
+
+
+def list_orphaned_engine_pids(parent_pid: int, proc_root: str = "/proc") -> tuple[int, ...]:
+    """PIDs of direct children of ``parent_pid`` whose command name marks
+    them as Prisma query engines. In the supervisor these are always
+    adopted orphans: live engines are children of workers, not of the
+    supervisor."""
+    try:
+        entries = os.listdir(proc_root)
+    except (FileNotFoundError, OSError):
+        return ()
+    candidate_pids = (int(entry) for entry in entries if entry.isdigit())
+    return tuple(
+        pid
+        for pid in candidate_pids
+        if (info := _read_comm_and_ppid(pid, proc_root)) is not None
+        and info[1] == parent_pid
+        and info[0].startswith(QUERY_ENGINE_COMM_PREFIX)
+    )
+
+
+def _try_reap(pid: int) -> bool:
+    try:
+        reaped_pid, _ = os.waitpid(pid, os.WNOHANG)
+    except ChildProcessError:
+        return True
+    except OSError:
+        return True
+    return reaped_pid == pid
+
+
+def _send_signal(pid: int, signum: int) -> None:
+    try:
+        os.kill(pid, signum)
+    except (ProcessLookupError, PermissionError, OSError):
+        pass
+
+
+def _await_reaped(pids: tuple[int, ...], timeout_seconds: float) -> tuple[int, ...]:
+    """Poll until every PID is reaped or the shared deadline passes.
+    Returns the PIDs still alive at the deadline."""
+    deadline = time.monotonic() + timeout_seconds
+    remaining = pids
+    while remaining and time.monotonic() < deadline:
+        remaining = tuple(pid for pid in remaining if not _try_reap(pid))
+        if remaining:
+            time.sleep(0.2)
+    return remaining
+
+
+def terminate_and_reap(pid: int, grace_seconds: float = SIGTERM_GRACE_SECONDS) -> None:
+    """SIGTERM the orphaned engine, escalate to SIGKILL after the grace
+    period, and reap it so it does not linger as a zombie."""
+    terminate_and_reap_all((pid,), grace_seconds=grace_seconds)
+
+
+def terminate_and_reap_all(
+    pids: tuple[int, ...],
+    grace_seconds: float = SIGTERM_GRACE_SECONDS,
+) -> None:
+    """Terminate a batch of orphaned engines concurrently: SIGTERM all of
+    them, share one grace period, SIGKILL the stragglers, and reap. The
+    shared deadline keeps cleanup time bounded when several workers die
+    at once instead of paying the grace period once per orphan."""
+    for pid in pids:
+        verbose_proxy_logger.warning(
+            "Reaping orphaned prisma query-engine PID %s (its worker process exited without cleanup).",
+            pid,
+        )
+        _send_signal(pid, signal.SIGTERM)
+    survivors = _await_reaped(pids, grace_seconds)
+    if not survivors:
+        return
+    for pid in survivors:
+        verbose_proxy_logger.warning(
+            "Orphaned prisma query-engine PID %s did not exit within %.1fs of SIGTERM; sending SIGKILL.",
+            pid,
+            grace_seconds,
+        )
+        _send_signal(pid, signal.SIGKILL)
+    unkillable = _await_reaped(survivors, 5.0)
+    for pid in unkillable:
+        verbose_proxy_logger.error(
+            "Orphaned prisma query-engine PID %s survived SIGKILL; will retry on the next scan.",
+            pid,
+        )
+
+
+def reap_orphaned_engines(parent_pid: int, proc_root: str = "/proc") -> tuple[int, ...]:
+    """One scan-and-reap pass. Returns the PIDs it acted on."""
+    orphaned_pids = list_orphaned_engine_pids(parent_pid, proc_root=proc_root)
+    if orphaned_pids:
+        terminate_and_reap_all(orphaned_pids)
+    return orphaned_pids
+
+
+def _reaper_loop(parent_pid: int) -> None:
+    while True:
+        try:
+            reap_orphaned_engines(parent_pid)
+        except Exception as scan_error:  # noqa: BLE001  # reaper thread must survive any scan failure
+            verbose_proxy_logger.debug("Orphaned query-engine scan failed: %s", scan_error)
+        time.sleep(REAPER_SCAN_INTERVAL_SECONDS)
+
+
+REAPER_THREAD_NAME = "litellm-orphan-query-engine-reaper"
+
+
+def start_query_engine_reaper() -> Optional[threading.Thread]:
+    """Start the reaper daemon thread in the supervisor process.
+
+    Must only be called from a process that never hosts the proxy app
+    itself (uvicorn with ``workers > 1``, the gunicorn arbiter): with a
+    single in-process uvicorn worker the query engine is a legitimate
+    direct child and must not be touched. Idempotent: a reaper already
+    running in this process is returned instead of starting a second one.
+    """
+    if not sys.platform.startswith("linux"):
+        return None
+    existing = next(
+        (thread for thread in threading.enumerate() if thread.name == REAPER_THREAD_NAME),
+        None,
+    )
+    if existing is not None:
+        return existing
+    set_child_subreaper()
+    reaper_thread = threading.Thread(
+        target=_reaper_loop,
+        args=(os.getpid(),),
+        daemon=True,
+        name=REAPER_THREAD_NAME,
+    )
+    reaper_thread.start()
+    verbose_proxy_logger.info(
+        "Started orphaned prisma query-engine reaper in supervisor process %s.",
+        os.getpid(),
+    )
+    return reaper_thread

litellm/proxy/enterprise_billing/billing_metrics.py

@@ -0,0 +1,323 @@
+"""
+Push-based OTLP metering for enterprise litellm deployments.
+
+Owns a dedicated OpenTelemetry meter provider and an OTLP/HTTP exporter
+authenticated to our global collector with a TLS client certificate. The
+collector front end terminates mutual TLS: the client certificate presented
+here is validated against our CA at the edge, and the verified subject is
+what identifies the deployment. It is intentionally isolated from the global
+meter provider so the customer's own OTEL metrics are untouched and ours
+never leak into their backend.
+
+The deployment's identity rides on the TLS client certificate, not on the
+payload; the secret license key is never sent as an attribute or header.
+"""
+
+import os
+import tempfile
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Optional, Union
+
+from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
+from opentelemetry.metrics import Counter
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
+from opentelemetry.sdk.resources import Resource
+
+from litellm._logging import verbose_proxy_logger
+from litellm.proxy.middleware.billable_request_metrics_middleware import (
+    BillableCategory,
+)
+
+if TYPE_CHECKING:
+    from litellm.proxy._types import EnterpriseLicenseData
+
+ENDPOINT_ENV = "LITELLM_BILLING_METRICS_ENDPOINT"
+CLIENT_CERT_ENV = "LITELLM_BILLING_METRICS_CLIENT_CERT"
+CLIENT_KEY_ENV = "LITELLM_BILLING_METRICS_CLIENT_KEY"
+CA_CERT_ENV = "LITELLM_BILLING_METRICS_CA_CERT"
+EXPORT_INTERVAL_ENV = "LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS"
+DEFAULT_EXPORT_INTERVAL_MS = 60_000
+SHUTDOWN_FLUSH_TIMEOUT_MS = 5_000
+_METRICS_PATH = "/v1/metrics"
+
+# The cert env vars take a path or the PEM itself. Secret stores that inject
+# values as env content cannot mount them as files, so inline PEM is written out.
+_PEM_PREFIX = "-----BEGIN"
+_PEM_DIR_PREFIX = "litellm-billing-mtls-"
+_PEM_FILE_MODE = 0o600
+_CLIENT_CERT_FILENAME = "client.crt"
+_CLIENT_KEY_FILENAME = "client.key"
+_CA_CERT_FILENAME = "ca.crt"
+
+METRIC_NAME = "litellm.enterprise.billable_requests"
+METER_NAME = "litellm.enterprise.billing"
+
+AttributeValue = Union[str, int]
+
+
+@dataclass(frozen=True, slots=True)
+class BillingMetricsConfig:
+    endpoint: str
+    client_cert_path: str
+    client_key_path: str
+    ca_cert_path: Optional[str]
+    export_interval_ms: int
+    litellm_version: str
+    license_id: Optional[str]
+
+
+def _metrics_endpoint(endpoint: str) -> str:
+    """The OTLP/HTTP metric exporter wants the full URL including the signal path."""
+    trimmed = endpoint.rstrip("/")
+    return trimmed if trimmed.endswith(_METRICS_PATH) else f"{trimmed}{_METRICS_PATH}"
+
+
+def _resource_attributes(config: BillingMetricsConfig) -> dict[str, AttributeValue]:
+    base: dict[str, AttributeValue] = {
+        "service.name": "litellm-proxy",
+        "litellm.version": config.litellm_version,
+    }
+    license_attr: dict[str, AttributeValue] = {"litellm.license.id": config.license_id} if config.license_id else {}
+    return {**base, **license_attr}
+
+
+def _billable_attributes(
+    category: BillableCategory, route: str, status_code: int, model_id: Optional[str]
+) -> dict[str, AttributeValue]:
+    base: dict[str, AttributeValue] = {
+        "litellm.endpoint.category": category.value,
+        "http.route": route,
+        "http.response.status_code": status_code,
+    }
+    model_attr: dict[str, AttributeValue] = {"litellm.model_id": model_id} if model_id else {}
+    return {**base, **model_attr}
+
+
+def build_mtls_meter_provider(config: BillingMetricsConfig) -> MeterProvider:
+    """OTLP/HTTP exporter presenting a TLS client certificate.
+
+    The collector's load balancer terminates mutual TLS and validates the client
+    certificate against our CA. Server verification uses the system trust store
+    (the collector presents a public web-PKI certificate); ca_cert_path overrides
+    it only for private/test collectors.
+    """
+    exporter = OTLPMetricExporter(
+        endpoint=_metrics_endpoint(config.endpoint),
+        # None -> exporter falls back to the system trust store.
+        certificate_file=config.ca_cert_path,
+        client_certificate_file=config.client_cert_path,
+        client_key_file=config.client_key_path,
+    )
+    reader = PeriodicExportingMetricReader(exporter, export_interval_millis=config.export_interval_ms)
+    return MeterProvider(metric_readers=[reader], resource=Resource.create(_resource_attributes(config)))
+
+
+class BillingMetricsRecorder:
+    """Increments one OTLP counter per billable request. The meter provider is injected (see the factory)."""
+
+    def __init__(self, provider: MeterProvider) -> None:
+        self._provider = provider
+        self._counter: Counter = provider.get_meter(METER_NAME).create_counter(
+            name=METRIC_NAME,
+            unit="{request}",
+            description="Count of 2xx HTTP requests to billable LLM/MCP/A2A endpoints",
+        )
+
+    def record(self, *, category: BillableCategory, route: str, status_code: int, model_id: Optional[str]) -> None:
+        self._counter.add(1, _billable_attributes(category, route, status_code, model_id))
+
+    def shutdown(self) -> None:
+        """Final flush + exporter-thread stop. Without this, up to one export
+        interval of billable counts is dropped on every proxy restart."""
+        self._provider.shutdown(timeout_millis=SHUTDOWN_FLUSH_TIMEOUT_MS)
+
+
+def _export_interval_ms() -> int:
+    raw = os.getenv(EXPORT_INTERVAL_ENV)
+    if raw is None:
+        return DEFAULT_EXPORT_INTERVAL_MS
+    try:
+        return int(raw)
+    except ValueError:
+        verbose_proxy_logger.warning(
+            "Invalid %s=%r, falling back to %d ms", EXPORT_INTERVAL_ENV, raw, DEFAULT_EXPORT_INTERVAL_MS
+        )
+        return DEFAULT_EXPORT_INTERVAL_MS
+
+
+@dataclass(frozen=True, slots=True)
+class _CredentialPaths:
+    client_cert_path: str
+    client_key_path: str
+    ca_cert_path: Optional[str]
+
+
+def _is_pem_content(value: str) -> bool:
+    return value.lstrip().startswith(_PEM_PREFIX)
+
+
+def _write_pem(directory: str, filename: str, pem: str) -> str:
+    path = os.path.join(directory, filename)
+    with open(path, "w", encoding="utf-8") as handle:
+        handle.write(pem if pem.endswith("\n") else f"{pem}\n")
+    os.chmod(path, _PEM_FILE_MODE)
+    return path
+
+
+def _resolve_credential_paths(*, client_cert: str, client_key: str, ca_cert: Optional[str]) -> _CredentialPaths:
+    """
+    Accept either a filesystem path or inline PEM content for each credential.
+
+    Secret stores that inject values as environment content rather than mounted
+    files (ECS tasks reading AWS Secrets Manager, Cloud Run reading Secret
+    Manager) can only deliver the certificate as a string. The OTLP exporter
+    takes paths, so inline PEM is written to a private directory once, when the
+    recorder is built. Raises OSError if that write fails; the caller disables
+    metering rather than propagating.
+    """
+    inline = tuple(value for value in (client_cert, client_key, ca_cert) if value and _is_pem_content(value))
+    if not inline:
+        return _CredentialPaths(client_cert, client_key, ca_cert)
+
+    # mkdtemp is 0o700, so the 0o600 key file it holds is unreachable by other users.
+    directory = tempfile.mkdtemp(prefix=_PEM_DIR_PREFIX)
+    return _CredentialPaths(
+        client_cert_path=(
+            _write_pem(directory, _CLIENT_CERT_FILENAME, client_cert) if _is_pem_content(client_cert) else client_cert
+        ),
+        client_key_path=(
+            _write_pem(directory, _CLIENT_KEY_FILENAME, client_key) if _is_pem_content(client_key) else client_key
+        ),
+        ca_cert_path=(
+            _write_pem(directory, _CA_CERT_FILENAME, ca_cert) if ca_cert and _is_pem_content(ca_cert) else ca_cert
+        ),
+    )
+
+
+def load_billing_metrics_config(
+    *, license_data: Optional["EnterpriseLicenseData"], litellm_version: str
+) -> Optional[BillingMetricsConfig]:
+    endpoint = os.getenv(ENDPOINT_ENV)
+    client_cert = os.getenv(CLIENT_CERT_ENV)
+    client_key = os.getenv(CLIENT_KEY_ENV)
+    # Optional: only for private/test collectors whose server cert is not on the
+    # public web PKI. The production collector needs no CA override.
+    ca_cert = os.getenv(CA_CERT_ENV)
+
+    missing = [
+        name
+        for name, value in (
+            (ENDPOINT_ENV, endpoint),
+            (CLIENT_CERT_ENV, client_cert),
+            (CLIENT_KEY_ENV, client_key),
+        )
+        if not value
+    ]
+    if not endpoint or not client_cert or not client_key:
+        verbose_proxy_logger.warning(
+            "Enterprise billing metrics disabled: licensed deployment missing config (%s)",
+            ", ".join(missing),
+        )
+        return None
+
+    try:
+        paths = _resolve_credential_paths(client_cert=client_cert, client_key=client_key, ca_cert=ca_cert)
+    except OSError as exc:
+        verbose_proxy_logger.warning(
+            "Enterprise billing metrics disabled: could not write inline certificate content to disk: %s", exc
+        )
+        return None
+
+    # Report the variable names, never their values. A value that is neither a
+    # readable path nor recognizable PEM is still secret material, and this
+    # warning would otherwise copy a client key straight into the proxy logs.
+    unreadable = [
+        env_name
+        for env_name, path in (
+            (CLIENT_CERT_ENV, paths.client_cert_path),
+            (CLIENT_KEY_ENV, paths.client_key_path),
+            (CA_CERT_ENV, paths.ca_cert_path),
+        )
+        if path and not os.path.isfile(path)
+    ]
+    if unreadable:
+        verbose_proxy_logger.warning(
+            "Enterprise billing metrics disabled: %s did not resolve to a readable certificate file. "
+            "Set each to a file path, or to inline PEM content beginning with '%s'.",
+            ", ".join(unreadable),
+            _PEM_PREFIX,
+        )
+        return None
+
+    return BillingMetricsConfig(
+        endpoint=endpoint,
+        client_cert_path=paths.client_cert_path,
+        client_key_path=paths.client_key_path,
+        ca_cert_path=paths.ca_cert_path,
+        export_interval_ms=_export_interval_ms(),
+        litellm_version=litellm_version,
+        license_id=(license_data or {}).get("user_id"),
+    )
+
+
+class _ActiveRecorderRegistry:
+    """One-slot registry linking the factory-built recorder to the shutdown
+    hook; the middleware instance holding the recorder is not reachable from
+    proxy_shutdown_event."""
+
+    def __init__(self) -> None:
+        self._recorder: Optional[BillingMetricsRecorder] = None
+
+    def set(self, recorder: BillingMetricsRecorder) -> None:
+        self._recorder = recorder
+
+    def pop(self) -> Optional[BillingMetricsRecorder]:
+        recorder = self._recorder
+        self._recorder = None
+        return recorder
+
+
+_ACTIVE_RECORDER = _ActiveRecorderRegistry()
+
+
+def build_billing_metrics_recorder(
+    *, premium: bool, license_data: Optional["EnterpriseLicenseData"], litellm_version: str
+) -> Optional[BillingMetricsRecorder]:
+    """Build the recorder, or None when the deployment is not licensed or metering is unconfigured."""
+    if not premium:
+        # Debug, not warning: unlicensed is the common case and a warning here
+        # would be noise on every OSS proxy. Every other disable path warns.
+        verbose_proxy_logger.debug("Enterprise billing metrics disabled: deployment is not licensed")
+        return None
+
+    config = load_billing_metrics_config(license_data=license_data, litellm_version=litellm_version)
+    if config is None:
+        return None
+
+    try:
+        recorder = BillingMetricsRecorder(build_mtls_meter_provider(config))
+    except Exception as exc:  # noqa: BLE001 -- metering must never break proxy startup
+        verbose_proxy_logger.warning("Enterprise billing metrics disabled: failed to initialize exporter: %s", exc)
+        return None
+    _ACTIVE_RECORDER.set(recorder)
+    # The only positive signal that this component meters. Without it, a silent
+    # return above is indistinguishable from a working exporter in the logs, and
+    # a component that carries the cert but no license would look healthy.
+    verbose_proxy_logger.info(
+        "Enterprise billing metrics enabled: exporting to %s every %d ms",
+        config.endpoint,
+        config.export_interval_ms,
+    )
+    return recorder
+
+
+def shutdown_billing_metrics_recorder() -> None:
+    """Flush and stop the active recorder, if any. Idempotent; never raises."""
+    recorder = _ACTIVE_RECORDER.pop()
+    if recorder is None:
+        return
+    try:
+        recorder.shutdown()
+    except Exception as exc:  # noqa: BLE001 -- shutdown must never block or fail proxy exit
+        verbose_proxy_logger.warning("Enterprise billing metrics: final flush failed: %s", exc)

litellm/proxy/guardrails/guardrail_endpoints.py

@@ -2238,7 +2238,10 @@ async def apply_guardrail(
         if litellm_logging_obj is not None:
             _patch_logging_obj_for_guardrail(litellm_logging_obj, request)
 
-        request_data: dict = {"messages": request.messages} if request.messages else {}
+        request_data: dict = {
+            **({"messages": request.messages} if request.messages is not None else {}),
+            **({"metadata": request.metadata} if request.metadata is not None else {}),
+        }
         _input_type = _resolve_guardrail_input_type(active_guardrail, request.input_type)
         guardrailed_inputs = await active_guardrail.apply_guardrail(
             inputs={"texts": [request.text]},

litellm/proxy/guardrails/guardrail_hooks/compresr/__init__.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from litellm.types.guardrails import (
+    GuardrailEventHooks,
+    Mode,
+    SupportedGuardrailIntegrations,
+)
+
+from .compresr import CompresrGuardrail
+
+if TYPE_CHECKING:
+    from litellm.types.guardrails import Guardrail, LitellmParams
+
+
+def _coerce_event_hook(
+    mode: str | list[str] | Mode,
+) -> GuardrailEventHooks | list[GuardrailEventHooks] | Mode:
+    if isinstance(mode, Mode):
+        return mode
+    if isinstance(mode, list):
+        return [GuardrailEventHooks(item) for item in mode]
+    return GuardrailEventHooks(mode)
+
+
+def _get_optional_value(litellm_params: LitellmParams, optional_params: object | None, attribute_name: str) -> object:
+    if optional_params is not None:
+        value = getattr(optional_params, attribute_name, None)
+        if value is not None:
+            return value
+    return getattr(litellm_params, attribute_name, None)
+
+
+def initialize_guardrail(litellm_params: LitellmParams, guardrail: Guardrail) -> CompresrGuardrail:
+    import litellm
+
+    optional_params = getattr(litellm_params, "optional_params", None)
+
+    _callback = CompresrGuardrail(
+        api_base=litellm_params.api_base,
+        api_key=litellm_params.api_key,
+        model=litellm_params.model,
+        target_compression_ratio=_get_optional_value(litellm_params, optional_params, "target_compression_ratio"),
+        coarse=_get_optional_value(litellm_params, optional_params, "coarse"),
+        min_chars_to_compress=_get_optional_value(litellm_params, optional_params, "min_chars_to_compress"),
+        compress_tool_outputs=_get_optional_value(litellm_params, optional_params, "compress_tool_outputs"),
+        compress_system=_get_optional_value(litellm_params, optional_params, "compress_system"),
+        compress_history=_get_optional_value(litellm_params, optional_params, "compress_history"),
+        compress_last_user=_get_optional_value(litellm_params, optional_params, "compress_last_user"),
+        enable_retrieval=_get_optional_value(litellm_params, optional_params, "enable_retrieval"),
+        max_bytes_per_call=_get_optional_value(litellm_params, optional_params, "max_bytes_per_call"),
+        allow_bypass_header=_get_optional_value(litellm_params, optional_params, "allow_bypass_header"),
+        dynamic=_get_optional_value(litellm_params, optional_params, "dynamic"),
+        dynamic_min_ratio=_get_optional_value(litellm_params, optional_params, "dynamic_min_ratio"),
+        dynamic_max_ratio=_get_optional_value(litellm_params, optional_params, "dynamic_max_ratio"),
+        compression_params=_get_optional_value(litellm_params, optional_params, "compression_params"),
+        guardrail_name=guardrail["guardrail_name"],
+        event_hook=_coerce_event_hook(litellm_params.mode),
+        default_on=litellm_params.default_on or False,
+        unreachable_fallback=litellm_params.unreachable_fallback,
+    )
+    litellm.logging_callback_manager.add_litellm_callback(  # pyright: ignore[reportUnknownMemberType]  # callback manager is untyped
+        _callback
+    )
+    return _callback
+
+
+guardrail_initializer_registry = {
+    SupportedGuardrailIntegrations.COMPRESR.value: initialize_guardrail,
+}
+
+guardrail_class_registry = {
+    SupportedGuardrailIntegrations.COMPRESR.value: CompresrGuardrail,
+}

litellm/proxy/guardrails/guardrail_registry.py

@@ -494,6 +494,7 @@ class InMemoryGuardrailHandler:
             guardrail_id=guardrail.get("guardrail_id"),
             guardrail_name=guardrail["guardrail_name"],
             litellm_params=litellm_params,
+            guardrail_info=guardrail.get("guardrail_info"),
         )
 
         # store references to the guardrail in memory
@@ -612,6 +613,27 @@ class InMemoryGuardrailHandler:
         """
         return self._sources.get(guardrail_id)
 
+    def list_config_guardrails(self) -> List[Guardrail]:
+        """
+        List in-memory guardrails owned by config.yaml.
+
+        DB-sourced entries are excluded: a read surface that also queries the DB
+        would double-count live ones, and a DB-sourced entry that's missing from
+        the DB is stale (deleted on another pod, awaiting reconciliation here).
+        """
+        return [g for gid, g in self.IN_MEMORY_GUARDRAILS.items() if self._sources.get(gid) == "config"]
+
+    def get_config_guardrail_by_id(self, guardrail_id: str) -> Optional[Guardrail]:
+        """
+        Get a config-owned in-memory guardrail by its ID, or None.
+
+        Mirrors the fallback in get_guardrail_info: a DB-sourced in-memory entry
+        that missed the DB lookup is stale and must not be surfaced.
+        """
+        if self._sources.get(guardrail_id) != "config":
+            return None
+        return self.IN_MEMORY_GUARDRAILS.get(guardrail_id)
+
     def reconcile_db_guardrails(self, db_guardrail_ids: Set[str]) -> List[str]:
         """
         Drop in-memory entries that originated from the DB but are no longer

litellm/proxy/guardrails/usage_endpoints.py

@@ -137,10 +137,26 @@ def _chart_from_metrics(metrics: Any) -> List[Dict[str, Any]]:
     return [{"date": d, "passed": v["passed"], "blocked": v["blocked"]} for d, v in sorted(chart_by_date.items())]
 
 
+def _get_guardrail_field(g: Any, field: str) -> Any:
+    """Read `field` off a guardrail whether it's a Prisma row (attr) or a dict/TypedDict (key)."""
+    if isinstance(g, dict):
+        return g.get(field)
+    return getattr(g, field, None)
+
+
+def _to_dict(value: Any) -> Dict[str, Any]:
+    """Coerce a pydantic model (e.g. LitellmParams) / dict value into a plain dict."""
+    if isinstance(value, BaseModel):
+        return value.model_dump(exclude_none=True)
+    if isinstance(value, dict):
+        return value
+    return {}
+
+
 def _get_guardrail_attrs(g: Any) -> tuple[Any, str]:
     """Get (guardrail_id, display_name) from guardrail - handles Prisma model or dict."""
-    gid = getattr(g, "guardrail_id", None) or (g.get("guardrail_id") if isinstance(g, dict) else None)
-    name = getattr(g, "guardrail_name", None) or (g.get("guardrail_name") if isinstance(g, dict) else None)
+    gid = _get_guardrail_field(g, "guardrail_id")
+    name = _get_guardrail_field(g, "guardrail_name")
     return gid, (name or gid or "")
 
 
@@ -163,9 +179,9 @@ def _guardrail_overview_rows(
                 break
         req, blocked = a["requests"], a["blocked"]
         fail_rate = (100.0 * blocked / req) if req else 0.0
-        litellm_params = (g.litellm_params or {}) if isinstance(g.litellm_params, dict) else {}
+        litellm_params = _to_dict(_get_guardrail_field(g, "litellm_params"))
         provider = str(litellm_params.get("guardrail", "Unknown"))
-        guardrail_info = (g.guardrail_info or {}) if isinstance(g.guardrail_info, dict) else {}
+        guardrail_info = _to_dict(_get_guardrail_field(g, "guardrail_info"))
         gtype = str(guardrail_info.get("type", "Guardrail"))
         prev_fail = 0.0
         for k in lookup_keys:
@@ -262,9 +278,15 @@ async def guardrails_usage_overview(
     end = end_date or now.strftime("%Y-%m-%d")
     start = start_date or (now - timedelta(days=7)).strftime("%Y-%m-%d")
 
+    from litellm.proxy.guardrails.guardrail_registry import IN_MEMORY_GUARDRAIL_HANDLER
+
     try:
-        # Guardrails from DB
-        guardrails = await GuardrailsRepository(prisma_client).table.find_many()
+        db_guardrails = await GuardrailsRepository(prisma_client).table.find_many()
+        seen_ids = {gid for g in db_guardrails if (gid := _get_guardrail_field(g, "guardrail_id")) is not None}
+        config_guardrails = [
+            g for g in IN_MEMORY_GUARDRAIL_HANDLER.list_config_guardrails() if g.get("guardrail_id") not in seen_ids
+        ]
+        guardrails: List[Any] = [*db_guardrails, *config_guardrails]
 
         # Daily metrics in range
         metrics = await DailyGuardrailMetricsRepository(prisma_client).table.find_many(
@@ -321,16 +343,18 @@ async def guardrails_usage_detail(
     end = end_date or now.strftime("%Y-%m-%d")
     start = start_date or (now - timedelta(days=7)).strftime("%Y-%m-%d")
 
-    guardrail = await GuardrailsRepository(prisma_client).table.find_unique(where={"guardrail_id": guardrail_id})
-    if not guardrail:
+    from litellm.proxy.guardrails.guardrail_registry import IN_MEMORY_GUARDRAIL_HANDLER
+
+    guardrail: Any = await GuardrailsRepository(prisma_client).table.find_unique(where={"guardrail_id": guardrail_id})
+    if guardrail is None:
+        guardrail = IN_MEMORY_GUARDRAIL_HANDLER.get_config_guardrail_by_id(guardrail_id=guardrail_id)
+    if guardrail is None:
         from fastapi import HTTPException
 
         raise HTTPException(status_code=404, detail="Guardrail not found")
 
     # Metrics are keyed by logical name (from spend log metadata), not UUID
-    logical_id = getattr(guardrail, "guardrail_name", None) or (
-        guardrail.get("guardrail_name") if isinstance(guardrail, dict) else None
-    )
+    logical_id = _get_guardrail_field(guardrail, "guardrail_name")
     metric_ids = [i for i in (logical_id, guardrail_id) if i]
 
     metrics = await DailyGuardrailMetricsRepository(prisma_client).table.find_many(
@@ -367,17 +391,9 @@ async def guardrails_usage_detail(
         {"date": d, "passed": v["passed"], "blocked": v["blocked"], "score": None}
         for d, v in sorted(ts_by_date.items())
     ]
-    _litellm_params = getattr(guardrail, "litellm_params", None) or (
-        guardrail.get("litellm_params") if isinstance(guardrail, dict) else None
-    )
-    litellm_params = _litellm_params if isinstance(_litellm_params, dict) else {}
-    _guardrail_info = getattr(guardrail, "guardrail_info", None) or (
-        guardrail.get("guardrail_info") if isinstance(guardrail, dict) else None
-    )
-    guardrail_info = _guardrail_info if isinstance(_guardrail_info, dict) else {}
-    _guardrail_name = getattr(guardrail, "guardrail_name", None) or (
-        guardrail.get("guardrail_name") if isinstance(guardrail, dict) else None
-    )
+    litellm_params = _to_dict(_get_guardrail_field(guardrail, "litellm_params"))
+    guardrail_info = _to_dict(_get_guardrail_field(guardrail, "guardrail_info"))
+    _guardrail_name = _get_guardrail_field(guardrail, "guardrail_name")
 
     return UsageDetailResponse(
         guardrail_id=guardrail_id,
@@ -548,11 +564,15 @@ async def guardrails_usage_logs(
         # Query by both so we match regardless of which was written.
         effective_guardrail_ids: List[str] = [guardrail_id] if guardrail_id else []
         if guardrail_id:
-            guardrail = await GuardrailsRepository(prisma_client).table.find_unique(
+            from litellm.proxy.guardrails.guardrail_registry import IN_MEMORY_GUARDRAIL_HANDLER
+
+            guardrail: Any = await GuardrailsRepository(prisma_client).table.find_unique(
                 where={"guardrail_id": guardrail_id}
             )
+            if guardrail is None:
+                guardrail = IN_MEMORY_GUARDRAIL_HANDLER.get_config_guardrail_by_id(guardrail_id=guardrail_id)
             if guardrail:
-                logical_name = getattr(guardrail, "guardrail_name", None)
+                logical_name = _get_guardrail_field(guardrail, "guardrail_name")
                 if logical_name and logical_name not in effective_guardrail_ids:
                     effective_guardrail_ids.append(logical_name)
 

litellm/proxy/hooks/model_max_budget_limiter.py

@@ -5,7 +5,7 @@ import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.caching.caching import DualCache
 from litellm.integrations.custom_logger import Span
-from litellm.proxy._types import UserAPIKeyAuth
+from litellm.proxy._types import Litellm_EntityType, UserAPIKeyAuth
 from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
 from litellm.types.llms.openai import AllMessageValues
 from litellm.types.utils import (
@@ -76,6 +76,8 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
                     message=f"LiteLLM Virtual Key: {user_api_key_dict.token}, key_alias: {user_api_key_dict.key_alias}, exceeded budget for model={model}",
                     current_cost=_current_spend,
                     max_budget=_current_model_budget_info.max_budget,
+                    entity_type=Litellm_EntityType.KEY.value,
+                    entity_id=user_api_key_dict.token,
                 )
 
         return True
@@ -140,6 +142,8 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
                     message=f"LiteLLM End User: {end_user_id}, exceeded budget for model={model}",
                     current_cost=_current_spend,
                     max_budget=_current_model_budget_info.max_budget,
+                    entity_type=Litellm_EntityType.END_USER.value,
+                    entity_id=end_user_id,
                 )
 
         return True

litellm/proxy/litellm_pre_call_utils.py

@@ -949,6 +949,10 @@ class LiteLLMProxyRequestSetup:
             user_api_key_alias=user_api_key_dict.key_alias,
             user_api_key_spend=user_api_key_dict.spend,
             user_api_key_max_budget=user_api_key_dict.max_budget,
+            user_api_key_user_spend=user_api_key_dict.user_spend,
+            user_api_key_user_max_budget=user_api_key_dict.user_max_budget,
+            user_api_key_team_spend=user_api_key_dict.team_spend,
+            user_api_key_team_max_budget=user_api_key_dict.team_max_budget,
             user_api_key_team_id=user_api_key_dict.team_id,
             user_api_key_project_id=user_api_key_dict.project_id,
             user_api_key_project_alias=user_api_key_dict.project_alias,

litellm/proxy/management_endpoints/key_management_endpoints.py

@@ -32,6 +32,7 @@ from litellm._uuid import uuid
 from litellm.constants import (
     LENGTH_OF_LITELLM_GENERATED_KEY,
     LITELLM_PROXY_ADMIN_NAME,
+    MINIMUM_CUSTOM_KEY_LENGTH,
     UI_SESSION_TOKEN_TEAM_ID,
 )
 from litellm.litellm_core_utils.duration_parser import duration_in_seconds
@@ -1022,6 +1023,14 @@ async def _common_key_generation_helper(
             detail={"error": f"Invalid key format. LiteLLM Virtual Key must start with 'sk-'. Received: {_masked}"},
         )
 
+    if data.key is not None and len(data.key) < MINIMUM_CUSTOM_KEY_LENGTH:
+        raise HTTPException(
+            status_code=400,
+            detail={
+                "error": f"Invalid key format. LiteLLM Virtual Key must be at least {MINIMUM_CUSTOM_KEY_LENGTH} characters long."
+            },
+        )
+
     # check org key limits - done here to handle inheriting org id from team
     if data.organization_id is not None:
         from litellm.proxy.proxy_server import prisma_client, user_api_key_cache
@@ -1474,7 +1483,7 @@ async def generate_key_fn(
     Parameters:
     - duration: Optional[str] - Specify the length of time the token is valid for. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
     - key_alias: Optional[str] - User defined key alias
-    - key: Optional[str] - User defined key value. If not set, a 16-digit unique sk-key is created for you.
+    - key: Optional[str] - User defined key value. Must start with 'sk-' and be at least 16 characters long. If not set, a 16-digit unique sk-key is created for you.
     - team_id: Optional[str] - The team id of the key
     - user_id: Optional[str] - The user id of the key
     - agent_id: Optional[str] - The agent id associated with the key.
@@ -1688,7 +1697,7 @@ async def generate_service_account_key_fn(
     Parameters:
     - duration: Optional[str] - Specify the length of time the token is valid for. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
     - key_alias: Optional[str] - User defined key alias
-    - key: Optional[str] - User defined key value. If not set, a 16-digit unique sk-key is created for you.
+    - key: Optional[str] - User defined key value. Must start with 'sk-' and be at least 16 characters long. If not set, a 16-digit unique sk-key is created for you.
     - team_id: Optional[str] - The team id of the key
     - user_id: Optional[str] - [NON-FUNCTIONAL] THIS WILL BE IGNORED. The user id of the key
     - budget_id: Optional[str] - The budget id associated with the key. Created by calling `/budget/new`.
@@ -4356,7 +4365,6 @@ async def get_new_token(data: Optional[RegenerateKeyRequest]) -> str:
     if data and data.new_key is not None:
         # Reject custom key values if disabled by admin
         await _check_custom_key_allowed(data.new_key)
-        new_token = data.new_key
         if not data.new_key.startswith("sk-"):
             raise HTTPException(
                 status_code=status.HTTP_400_BAD_REQUEST,
@@ -4364,6 +4372,12 @@ async def get_new_token(data: Optional[RegenerateKeyRequest]) -> str:
                     "error": "New key must start with 'sk-'. This is to distinguish a key hash (used by litellm for logging / internal logic) from the actual key."
                 },
             )
+        if len(data.new_key) < MINIMUM_CUSTOM_KEY_LENGTH:
+            raise HTTPException(
+                status_code=status.HTTP_400_BAD_REQUEST,
+                detail={"error": f"New key must be at least {MINIMUM_CUSTOM_KEY_LENGTH} characters long."},
+            )
+        new_token = data.new_key
     else:
         new_token = f"sk-{secrets.token_urlsafe(LENGTH_OF_LITELLM_GENERATED_KEY)}"
     return new_token
@@ -4470,7 +4484,7 @@ async def _execute_virtual_key_regeneration(
 
     new_token = await get_new_token(data=data)
     new_token_hash = hash_token(new_token)
-    new_token_key_name = f"sk-...{new_token[-4:]}"
+    new_token_key_name = abbreviate_api_key(api_key=new_token)
     update_data = {"token": new_token_hash, "key_name": new_token_key_name}
 
     non_default_values = {}
@@ -4550,7 +4564,7 @@ async def regenerate_key_fn(
     - data: Optional[RegenerateKeyRequest] - Request body containing optional parameters to update
         - key: Optional[str] - The key to regenerate.
         - new_master_key: Optional[str] - The new master key to use, if key is the master key.
-        - new_key: Optional[str] - The new key to use, if key is not the master key. If both set, new_master_key will be used.
+        - new_key: Optional[str] - The new key to use, if key is not the master key. Must start with 'sk-' and be at least 16 characters long. If both set, new_master_key will be used.
         - key_alias: Optional[str] - User-friendly key alias
         - user_id: Optional[str] - User ID associated with key
         - team_id: Optional[str] - Team ID associated with key

litellm/proxy/management_endpoints/mcp_management_endpoints.py

@@ -536,6 +536,7 @@ if MCP_AVAILABLE:
         sanitized.env = {}
         sanitized.command = None
         sanitized.args = []
+        sanitized.issuer = None
         sanitized.authorization_url = None
         sanitized.token_url = None
         sanitized.registration_url = None
@@ -581,6 +582,7 @@ if MCP_AVAILABLE:
         sanitized.teams = []
         sanitized.env_vars = None
 
+        sanitized.issuer = None
         sanitized.authorization_url = None
         sanitized.token_url = None
         sanitized.registration_url = None
@@ -686,6 +688,7 @@ if MCP_AVAILABLE:
             command=payload.command,
             args=payload.args,
             env=payload.env,
+            issuer=payload.issuer,
             authorization_url=payload.authorization_url,
             token_url=payload.token_url,
             registration_url=payload.registration_url,

litellm/proxy/management_endpoints/team_endpoints.py

@@ -14,7 +14,7 @@ import json
 import math
 import traceback
 from datetime import datetime, timezone
-from typing import Annotated, Any, Dict, List, Optional, Tuple, Union, cast
+from typing import Annotated, Any, Dict, List, Mapping, Optional, Tuple, Union, cast
 
 import fastapi
 from fastapi import APIRouter, Depends, Header, HTTPException, Request, status
@@ -894,6 +894,17 @@ def _check_team_budget_update_authority(
         )
 
 
+def _should_auto_add_team_creator(
+    user_api_key_dict: UserAPIKeyAuth,
+    general_settings: Mapping[str, object],
+) -> bool:
+    if user_api_key_dict.user_id is None:
+        return False
+    if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN:
+        return True
+    return general_settings.get("disable_auto_add_proxy_admin_to_teams") is not True
+
+
 #### TEAM MANAGEMENT ####
 @router.post(
     "/team/new",
@@ -997,6 +1008,7 @@ async def new_team(
         from litellm.proxy.proxy_server import (
             _license_check,
             create_audit_log_for_update,
+            general_settings,
             litellm_proxy_admin_name,
             prisma_client,
             user_api_key_cache,
@@ -1123,13 +1135,11 @@ async def new_team(
                     user_api_key_cache=user_api_key_cache,
                 )
 
-        if user_api_key_dict.user_id is not None:
-            creating_user_in_list = False
-            for member in data.members_with_roles:
-                if member.user_id == user_api_key_dict.user_id:
-                    creating_user_in_list = True
-
-            if creating_user_in_list is False:
+        if _should_auto_add_team_creator(user_api_key_dict, general_settings):
+            creating_user_in_list = any(
+                member.user_id == user_api_key_dict.user_id for member in data.members_with_roles
+            )
+            if not creating_user_in_list:
                 data.members_with_roles.append(Member(role="admin", user_id=user_api_key_dict.user_id))
 
         _check_passthrough_routes_caller_permission(data, user_api_key_dict, entity="team")
@@ -1621,6 +1631,7 @@ async def update_team(
     - allowed_passthrough_routes: Optional[List[str]] - List of allowed pass through routes for the team.
     - model_rpm_limit: Optional[Dict[str, int]] - The RPM (Requests Per Minute) limit per model for this team. Example: {"gpt-4": 100, "gpt-3.5-turbo": 200}
     - model_tpm_limit: Optional[Dict[str, int]] - The TPM (Tokens Per Minute) limit per model for this team. Example: {"gpt-4": 10000, "gpt-3.5-turbo": 20000}
+    - mcp_rpm_limit: Optional[Dict[str, int]] - Per-MCP-server RPM limit for this team, keyed by MCP server name (alias if set, else the configured name). Example: {"github": 100, "slack": 200}. Applied across all keys for this team.
     Example - update team TPM Limit
     - allowed_vector_store_indexes: Optional[List[dict]] - List of allowed vector store indexes for the key. Example - [{"index_name": "my-index", "index_permissions": ["write", "read"]}]. If specified, the key will only be able to use these specific vector store indexes. Create index, using `/v1/indexes` endpoint.
     - secret_manager_settings: Optional[dict] - Secret manager settings for the team. [Docs](https://docs.litellm.ai/docs/secret_managers/overview)

litellm/proxy/middleware/billable_request_metrics_middleware.py

@@ -0,0 +1,234 @@
+"""
+Counts billable HTTP requests on enterprise deployments.
+
+A billable request is an inbound request to an LLM inference, MCP, or A2A
+endpoint that returns a 2xx status. The actual export happens in an injected
+recorder (see litellm.proxy.enterprise_billing.billing_metrics); when no
+recorder is injected (non-enterprise, or metering misconfigured) this
+middleware is a transparent pass-through.
+"""
+
+import re
+import threading
+from enum import Enum
+from typing import Callable, Optional, Protocol, Sequence, runtime_checkable
+
+from starlette.types import ASGIApp, Message, Receive, Scope, Send
+
+from litellm._logging import verbose_proxy_logger
+from litellm.proxy._types import LiteLLMRoutes
+
+
+class BillableCategory(str, Enum):
+    LLM = "llm"
+    MCP = "mcp"
+    A2A = "a2a"
+
+
+@runtime_checkable
+class BillingRecorder(Protocol):
+    def record(self, *, category: BillableCategory, route: str, status_code: int, model_id: Optional[str]) -> None: ...
+
+
+_MODEL_ID_HEADER = b"x-litellm-model-id"
+
+# Ordered: a longer suffix that shares an ending with a shorter one must come
+# first, e.g. "/chat/completions" before "/completions". This is the POST
+# inference surface that writes a SpendLogs row on success, so the exported
+# count lines up with the admin UI usage page for inference traffic. Billing
+# is a deliberate lower bound on SpendLogs rows: management writes that also
+# log (batch/file/fine-tuning creation, interaction cancel) and non-POST calls
+# that log (passthrough reads) never bill, so drift only ever undercounts.
+_LLM_ROUTE_SUFFIXES: tuple[str, ...] = (
+    "/chat/completions",
+    "/completions",
+    "/embeddings",
+    "/responses",
+    "/rerank",
+    "/moderations",
+    "/images/generations",
+    "/images/edits",
+    "/images/variations",
+    "/audio/transcriptions",
+    "/audio/translations",
+    "/audio/speech",
+    "/videos",  # create; GET list is excluded by the POST gate
+    "/remix",  # /v1/videos/{id}/remix
+    "/ocr",
+    "/search",  # /v1/search and /v1/vector_stores/{id}/search
+    "/rag/query",
+    "/rag/ingest",
+    ":generateContent",  # Gemini-native /v1beta/models/{model}:generateContent
+    ":streamGenerateContent",
+)
+
+# Exact paths only: a suffix match would also catch non-inference resources that
+# share the ending, e.g. the OpenAI Assistants route /v1/threads/{id}/messages
+# writes no SpendLogs row and must not bill, unlike Anthropic /v1/messages.
+_LLM_ROUTE_EXACT: tuple[str, ...] = (
+    "/v1/messages",
+    "/interactions",  # Google Interactions create; /{id} reads and /cancel do not match
+    "/v1beta/interactions",
+)
+
+# Provider passthrough prefixes (e.g. /bedrock/..., /vertex-ai/...) carry real
+# inference calls that write SpendLogs rows, so they bill. Anchored to the
+# routes enum so new providers are picked up without touching this module.
+# /langfuse forwards observability traffic, not inference: it writes no
+# SpendLogs row and must not bill.
+_NON_BILLABLE_PASSTHROUGH_PREFIXES = frozenset({"/langfuse"})
+_PASSTHROUGH_PREFIXES: tuple[str, ...] = tuple(
+    prefix
+    for prefix in LiteLLMRoutes.mapped_pass_through_routes.value
+    if prefix not in _NON_BILLABLE_PASSTHROUGH_PREFIXES
+)
+
+
+def _classify_llm_route(path: str) -> Optional[str]:
+    exact_match = next((route for route in _LLM_ROUTE_EXACT if path == route), None)
+    if exact_match is not None:
+        return exact_match
+    suffix_match = next((suffix for suffix in _LLM_ROUTE_SUFFIXES if path == suffix or path.endswith(suffix)), None)
+    if suffix_match is not None:
+        return suffix_match
+    # Deep passthrough paths only: the bare prefix itself is not an inference call.
+    return next((prefix for prefix in _PASSTHROUGH_PREFIXES if path.startswith(f"{prefix}/")), None)
+
+
+_MCP_MANAGEMENT_PREFIX = "/v1/mcp"
+_MCP_DYNAMIC_TRANSPORT = re.compile(r"/(?:toolset/)?[^/]+/mcp")
+# The REST wrapper's tool-call endpoint executes a tool and fires the same MCP
+# spend logging as the /mcp transport; its list/test siblings do not bill.
+_MCP_REST_TOOL_CALL = "/mcp-rest/tools/call"
+
+_A2A_INVOKE_SUFFIX = "/message/send"
+_A2A_TRANSPORT_PREFIXES: tuple[str, ...] = ("/v1/a2a/", "/a2a/")
+# Bare POST /a2a/{agent_id} carries the JSON-RPC method in the body, not the
+# path. Only message/send and message/stream write a SpendLogs row there; the
+# task RPCs (tasks/get, tasks/cancel, tasks/pushNotificationConfig/*, ...) are
+# forwarded upstream and write none. A path-only classifier cannot separate
+# them, so the bare route does not bill: counting a task RPC would overcount,
+# while missing a bare-path message/send only undercounts, and undercounting is
+# the sole direction this metric is allowed to drift. The /mcp transport is
+# method-agnostic by contrast because its list path logs a SpendLogs row too.
+
+
+def _classify_mcp_route(path: str) -> Optional[str]:
+    if path == _MCP_MANAGEMENT_PREFIX or path.startswith(f"{_MCP_MANAGEMENT_PREFIX}/"):
+        return None
+    if path == "/mcp" or path.startswith("/mcp/"):
+        return "/mcp"
+    if path == _MCP_REST_TOOL_CALL:
+        return "/mcp"
+    if _MCP_DYNAMIC_TRANSPORT.fullmatch(path) is not None:
+        return "/mcp"
+    return None
+
+
+def _classify_a2a_route(path: str) -> Optional[str]:
+    if path.endswith(_A2A_INVOKE_SUFFIX) and any(path.startswith(prefix) for prefix in _A2A_TRANSPORT_PREFIXES):
+        return "/a2a"
+    return None
+
+
+def classify_billable_request(path: str, method: str = "POST") -> Optional[tuple[BillableCategory, str]]:
+    """Map a request path to its (category, normalized route), or None if not billable."""
+    normalized = path.rstrip("/") or "/"
+
+    mcp_route = _classify_mcp_route(normalized)
+    if mcp_route is not None:
+        return (BillableCategory.MCP, mcp_route)
+
+    a2a_route = _classify_a2a_route(normalized)
+    if a2a_route is not None:
+        return (BillableCategory.A2A, a2a_route)
+
+    # POST-only is a conservative gate: non-POST calls can still write a
+    # SpendLogs row (passthrough reads, resource GETs) but must not bill, so
+    # any classifier-vs-dashboard mismatch is an undercount, never an overcount.
+    if method.upper() != "POST":
+        return None
+
+    llm_route = _classify_llm_route(normalized)
+    if llm_route is not None:
+        return (BillableCategory.LLM, llm_route)
+    return None
+
+
+def _extract_model_id(headers: Sequence[tuple[bytes, bytes]]) -> Optional[str]:
+    return next(
+        (value.decode("latin-1") for name, value in headers if name.lower() == _MODEL_ID_HEADER and value),
+        None,
+    )
+
+
+class BillableRequestMetricsMiddleware:
+    """
+    Pure ASGI middleware that records one billable request per 2xx response to a
+    billable endpoint. Modeled on InFlightRequestsMiddleware: it wraps `send`,
+    reads the final status and the x-litellm-model-id header off the
+    `http.response.start` message, and never blocks or fails the request path.
+    """
+
+    def __init__(
+        self,
+        app: ASGIApp,
+        recorder: Optional[BillingRecorder] = None,
+        recorder_factory: Optional[Callable[[], Optional[BillingRecorder]]] = None,
+    ) -> None:
+        self.app = app
+        self.recorder = recorder
+        # The factory defers recorder construction to the first request, AFTER the
+        # startup event has loaded the YAML config's environment_variables (license
+        # and cert env vars). Building at import time captured recorder=None for
+        # deployments configured that way. Resolved exactly once; the result
+        # (including None) is cached.
+        self._recorder_factory = recorder_factory
+        self._resolved = recorder_factory is None
+        self._resolve_lock = threading.Lock()
+
+    def _resolve_recorder(self) -> Optional[BillingRecorder]:
+        if self._resolved:
+            return self.recorder
+        # The lock keeps concurrent first requests from each building their own
+        # MeterProvider (and leaking its background exporter thread).
+        with self._resolve_lock:
+            if not self._resolved:
+                factory = self._recorder_factory
+                self.recorder = factory() if factory is not None else self.recorder
+                self._resolved = True
+        return self.recorder
+
+    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+        if scope["type"] != "http":
+            await self.app(scope, receive, send)
+            return
+
+        recorder = self._resolve_recorder()
+        if recorder is None:
+            await self.app(scope, receive, send)
+            return
+
+        classification = classify_billable_request(scope.get("path", ""), scope.get("method", "POST"))
+        if classification is None:
+            await self.app(scope, receive, send)
+            return
+
+        category, route = classification
+        status_code = 0
+        model_id: Optional[str] = None
+
+        async def send_wrapper(message: Message) -> None:
+            nonlocal status_code, model_id
+            if message["type"] == "http.response.start":
+                status_code = message["status"]
+                model_id = _extract_model_id(message.get("headers", []))
+            await send(message)
+
+        await self.app(scope, receive, send_wrapper)
+
+        if 200 <= status_code < 300:
+            try:
+                recorder.record(category=category, route=route, status_code=status_code, model_id=model_id)
+            except Exception:  # noqa: BLE001 -- metering must never fail a request that was already served
+                verbose_proxy_logger.warning("billable request metering failed for %s", route, exc_info=True)

litellm/proxy/proxy_cli.py

@@ -15,6 +15,7 @@ from dotenv import load_dotenv
 
 import litellm
 from litellm.constants import DEFAULT_NUM_WORKERS_LITELLM_PROXY
+from litellm.proxy.db.query_engine_reaper import start_query_engine_reaper
 from litellm.secret_managers.main import get_secret_bool
 
 if TYPE_CHECKING:
@@ -495,6 +496,7 @@ class ProxyInitializationHelpers:
             gunicorn_options["certfile"] = ssl_certfile_path
             gunicorn_options["keyfile"] = ssl_keyfile_path
 
+        start_query_engine_reaper()
         StandaloneApplication(app=app, options=gunicorn_options).run()  # Run gunicorn
 
     @staticmethod
@@ -1261,6 +1263,8 @@ def run_server(
             if reload:
                 ProxyInitializationHelpers._configure_dev_reload(uvicorn_args, config)
 
+            if num_workers > 1:
+                start_query_engine_reaper()
             uvicorn.run(
                 **uvicorn_args,
                 workers=num_workers,

litellm/proxy/proxy_server.py

@@ -438,10 +438,30 @@ from litellm.proxy.management_helpers.audit_logs import (
     create_object_audit_log,
 )
 from litellm.proxy.memory.memory_endpoints import router as memory_router
+from litellm.proxy.middleware.billable_request_metrics_middleware import (
+    BillableRequestMetricsMiddleware,
+    BillingRecorder,
+)
 from litellm.proxy.plugin_routes import (
-    router as plugin_router,
     register_plugins_from_config,
 )
+from litellm.proxy.plugin_routes import (
+    router as plugin_router,
+)
+
+try:
+    from litellm.proxy.enterprise_billing.billing_metrics import (
+        build_billing_metrics_recorder as _build_billing_metrics_recorder,
+    )
+    from litellm.proxy.enterprise_billing.billing_metrics import (
+        shutdown_billing_metrics_recorder as _shutdown_billing_metrics_recorder,
+    )
+
+    build_billing_metrics_recorder: Optional[Callable[..., Optional[BillingRecorder]]] = _build_billing_metrics_recorder
+    shutdown_billing_metrics_recorder: Optional[Callable[[], None]] = _shutdown_billing_metrics_recorder
+except ImportError:
+    build_billing_metrics_recorder = None
+    shutdown_billing_metrics_recorder = None
 from litellm.proxy.middleware.in_flight_requests_middleware import (
     InFlightRequestsMiddleware,
 )
@@ -461,13 +481,11 @@ from litellm.proxy.openai_files_endpoints.files_endpoints import (
 )
 from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
     passthrough_endpoint_router,
+    vertex_ai_live_websocket_passthrough,
 )
 from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
     router as llm_passthrough_router,
 )
-from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
-    vertex_ai_live_websocket_passthrough,
-)
 from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
     initialize_pass_through_endpoints,
 )
@@ -552,22 +570,19 @@ from litellm.types.proxy.management_endpoints.ui_sso import (
 from litellm.types.realtime import RealtimeQueryParams
 from litellm.types.router import (
     DeploymentTypedDict,
-)
-from litellm.types.router import ModelInfo as RouterModelInfo
-from litellm.types.router import (
     RouterGeneralSettings,
     RoutingPlugin,
     SearchToolTypedDict,
     updateDeployment,
 )
+from litellm.types.router import ModelInfo as RouterModelInfo
 from litellm.types.scheduler import DefaultPriorities
 from litellm.types.secret_managers.main import (
     KeyManagementSettings,
     KeyManagementSystem,
 )
-from litellm.types.utils import CredentialItem, CustomHuggingfaceTokenizer
+from litellm.types.utils import CredentialItem, CustomHuggingfaceTokenizer, RawRequestTypedDict, StandardLoggingPayload
 from litellm.types.utils import ModelInfo as ModelMapInfo
-from litellm.types.utils import RawRequestTypedDict, StandardLoggingPayload
 from litellm.utils import _add_custom_logger_callback_to_specific_event
 
 try:
@@ -768,6 +783,11 @@ async def proxy_shutdown_event():
     if db_writer_client is not None:
         await db_writer_client.close()  # type: ignore[reportGeneralTypeIssues]
 
+    # final flush of billable-request counts: without it, up to one export
+    # interval of enterprise billing data is dropped on every restart
+    if shutdown_billing_metrics_recorder is not None:
+        shutdown_billing_metrics_recorder()
+
     # flush remaining langfuse logs
     if "langfuse" in litellm.success_callback:
         try:
@@ -973,11 +993,11 @@ async def proxy_startup_event(app: FastAPI):
         if is_otel_v2_enabled():
             from opentelemetry import trace as _otel_trace
 
-            from litellm.litellm_core_utils.litellm_logging import _in_memory_loggers
             from litellm.integrations.otel.logger import (
                 OpenTelemetryV2,
                 publish_global_otel_v2_provider,
             )
+            from litellm.litellm_core_utils.litellm_logging import _in_memory_loggers
 
             registered = open_telemetry_logger if isinstance(open_telemetry_logger, OpenTelemetryV2) else None
             publish_global_otel_v2_provider(
@@ -1781,6 +1801,31 @@ app.add_middleware(
 )
 
 app.add_middleware(PrometheusAuthMiddleware)
+# Added before InFlightRequestsMiddleware so it nests *inside* it: Starlette
+# makes the last-added middleware outermost. The billable count is recorded
+# after the inner app returns, so if this sat outside the in-flight tracker a
+# request could be counted as drained while its record() had not yet run, and
+# proxy_shutdown_event could flush and stop the exporter underneath it.
+app.add_middleware(
+    BillableRequestMetricsMiddleware,
+    # Factory, not an instance: the recorder is resolved on the first request so
+    # it sees premium_user and the billing env vars AFTER proxy_startup_event has
+    # loaded the YAML config's environment_variables. Building it here at import
+    # time would permanently capture recorder=None for YAML-configured
+    # deployments. The lambda reads the module globals at call time.
+    recorder_factory=lambda: (
+        build_billing_metrics_recorder(
+            premium=premium_user,
+            # Read from the license check, not the premium_user_data module
+            # global: that global is bound once at import and goes stale when
+            # the license arrives via the YAML config's environment_variables.
+            license_data=_license_check.airgapped_license_data,
+            litellm_version=version,
+        )
+        if build_billing_metrics_recorder is not None
+        else None
+    ),
+)
 app.add_middleware(InFlightRequestsMiddleware)
 app.add_middleware(SecurityHeadersMiddleware)
 
@@ -4380,6 +4425,15 @@ class ProxyConfig:
                     litellm.default_max_internal_user_budget = float(value)
                     if litellm.max_internal_user_budget is None:
                         litellm.max_internal_user_budget = litellm.default_max_internal_user_budget
+                elif key == "default_internal_user_params" and isinstance(value, dict):
+                    litellm.default_internal_user_params = (
+                        {**value, "max_budget": float(value["max_budget"])}
+                        if value.get("max_budget") is not None
+                        else value
+                    )
+                    verbose_proxy_logger.debug(
+                        f"{blue_color_code} setting litellm.{key}={_redact_general_setting_value(key, litellm.default_internal_user_params, is_full_admin=False)}{reset_color_code}"
+                    )
                 elif key == "custom_provider_map":
                     from litellm.utils import custom_llm_setup
 
@@ -5740,6 +5794,13 @@ class ProxyConfig:
                 # For other types, convert to bool
                 general_settings["store_prompts_in_spend_logs"] = bool(value)
 
+        if "disable_auto_add_proxy_admin_to_teams" in _general_settings:
+            value = _general_settings["disable_auto_add_proxy_admin_to_teams"]
+            if isinstance(value, str):
+                general_settings["disable_auto_add_proxy_admin_to_teams"] = value.lower() == "true"
+            else:
+                general_settings["disable_auto_add_proxy_admin_to_teams"] = value if value is None else bool(value)
+
         ## STORE MODEL IN DB ##
         if "store_model_in_db" in _general_settings:
             value = _general_settings["store_model_in_db"]
@@ -14853,6 +14914,7 @@ async def get_config_list(
         "mcp_required_fields": {"type": "List"},
         "cancel_on_disconnect": {"type": "Boolean"},
         "skip_user_budget_on_team_key": {"type": "Boolean"},
+        "disable_auto_add_proxy_admin_to_teams": {"type": "Boolean"},
     }
 
     return_val = []

litellm/proxy/route_llm_request.py

@@ -6,6 +6,7 @@ from fastapi import HTTPException, status
 
 import litellm
 from litellm.proxy._types import UserAPIKeyAuth
+from litellm.router_utils.common_utils import _is_proxy_admin_request
 
 # Router-internal mock_testing_* flag names — kept in sync with
 # ``litellm.types.router.MockRouterTestingParams`` by the test
@@ -361,8 +362,11 @@ async def route_request(
     for _key in _MOCK_TESTING_KWARG_NAMES:
         data.pop(_key, None)
 
+    data.pop("enable_tag_filtering", None)
+
     team_id = get_team_id_from_data(data)
     router_model_names = llm_router.model_names if llm_router is not None else []
+    is_proxy_admin_without_team = team_id is None and _is_proxy_admin_request(data)
 
     # Preprocess Google GenAI generate content requests
     if route_type in ["agenerate_content", "agenerate_content_stream"]:
@@ -407,6 +411,8 @@ async def route_request(
             "num_retries",
             "timeout",
             "model_group_retry_policy",
+            "routing_strategy",
+            "enable_tag_filtering",
         ]
 
         # Merge override settings into data (only if not already set in request)
@@ -517,6 +523,13 @@ async def route_request(
             data["model"] = team_model_name
             return getattr(llm_router, f"{route_type}")(**data)
 
+        elif (
+            is_proxy_admin_without_team
+            and data["model"] not in router_model_names
+            and data["model"] in llm_router.team_public_model_names
+        ):
+            return getattr(llm_router, f"{route_type}")(**data)
+
         elif data["model"] in router_model_names or llm_router.has_model_id(data["model"]):
             return getattr(llm_router, f"{route_type}")(**data)
 

litellm/proxy/schema.prisma

@@ -325,6 +325,7 @@ model LiteLLM_MCPServerTable {
   command      String?
   args         String[]       @default([])
   env          Json?          @default("{}")
+  issuer             String?
   authorization_url  String?
   token_url          String?
   registration_url   String?

litellm/proxy/spend_tracking/budget_reservation.py

@@ -4,7 +4,7 @@ import asyncio
 import json
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
-from typing import Any, Dict, List, Optional, Sequence, cast
+from typing import Any, Dict, List, Mapping, Optional, Sequence, cast
 
 import litellm
 from litellm._logging import verbose_proxy_logger
@@ -12,6 +12,7 @@ from litellm.caching import DualCache
 from litellm.litellm_core_utils.duration_parser import duration_in_seconds
 from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import select_tier_for_input, tier_rate
 from litellm.proxy._types import (
+    Litellm_EntityType,
     LiteLLM_TeamMembership,
     LiteLLM_TeamTable,
     LiteLLM_UserTable,
@@ -36,6 +37,17 @@ class _BudgetCounter:
     window_start: Optional[datetime] = None
 
 
+_COUNTER_ENTITY_TYPES: Mapping[str, str] = {
+    "Key": Litellm_EntityType.KEY.value,
+    "Team": Litellm_EntityType.TEAM.value,
+    "TeamMember": Litellm_EntityType.TEAM_MEMBER.value,
+    "User": Litellm_EntityType.USER.value,
+    "EndUser": Litellm_EntityType.END_USER.value,
+    "Tag": Litellm_EntityType.TAG.value,
+    "Organization": Litellm_EntityType.ORGANIZATION.value,
+}
+
+
 class _CounterReservationUnavailable(Exception):
     def __init__(
         self,
@@ -108,6 +120,8 @@ async def _apply_over_budget_reservation_policy(
             f"Current cost: {current_spend}, "
             f"Max budget: {counter.max_budget}"
         ),
+        entity_type=_COUNTER_ENTITY_TYPES.get(counter.entity_type),
+        entity_id=counter.spend_log_entity_id or counter.entity_id,
     )
 
 

litellm/proxy/utils.py

@@ -833,7 +833,7 @@ class ProxyLogging:
     def get_combined_callback_list(self, dynamic_success_callbacks: Optional[List], global_callbacks: List) -> List:
         if dynamic_success_callbacks is None:
             return list(global_callbacks)
-        return list(set(dynamic_success_callbacks + global_callbacks))
+        return list(dict.fromkeys(dynamic_success_callbacks + global_callbacks))
 
     def _parse_pre_mcp_call_hook_response(
         self,

litellm/responses/streaming_iterator.py

@@ -718,6 +718,12 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
         except StopAsyncIteration:
             # Normal end of stream - don't log as failure
             raise
+        except (httpx.ReadError, httpx.RemoteProtocolError) as e:
+            self.finished = True
+            if self.completed_response is None:
+                self._handle_failure(e)
+                raise
+            raise StopAsyncIteration from e
         except httpx.HTTPError as e:
             # Handle HTTP errors
             self.finished = True
@@ -794,6 +800,12 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
         except StopIteration:
             # Normal end of stream - don't log as failure
             raise
+        except (httpx.ReadError, httpx.RemoteProtocolError) as e:
+            self.finished = True
+            if self.completed_response is None:
+                self._handle_failure(e)
+                raise
+            raise StopIteration from e
         except httpx.HTTPError as e:
             # Handle HTTP errors
             self.finished = True

litellm/router.py

@@ -26,6 +26,7 @@ from typing import (
     AsyncGenerator,
     Callable,
     Dict,
+    FrozenSet,
     Generator,
     List,
     Literal,
@@ -108,6 +109,7 @@ from litellm.router_utils.clientside_credential_handler import (
     is_clientside_credential,
 )
 from litellm.router_utils.common_utils import (
+    _is_proxy_admin_request,
     filter_team_based_models,
     filter_web_search_deployments,
 )
@@ -249,6 +251,15 @@ else:
     PreRoutingHookResponse = Any
 
 
+def _cost_value_as_float(value: Union[str, int, float, None]) -> Optional[float]:
+    if value is None:
+        return None
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
 class RoutingArgs(enum.Enum):
     ttl = 60  # 1min (RPM/TPM expire key)
 
@@ -494,6 +505,7 @@ class Router:
         self.model_name_to_deployment_indices: Dict[str, List[int]] = {}
         # Maps (team_id, team_public_model_name) -> list of indices in model_list
         self.team_model_to_deployment_indices: Dict[Tuple[str, str], List[int]] = {}
+        self.team_public_model_names: FrozenSet[str] = frozenset()
 
         # Initialize cache attributes that ``_invalidate_model_group_info_cache``
         # touches *before* the first ``set_model_list`` below (which calls
@@ -619,6 +631,8 @@ class Router:
                 routing_strategy_args=routing_strategy_args,
             )
         self._init_routing_groups(self._routing_groups_input)
+        self._override_selectors: dict[str, Any] = {}
+        self._override_selectors_lock = threading.Lock()
         self.access_groups = None
         ## USAGE TRACKING ##
         if isinstance(litellm._async_success_callback, list):
@@ -890,7 +904,9 @@ class Router:
 
         self._unregister_router_selectors(
             [getattr(self, attr, None) for attr in self._DEFAULT_SELECTOR_ATTR_BY_STRATEGY.values()]
+            + list(getattr(self, "_override_selectors", {}).values())
         )
+        self._override_selectors = {}
 
         self.leastbusy_logger: Optional[LeastBusyLoggingHandler] = None
         self.lowesttpm_logger: Optional[LowestTPMLoggingHandler] = None
@@ -980,12 +996,67 @@ class Router:
                 {strategy_value: group_selector} if group_selector is not None else {}
             )
 
-    def _get_routing_context(self, model: str) -> Tuple[Optional[str], Optional[Any]]:
+    _OVERRIDABLE_ROUTING_STRATEGIES: frozenset[str] = frozenset({"simple-shuffle", *_DEFAULT_SELECTOR_ATTR_BY_STRATEGY})
+
+    def _get_request_routing_strategy_override(self, request_kwargs: Optional[dict]) -> Optional[str]:
+        """
+        Reads a per-request `routing_strategy` override (forwarded by the proxy
+        from key/team `router_settings`) out of the request kwargs.
+
+        Only strategies with a per-request-capable selector are honored;
+        anything else (unknown strings, `lar1`, `provider-budget-routing`) is
+        ignored with a warning so a bad value stored on a key or team can
+        never take down that caller's traffic.
+        """
+        if not request_kwargs:
+            return None
+        raw_strategy = request_kwargs.get("routing_strategy")
+        if raw_strategy is None:
+            return None
+        strategy = self._normalize_strategy(raw_strategy) if isinstance(raw_strategy, (str, RoutingStrategy)) else None
+        if not isinstance(strategy, str) or strategy not in self._OVERRIDABLE_ROUTING_STRATEGIES:
+            verbose_router_logger.warning(
+                "Ignoring per-request routing_strategy override '%s'; supported overrides: %s.",
+                raw_strategy,
+                sorted(self._OVERRIDABLE_ROUTING_STRATEGIES),
+            )
+            return None
+        return strategy
+
+    def _get_override_strategy_selector(self, strategy: str) -> Optional[Any]:
+        """
+        Returns the selector for a per-request strategy override.
+
+        Reuses the default group's selector when the override matches the
+        router's configured strategy (so shared state keeps accumulating in
+        one place); otherwise lazily builds one selector per strategy and
+        caches it for the router's lifetime so its usage/latency state
+        persists across requests.
+        """
+        if strategy == self._normalize_strategy(self.routing_strategy):
+            attr = self._DEFAULT_SELECTOR_ATTR_BY_STRATEGY.get(strategy)
+            return getattr(self, attr, None) if attr is not None else None
+        with self._override_selectors_lock:
+            if strategy not in self._override_selectors:
+                self._override_selectors[strategy] = self._build_strategy_selector(
+                    strategy=strategy,
+                    routing_strategy_args={},
+                )
+            return self._override_selectors[strategy]
+
+    def _get_routing_context(
+        self, model: str, request_kwargs: Optional[dict] = None
+    ) -> tuple[Optional[str], Optional[Any]]:
         """
         Resolves the routing strategy and selector to use for the given model.
 
-        Every model belongs to exactly one group: an explicit entry from
-        `routing_groups`, or the implicit `"default"` group driven by the
+        A per-request `routing_strategy` in `request_kwargs` (forwarded by the
+        proxy from key/team `router_settings`) takes precedence over both the
+        model's routing group and the router's top-level strategy, since it is
+        the most specific expression of caller intent.
+
+        Otherwise every model belongs to exactly one group: an explicit entry
+        from `routing_groups`, or the implicit `"default"` group driven by the
         router's top-level `routing_strategy` / `routing_strategy_args`.
 
         `self.routing_strategy` may be either a string or a `RoutingStrategy`
@@ -993,6 +1064,11 @@ class Router:
         string here. Downstream call sites and `_select_deployment_*` arms
         compare against string literals.
         """
+        override = self._get_request_routing_strategy_override(request_kwargs)
+        if override is not None:
+            verbose_router_logger.debug("routing_group=request-override model=%s strategy=%s", model, override)
+            return override, self._get_override_strategy_selector(override)
+
         group_name = self._model_to_group.get(model)
         if group_name is None:
             strategy = self._normalize_strategy(self.routing_strategy)
@@ -2983,7 +3059,7 @@ class Router:
         # here before it's wiped below, instead of relying on that attempt's
         # (possibly still-pending) failure event to do it.
         refund_stale_reservation_before_retry(self.cache, kwargs)
-        set_io_token_rate_limit_request_kwargs(kwargs)
+        set_io_token_rate_limit_request_kwargs(kwargs, store_in_context=deployment_has_io_token_limits(deployment))
 
         ## DEPLOYMENT-LEVEL TAGS
         deployment_tags = deployment.get("litellm_params", {}).get("tags")
@@ -5933,7 +6009,7 @@ class Router:
         input_kwargs: dict,
     ) -> Optional[Any]:
         """Same-model-group retry after a failed deployment; returns None if not applicable."""
-        strategy, _ = self._get_routing_context(original_model_group)
+        strategy, _ = self._get_routing_context(original_model_group, kwargs)
         if strategy != "simple-shuffle":
             return None
 
@@ -7800,6 +7876,7 @@ class Router:
         self.model_id_to_deployment_index_map = {}  # Reset the index
         self.model_name_to_deployment_indices = {}  # Reset the model_name index
         self.team_model_to_deployment_indices = {}  # Reset the team_model index
+        self.team_public_model_names = frozenset()
         # Reset per-strategy router registries so hot-reload doesn't leave
         # stale routers pointing at the old model_list.
         self.quality_routers = {}
@@ -8151,6 +8228,9 @@ class Router:
                 self.team_model_to_deployment_indices[key] = updated_indices
             else:
                 del self.team_model_to_deployment_indices[key]
+        self.team_public_model_names = frozenset(
+            public_model_name for _, public_model_name in self.team_model_to_deployment_indices
+        )
 
     def _update_team_model_index(self, model: dict, idx: int) -> None:
         """
@@ -8164,6 +8244,7 @@ class Router:
         team_public_model_name = (model.get("model_info") or {}).get("team_public_model_name")
         if team_id and team_public_model_name:
             key = (team_id, team_public_model_name)
+            self.team_public_model_names = self.team_public_model_names | frozenset({team_public_model_name})
             if key not in self.team_model_to_deployment_indices:
                 self.team_model_to_deployment_indices[key] = []
             if idx not in self.team_model_to_deployment_indices[key]:
@@ -8742,8 +8823,8 @@ class Router:
                 # Get mode from database model_info if available, otherwise default to "chat"
                 db_model_info = model.get("model_info", {})
                 mode = db_model_info.get("mode", "chat")
-                input_cost_per_token = db_model_info.get("input_cost_per_token")
-                output_cost_per_token = db_model_info.get("output_cost_per_token")
+                input_cost_per_token = _cost_value_as_float(db_model_info.get("input_cost_per_token"))
+                output_cost_per_token = _cost_value_as_float(db_model_info.get("output_cost_per_token"))
 
                 model_info = ModelMapInfo(
                     key=model_group,
@@ -8794,16 +8875,18 @@ class Router:
                     )
                 ):
                     model_group_info.max_output_tokens = model_info["max_output_tokens"]
-                if model_info.get("input_cost_per_token", None) is not None and (
+                _input_cost_per_token = _cost_value_as_float(model_info.get("input_cost_per_token"))
+                if _input_cost_per_token is not None and (
                     model_group_info.input_cost_per_token is None
-                    or (model_info["input_cost_per_token"] or 0.0) > (model_group_info.input_cost_per_token or 0.0)
+                    or _input_cost_per_token > (model_group_info.input_cost_per_token or 0.0)
                 ):
-                    model_group_info.input_cost_per_token = model_info["input_cost_per_token"]
-                if model_info.get("output_cost_per_token", None) is not None and (
+                    model_group_info.input_cost_per_token = _input_cost_per_token
+                _output_cost_per_token = _cost_value_as_float(model_info.get("output_cost_per_token"))
+                if _output_cost_per_token is not None and (
                     model_group_info.output_cost_per_token is None
-                    or (model_info["output_cost_per_token"] or 0.0) > (model_group_info.output_cost_per_token or 0.0)
+                    or _output_cost_per_token > (model_group_info.output_cost_per_token or 0.0)
                 ):
-                    model_group_info.output_cost_per_token = model_info["output_cost_per_token"]
+                    model_group_info.output_cost_per_token = _output_cost_per_token
                 if (
                     model_info.get("supports_parallel_function_calling", None) is not None
                     and model_info["supports_parallel_function_calling"] is True  # type: ignore
@@ -9118,6 +9201,7 @@ class Router:
         """
         self.model_name_to_deployment_indices.clear()
         self.team_model_to_deployment_indices.clear()
+        self.team_public_model_names = frozenset()
 
         for idx, model in enumerate(model_list):
             model_name = model.get("model_name")
@@ -9694,6 +9778,7 @@ class Router:
             "retry_policy",
             "model_group_alias",
             "enable_weighted_failover",
+            "enable_tag_filtering",
         ]
 
         for var in vars_to_include:
@@ -9730,6 +9815,7 @@ class Router:
             "model_group_retry_policy",
             "model_group_alias",
             "enable_weighted_failover",
+            "enable_tag_filtering",
         ]
 
         _int_settings = [
@@ -10026,7 +10112,10 @@ class Router:
         return [m for m in self.model_list if m["litellm_params"]["model"] == model]
 
     def _try_early_resolve_deployments_for_model_not_in_names(
-        self, model: str, request_team_id: Optional[str]
+        self,
+        model: str,
+        request_team_id: Optional[str],
+        include_team_models: bool = False,
     ) -> Optional[Tuple[str, Union[List, Dict]]]:
         """
         When ``model`` is not in ``self.model_names``, try team routes, pattern routes,
@@ -10041,6 +10130,30 @@ class Router:
             team_deployments = self._get_all_deployments(model_name=model, team_id=request_team_id)
             if team_deployments:
                 return model, team_deployments
+        elif include_team_models:
+            team_deployments = [
+                self.model_list[index]
+                for (_, public_model_name), indices in self.team_model_to_deployment_indices.items()
+                if public_model_name == model
+                for index in indices
+            ]
+            team_ids = {
+                team_id
+                for deployment in team_deployments
+                for team_id in [(deployment.get("model_info") or {}).get("team_id")]
+                if team_id is not None
+            }
+            if len(team_ids) > 1:
+                raise litellm.BadRequestError(
+                    message=(
+                        f"Model name '{model}' matches deployments from multiple teams. "
+                        "Specify the deployment ID directly to disambiguate."
+                    ),
+                    model=model,
+                    llm_provider="",
+                )
+            if team_deployments:
+                return model, team_deployments
 
         pattern_deployments = self.pattern_router.get_deployments_by_pattern(
             model=model,
@@ -10105,7 +10218,11 @@ class Router:
         if _model_from_alias is not None:
             model = _model_from_alias
 
-        early = self._try_early_resolve_deployments_for_model_not_in_names(model=model, request_team_id=request_team_id)
+        early = self._try_early_resolve_deployments_for_model_not_in_names(
+            model=model,
+            request_team_id=request_team_id,
+            include_team_models=_is_proxy_admin_request(request_kwargs),
+        )
         if early is not None:
             return early
 
@@ -10422,7 +10539,7 @@ class Router:
             # Resolve the strategy and logger AFTER the pre-routing hook, since
             # the hook can replace `model` and routing-group lookup must key
             # off the final model name.
-            strategy, strategy_selector = self._get_routing_context(model)
+            strategy, strategy_selector = self._get_routing_context(model, request_kwargs)
 
             healthy_deployments = await self.async_get_healthy_deployments(
                 model=model,
@@ -10566,7 +10683,7 @@ class Router:
 
             # 5. Apply load balancing strategy
             start_time = time.perf_counter()
-            strategy, strategy_selector = self._get_routing_context(model)
+            strategy, strategy_selector = self._get_routing_context(model, request_kwargs)
             if strategy == "simple-shuffle":
                 return simple_shuffle(
                     llm_router_instance=self,
@@ -10853,7 +10970,7 @@ class Router:
                 cooldown_list=_cooldown_list,
             )
 
-        strategy, strategy_selector = self._get_routing_context(model)
+        strategy, strategy_selector = self._get_routing_context(model, request_kwargs)
         if strategy == "simple-shuffle":
             # if users pass rpm or tpm, we do a random weighted pick - based on rpm/tpm
             ############## Check 'weight' param set for weighted pick #################
@@ -10993,7 +11110,7 @@ class Router:
             )
 
         # 6. Apply load balancing strategy
-        strategy, strategy_selector = self._get_routing_context(model)
+        strategy, strategy_selector = self._get_routing_context(model, request_kwargs)
         if strategy == "simple-shuffle":
             return simple_shuffle(
                 llm_router_instance=self,

litellm/router_strategy/complexity_router/complexity_router.py

@@ -98,9 +98,9 @@ def _sanitize_user_api_key_auth(auth: Any) -> Any:
     return auth
 
 
-def _classifier_call_metadata(metadata: dict[str, Any] | None) -> dict[str, Any] | None:
+def _classifier_call_metadata(metadata: dict[str, Any] | None) -> dict[str, Any]:
     if not metadata:
-        return metadata
+        return {}
     return {
         k: _sanitize_user_api_key_auth(v) if k == "user_api_key_auth" else v
         for k, v in metadata.items()
@@ -763,8 +763,8 @@ class ComplexityRouter(CustomLogger):
         # embedding call. Forwarding it would let the embedding's cost callback finalize the
         # reservation, so the routed completion's own callback then skips incrementing the
         # key/team budget. Key/team attribution fields are preserved for spend logging.
-        metadata = _classifier_call_metadata(request_kwargs.get("metadata")) or {}
-        litellm_metadata = _classifier_call_metadata(request_kwargs.get("litellm_metadata")) or {}
+        metadata = _classifier_call_metadata(request_kwargs.get("metadata"))
+        litellm_metadata = _classifier_call_metadata(request_kwargs.get("litellm_metadata"))
         query_vector = (
             await encoder.aencode_queries([user_message], metadata=metadata, litellm_metadata=litellm_metadata)
         )[0]

litellm/router_strategy/complexity_router/config.py

@@ -363,10 +363,13 @@ class ComplexityRouterConfig(BaseModel):
 
     # Session affinity: pin the first turn's routed model for the rest of the session
     session_affinity: bool = Field(
-        default=False,
+        default=True,
         description=(
             "When True and a session_id is resolvable on the request, pin the model chosen on the "
-            "session's first turn and reuse it for every later turn, skipping re-classification."
+            "session's first turn and reuse it for every later turn, skipping re-classification. "
+            "On by default so multi-turn sessions stay on one model, preserving provider prompt "
+            "caches and avoiding cross-model conversation-history errors. Set False to reclassify "
+            "every turn."
         ),
     )
     session_affinity_ttl_seconds: int = Field(

litellm/router_strategy/tag_based_routing.py

@@ -160,8 +160,14 @@ async def get_deployments_for_tag(
     Returns a list of deployments that match the requested model and tags in the request.
 
     Executes tag based filtering based on the tags in request metadata and the tags on the deployments
+
+    Runs when the router-level `enable_tag_filtering` is True or the request carries
+    `enable_tag_filtering=True` (set from key/team router_settings by the proxy).
+    A request-level False never disables a router-level True, so per-request settings
+    cannot escape an operator's global tag-routing policy.
     """
-    if llm_router_instance.enable_tag_filtering is not True:
+    request_enable_tag_filtering = request_kwargs.get("enable_tag_filtering") if request_kwargs else None
+    if request_enable_tag_filtering is not True and llm_router_instance.enable_tag_filtering is not True:
         return healthy_deployments
 
     if request_kwargs is None:

litellm/router_utils/common_utils.py

@@ -1,14 +1,27 @@
 import hashlib
 import json
+from collections.abc import Mapping
 from typing import TYPE_CHECKING, Dict, List, Optional, Union
 
 if TYPE_CHECKING:
     from litellm.types.llms.openai import OpenAIFileObject
 
+from litellm.exceptions import BadRequestError
 from litellm.types.router import CredentialLiteLLMParams
 from litellm._logging import verbose_logger
 
 
+def _is_proxy_admin_request(request_kwargs: Optional[Mapping[str, object]]) -> bool:
+    if request_kwargs is None:
+        return False
+    metadata_value = request_kwargs.get("metadata")
+    litellm_metadata_value = request_kwargs.get("litellm_metadata")
+    metadata = metadata_value if isinstance(metadata_value, Mapping) else {}
+    litellm_metadata = litellm_metadata_value if isinstance(litellm_metadata_value, Mapping) else {}
+    user_api_key_auth = metadata.get("user_api_key_auth") or litellm_metadata.get("user_api_key_auth")
+    return getattr(user_api_key_auth, "user_role", None) == "proxy_admin"
+
+
 def get_litellm_params_sensitive_credential_hash(litellm_params: dict) -> str:
     """
     Hash of the credential params, used for mapping the file id to the right model
@@ -59,6 +72,40 @@ def filter_team_based_models(
     metadata = request_kwargs.get("metadata") or {}
     litellm_metadata = request_kwargs.get("litellm_metadata") or {}
     request_team_id = metadata.get("user_api_key_team_id") or litellm_metadata.get("user_api_key_team_id")
+    if request_team_id is None and _is_proxy_admin_request(request_kwargs) and isinstance(healthy_deployments, list):
+        requested_model = (
+            request_kwargs.get("model") or metadata.get("model_group") or litellm_metadata.get("model_group")
+        )
+        candidate_deployments = tuple(
+            (deployment.get("model_name"), deployment.get("model_info") or {}) for deployment in healthy_deployments
+        )
+        team_ids = frozenset(
+            team_id
+            for _, model_info in candidate_deployments
+            for team_id in [model_info.get("team_id")]
+            if team_id is not None
+        )
+        matches_requested_model = (
+            isinstance(requested_model, str)
+            and bool(candidate_deployments)
+            and all(
+                model_info.get("team_id") is not None
+                and (model_name == requested_model or model_info.get("team_public_model_name") == requested_model)
+                for model_name, model_info in candidate_deployments
+            )
+        )
+        if matches_requested_model and len(team_ids) > 1:
+            raise BadRequestError(
+                message=(
+                    f"Model name '{requested_model}' matches deployments from multiple teams. "
+                    "Specify the deployment ID directly to disambiguate."
+                ),
+                model=requested_model,
+                llm_provider="",
+            )
+        if matches_requested_model:
+            return healthy_deployments
+
     ids_to_remove = set()
     if isinstance(healthy_deployments, dict):
         return healthy_deployments

litellm/router_utils/pre_call_checks/io_token_rate_limit_check.py

@@ -43,14 +43,21 @@ ITPM_CACHE_KEY = "_litellm_itpm_cache_key"
 OTPM_CACHE_KEY = "_litellm_otpm_cache_key"
 
 
-def set_io_token_rate_limit_request_kwargs(kwargs: Optional[dict[str, Any]]) -> None:
+def set_io_token_rate_limit_request_kwargs(kwargs: Optional[dict[str, Any]], store_in_context: bool = True) -> None:
     # The reservation sentinels are server-only, but `metadata` is caller
     # controlled on proxy requests. Strip any client-supplied copies here (this
     # runs before the router stashes its own reservation) so a forged
     # reservation can't drive the post-call reconcile/refund against an
     # arbitrary counter and bypass the configured limits.
     _clear_reservation_from_kwargs(kwargs)
-    _io_token_rate_limit_request_kwargs.set(kwargs)
+    # The context slot pins the entire request kwargs (messages included) for
+    # the lifetime of the surrounding context, which outlives the request when
+    # the context is captured by pooled resources (e.g. a redis connection
+    # created mid-request). Only ITPM/OTPM-limited deployments read it, so for
+    # every other deployment overwrite the slot with None instead of the
+    # kwargs; overwriting (rather than skipping) also releases a previous
+    # request's kwargs when a context is reused.
+    _io_token_rate_limit_request_kwargs.set(kwargs if store_in_context else None)
 
 
 def get_io_token_rate_limit_request_kwargs() -> Optional[dict[str, Any]]:

litellm/types/guardrails.py

@@ -56,6 +56,9 @@ from litellm.types.proxy.guardrails.guardrail_hooks.cisco_ai_defense import (
 from litellm.types.proxy.guardrails.guardrail_hooks.headroom import (
     HeadroomGuardrailConfigModel,
 )
+from litellm.types.proxy.guardrails.guardrail_hooks.compresr import (
+    CompresrGuardrailConfigModel,
+)
 
 """
 Pydantic object defining how to set guardrails on litellm proxy
@@ -123,6 +126,7 @@ class SupportedGuardrailIntegrations(Enum):
     VIGIL_GUARD = "vigil_guard"
     REPELLOAI = "repelloai"
     HEADROOM = "headroom"
+    COMPRESR = "compresr"
 
 
 class Role(Enum):
@@ -806,7 +810,7 @@ class BaseLitellmParams(ContentFilterConfigModel):  # works for new and patch up
         default="fail_closed",
         description=(
             "Behavior when a guardrail endpoint is unreachable due to network errors. "
-            "Implemented by guardrail='generic_guardrail_api', 'akto', 'vigil_guard', 'repelloai', and 'headroom'. "
+            "Implemented by guardrail='generic_guardrail_api', 'akto', 'vigil_guard', 'repelloai', 'headroom', and 'compresr'. "
             "'fail_closed' raises an error (default). 'fail_open' logs a critical error and allows the request to proceed."
         ),
     )
@@ -899,6 +903,7 @@ class LitellmParams(
     BedrockGuardrailConfigModel,
     LakeraV2GuardrailConfigModel,
     HeadroomGuardrailConfigModel,
+    CompresrGuardrailConfigModel,
     RepelloAIGuardrailConfigModel,
     LassoGuardrailConfigModel,
     PillarGuardrailConfigModel,
@@ -1034,6 +1039,7 @@ class ApplyGuardrailRequest(BaseModel):
     entities: Optional[List[PiiEntityType]] = None
     input_type: str = "request"
     messages: Optional[List[Dict[str, Any]]] = None
+    metadata: Dict[str, Any] | None = None
 
 
 class ApplyGuardrailResponse(BaseModel):

litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py

@@ -14,3 +14,4 @@ class UsagePerChunk(TypedDict):
     web_search_requests: Optional[int]
     completion_tokens_details: Optional[CompletionTokensDetails]
     prompt_tokens_details: Optional[PromptTokensDetailsWrapper]
+    cost: Optional[float]

litellm/types/mcp_server/mcp_server_manager.py

@@ -17,9 +17,20 @@ MCPInfo = Dict[str, Any]
 
 class MCPOAuthMetadata(BaseModel):
     scopes: Optional[List[str]] = None
+    """Resource-driven scopes for the authorization request: the RFC 9728 protected-resource
+    ``scopes_supported``, or the ``scope`` from the WWW-Authenticate 401 challenge when the resource
+    supplied one, else the authorization server's ``scopes_supported``. This is the scope value a
+    client requests per the MCP authorization spec Scope Selection Strategy; scope minimization and
+    inflation control are the authorization server's and user's job at consent (RFC 6749 §3.3), not
+    the client's."""
     authorization_url: Optional[str] = None
     token_url: Optional[str] = None
     registration_url: Optional[str] = None
+    discovered_issuer: Optional[str] = None
+    """The ``issuer`` the authorization-server metadata document self-attests (RFC 8414). Persisted
+    trust-on-first-use as the server's ``issuer`` when none is configured, so that later rebuilds
+    anchor discovery on it (RFC 8414 §3.3) and a subsequently compromised resource cannot re-point
+    it. Never overwrites an admin-configured issuer."""
     from_origin_fallback: bool = False
     """True when the metadata came from guessing the resource origin as its authorization
     server rather than from an RFC 9728/8414-advertised document. Guessed endpoints are
@@ -54,6 +65,8 @@ class MCPServer(BaseModel):
     # OAuth-specific fields
     client_id: Optional[str] = None
     client_secret: Optional[str] = None
+    issuer: Optional[str] = None
+    issuer_is_anchored: bool = False
     scopes: Optional[List[str]] = None
     authorization_url: Optional[str] = None
     token_url: Optional[str] = None
@@ -122,9 +135,10 @@ class MCPServer(BaseModel):
     # response (supports dot-notation for nested fields, e.g. "team.enterprise_id").
     # Tokens that fail validation are rejected before storage.
     token_validation: Optional[Dict[str, Any]] = None
-    # Optional TTL override (seconds) for the Redis per-user token cache.
-    # Defaults to the token's expires_in minus the expiry buffer, or
-    # MCP_PER_USER_TOKEN_DEFAULT_TTL when expires_in is absent.
+    # Optional TTL override (seconds) for the Redis per-user token cache, capped
+    # at the token's expires_in minus the expiry buffer so a cached entry never
+    # outlives the token. Defaults to the token's expires_in minus the expiry
+    # buffer, or MCP_PER_USER_TOKEN_DEFAULT_TTL when expires_in is absent.
     token_storage_ttl_seconds: Optional[int] = None
     timeout: Optional[float] = None
     # Max concurrent outbound tool calls to this server; excess calls queue.