Merge branch 'litellm_internal_staging' into litellm_internal-tag-usage-scope-26bd

Resolved conflicts:
- litellm/proxy/_types.py: combine HEAD's tag routes and staging's
  compliance_check_routes into internal_user_view_only_routes.
- litellm/proxy/management_endpoints/tag_management_endpoints.py:
  keep HEAD's early dynamic_tag_rows query (needed for scope) and
  fold in staging's start_date/end_date filter; drop staging's
  now-duplicate dynamic-tags block.
- tests/test_litellm/proxy/management_endpoints/test_tag_management_endpoints.py:
  keep both branches' new tests as separate functions.

Also drop the local _is_internal_user_role helper in favor of the
existing LitellmUserRoles.is_internal_user_role property.

.github/workflows/test-unit-proxy-db.yml

@@ -141,6 +141,7 @@ jobs:
               tests/proxy_unit_tests/test_server_root_path.py
               tests/proxy_unit_tests/test_proxy_pass_user_config.py
               tests/proxy_unit_tests/test_proxy_token_counter.py
+              tests/proxy_unit_tests/test_request_size_limit_middleware.py
             workers: 4
             dist: loadscope
             timeout: 15

.gitignore

@@ -100,4 +100,5 @@ STABILIZATION_TODO.md
 **/playwright-report
 **/*.storageState.json
 **/coverage
-test-config
+test-config
+.vscode

AGENTS.md

@@ -241,10 +241,27 @@ When opening issues or pull requests, follow these templates:
 
 ### Running the proxy server
 
-Start the proxy with a config file:
+Create a minimal config file and start the proxy:
+
+```yaml
+# config.yaml
+model_list:
+  - model_name: fake-openai-endpoint
+    litellm_params:
+      model: openai/fake-model
+      api_key: fake-key
+      api_base: https://fake-api.example.com
+
+general_settings:
+  master_key: sk-1234
+
+litellm_settings:
+  drop_params: True
+  telemetry: False
+```
 
 ```bash
-uv run litellm --config dev_config.yaml --port 4000
+uv run litellm --config config.yaml --port 4000
 ```
 
 The proxy takes ~15-20 seconds to fully start (it runs Prisma migrations on boot). Wait for `/health` to return before sending requests. Without a PostgreSQL `DATABASE_URL`, the proxy connects to a default Neon dev database embedded in the `litellm-proxy-extras` package.

CLAUDE.md

@@ -146,7 +146,7 @@ LiteLLM is a unified interface for 100+ LLM providers with two main components:
 - **Bound large result sets.** Prisma materializes full results in memory. For results over ~10 MB, paginate with `take`/`skip` or `cursor`/`take`, always with an explicit `order`. Prefer cursor-based pagination (`skip` is O(n)). Don't paginate naturally small result sets.
 - **Limit fetched columns on wide tables.** Use `select` to fetch only needed fields — returns a partial object, so downstream code must not access unselected fields.
 - **Check index coverage.** For new or modified queries, check `schema.prisma` for a supporting index. Prefer extending an existing index (e.g. `@@index([a])` → `@@index([a, b])`) over adding a new one, unless it's a `@@unique`. Only add indexes for large/frequent queries.
-- **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`, `litellm-js/spend-logs/` for SpendLogs) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`.
+- **Keep schema files in sync.** Apply schema changes to all `schema.prisma` copies (`schema.prisma`, `litellm/proxy/`, `litellm-proxy-extras/`) with a migration under `litellm-proxy-extras/litellm_proxy_extras/migrations/`.
 
 ### Setup Wizard (`litellm/setup_wizard.py`)
 - The wizard is implemented as a single `SetupWizard` class with `@staticmethod` methods — keep it that way. No module-level functions except `run_setup_wizard()` (the public entrypoint) and pure helpers (color, ANSI).

Dockerfile

@@ -1,8 +1,8 @@
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:3258be472764337fd13095bcbb3182da170243b5819fd67ad4c0754590588b31
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:31da6565f35af6401031c1d7aa91dc84ac76c5c48edd17fb90f0ed9e3173c7a9
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 
 FROM $UV_IMAGE AS uvbin
@@ -68,8 +68,8 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 
 USER root
 
-RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile supervisor && \
-    npm install -g npm@11.12.1 tar@7.5.11 glob@13.0.6 @isaacs/brace-expansion@5.0.1 brace-expansion@5.0.5 minimatch@10.2.4 diff@8.0.3 picomatch@4.0.4 && \
+RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile && \
+    npm install -g npm@11.14.0 tar@7.5.11 glob@13.0.6 @isaacs/brace-expansion@5.0.1 brace-expansion@5.0.5 minimatch@10.2.4 diff@8.0.3 picomatch@4.0.4 && \
     GLOBAL="$(npm root -g)" && \
     for pkg in tar glob @isaacs/brace-expansion brace-expansion minimatch diff picomatch; do \
         name="${pkg##*/}"; \
@@ -85,17 +85,17 @@ ENV PATH="/app/.venv/bin:${PATH}"
 
 COPY --from=builder /app /app
 # Prisma binaries live in $HOME/.cache (default prisma-python location),
-# which is /root/.cache here. Copy them from the builder so they survive
-# deployments that volume-mount /app/.cache (e.g. readOnlyRootFilesystem
-# + emptyDir) — otherwise the mount would shadow the baked-in query engine.
-COPY --from=builder /root/.cache /root/.cache
+# which is /root/.cache here. Copy only the Prisma subdirs — copying the
+# whole /root/.cache drags in the uv build cache (~660 MB, includes a
+# setuptools wheel that surfaces as a CVE finding even though it's not
+# on the runtime sys.path).
+COPY --from=builder /root/.cache/prisma /root/.cache/prisma
+COPY --from=builder /root/.cache/prisma-python /root/.cache/prisma-python
 
 RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
     find /app/.venv -type d -path "*/tornado/test" -delete
 
 EXPOSE 4000/tcp
 
-COPY docker/supervisord.conf /etc/supervisord.conf
-
 ENTRYPOINT ["docker/prod_entrypoint.sh"]
 CMD ["--port", "4000"]

codecov.yaml

@@ -1,3 +1,8 @@
+codecov:
+  require_ci_to_pass: false # post coverage status even if CI has unrelated failures
+  notify:
+    wait_for_ci: false # post as soon as expected uploads arrive, don't wait on CI
+
 component_management:
   individual_components:
     - component_id: "Router"
@@ -28,7 +33,7 @@ coverage:
     project:
       default:
         target: auto
-        threshold: 1% # at maximum allow project coverage to drop by 1%
+        threshold: 0% # do not allow project coverage to drop
     patch:
       default:
         target: auto

deploy/Dockerfile.ghcr_base

@@ -1,18 +0,0 @@
-# Use the provided base image
-FROM ghcr.io/berriai/litellm:main-latest@sha256:7c311546c25e7bb6e8cafede9fcd3d0d622ac636b5c9418befaa32e85dfb0186
-
-# Set the working directory to /app
-WORKDIR /app
-
-# Copy the configuration file into the container at /app
-COPY config.yaml .
-
-# Make sure your docker/entrypoint.sh is executable
-# Convert Windows line endings to Unix
-RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
-
-# Expose the necessary port
-EXPOSE 4000/tcp
-
-# Override the CMD instruction with your desired command and arguments
-CMD ["--port", "4000", "--config", "config.yaml", "--detailed_debug", "--run_gunicorn"]

deploy/charts/litellm-helm/templates/deployment.yaml

@@ -100,6 +100,16 @@ spec:
             - name: DATABASE_URL
               value: {{ .Values.db.url | quote }}
             {{- end }}
+            {{- if and .Values.db.useExisting .Values.db.secret.readReplicaUrlKey }}
+            - name: DATABASE_URL_READ_REPLICA
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Values.db.secret.name }}
+                  key: {{ .Values.db.secret.readReplicaUrlKey }}
+            {{- else if .Values.db.readReplicaUrl }}
+            - name: DATABASE_URL_READ_REPLICA
+              value: {{ .Values.db.readReplicaUrl | quote }}
+            {{- end }}
             - name: PROXY_MASTER_KEY
               valueFrom:
                 secretKeyRef:
@@ -129,12 +139,6 @@ spec:
               value: {{ $val | quote }}
             {{- end }}
             {{- end }}
-            {{- if .Values.separateHealthApp }}
-            - name: SEPARATE_HEALTH_APP
-              value: "1"
-            - name: SEPARATE_HEALTH_PORT
-              value: {{ .Values.separateHealthPort | default "8081" | quote }}
-            {{- end }}
             {{- with .Values.extraEnvVars }}
               {{- toYaml . | nindent 12 }}
             {{- end }}
@@ -175,15 +179,10 @@ spec:
             - name: http
               containerPort: {{ .Values.service.port }}
               protocol: TCP
-            {{- if .Values.separateHealthApp }}
-            - name: health
-              containerPort: {{ .Values.separateHealthPort | default 8081 }}
-              protocol: TCP
-            {{- end }}
           livenessProbe:
             httpGet:
               path: {{ .Values.livenessProbe.path | quote }}
-              port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
+              port: "http"
             initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
             timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
@@ -192,7 +191,7 @@ spec:
           readinessProbe:
             httpGet:
               path: {{ .Values.readinessProbe.path | quote }}
-              port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
+              port: "http"
             initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
             timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
@@ -201,7 +200,7 @@ spec:
           startupProbe:
             httpGet:
               path: {{ .Values.startupProbe.path | quote }}
-              port: {{ if .Values.separateHealthApp }}"health"{{ else }}"http"{{ end }}
+              port: "http"
             initialDelaySeconds: {{ .Values.startupProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.startupProbe.periodSeconds }}
             timeoutSeconds: {{ .Values.startupProbe.timeoutSeconds }}

deploy/charts/litellm-helm/values.yaml

@@ -88,12 +88,6 @@ service:
   # optionally specify loadBalancerClass
   # loadBalancerClass: tailscale
 
-# Separate health app configuration
-# When enabled, health checks will use a separate port and the application
-# will receive SEPARATE_HEALTH_APP=1 and SEPARATE_HEALTH_PORT from environment variables
-separateHealthApp: false
-separateHealthPort: 8081
-
 # Probes for LiteLLM gateway container
 livenessProbe:
   path: /health/liveliness
@@ -258,6 +252,26 @@ db:
     passwordKey: password
     # Optional: when set, DATABASE_HOST will be sourced from this secret key instead of db.endpoint
     endpointKey: ""
+    # Optional: when set, DATABASE_URL_READ_REPLICA will be sourced from this
+    # secret key instead of db.readReplicaUrl. Prefer this over the plain
+    # value: read-replica URLs typically embed credentials, and a value
+    # written to db.readReplicaUrl ends up visible in the rendered pod spec
+    # and the Helm release secret.
+    readReplicaUrlKey: ""
+
+  # Optional read-replica routing. When set, the proxy sends read-only
+  # queries (find_*, count, group_by, query_raw/_first) to this URL while
+  # writes continue to go to db.url. Useful for Aurora-style clusters with
+  # separate reader/writer endpoints. Leave empty to keep single-DB behavior.
+  # When IAM_TOKEN_DB_AUTH is enabled, the reader URL is auto-refreshed
+  # alongside the writer (host/port/user/db are parsed from this URL once
+  # at startup; only the IAM token rotates).
+  #
+  # If the URL embeds credentials, prefer db.secret.readReplicaUrlKey over
+  # this field — the plain value is rendered into the pod spec and the
+  # Helm release secret. This field is intended for credential-less URLs
+  # only (e.g. when IAM_TOKEN_DB_AUTH supplies the token at runtime).
+  readReplicaUrl: ""
 
   # Use the Stackgres Helm chart to deploy an instance of a Stackgres cluster.
   #  The Stackgres Operator must already be installed within the target

deploy/kubernetes/kub.yaml

@@ -1,56 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: litellm-deployment
-spec:
-  replicas: 3
-  selector:
-    matchLabels:
-      app: litellm
-  template:
-    metadata:
-      labels:
-        app: litellm
-    spec:
-      containers:
-        - name: litellm-container
-          image: ghcr.io/berriai/litellm:main-latest
-          imagePullPolicy: Always
-          env:
-            - name: AZURE_API_KEY
-              value: "d6f****"
-            - name: AZURE_API_BASE
-              value: "https://openai"
-            - name: LITELLM_MASTER_KEY
-              value: "sk-1234"
-            - name: DATABASE_URL
-              value: "postgresql://ishaan*********"
-          args:
-            - "--config"
-            - "/app/proxy_config.yaml"  # Update the path to mount the config file
-          volumeMounts:                 # Define volume mount for proxy_config.yaml
-            - name: config-volume
-              mountPath: /app
-              readOnly: true
-          livenessProbe:
-            httpGet:
-              path: /health/liveliness
-              port: 4000
-            initialDelaySeconds: 120
-            periodSeconds: 15
-            successThreshold: 1
-            failureThreshold: 3
-            timeoutSeconds: 10
-          readinessProbe:
-            httpGet:
-              path: /health/readiness
-              port: 4000
-            initialDelaySeconds: 120
-            periodSeconds: 15
-            successThreshold: 1
-            failureThreshold: 3
-            timeoutSeconds: 10
-      volumes:  # Define volume to mount proxy_config.yaml
-        - name: config-volume
-          configMap:
-            name: litellm-config  

deploy/kubernetes/service.yaml

@@ -1,12 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: litellm-service
-spec:
-  selector:
-    app: litellm
-  ports:
-    - protocol: TCP
-      port: 4000
-      targetPort: 4000
-  type: LoadBalancer

dev_config.yaml

@@ -1,13 +0,0 @@
-model_list:
-  - model_name: fake-openai-endpoint
-    litellm_params:
-      model: openai/fake-model
-      api_key: fake-key
-      api_base: https://exampleopenaiendpoint-production.up.railway.app/
-
-general_settings:
-  master_key: sk-1234
-
-litellm_settings:
-  drop_params: True
-  telemetry: False

docker-compose.yml

@@ -16,6 +16,11 @@ services:
       - "4000:4000" # Map the container port to the host, change the host port if necessary
     environment:
       DATABASE_URL: "postgresql://llmproxy:dbpassword9090@db:5432/litellm"
+      # Optional: route read-only queries (find_*, count, group_by, query_raw/_first)
+      # to a separate reader endpoint, e.g. an Aurora reader. Leave unset for
+      # single-DB deployments. With IAM_TOKEN_DB_AUTH enabled, the reader URL
+      # is auto-refreshed alongside the writer.
+      # DATABASE_URL_READ_REPLICA: "postgresql://llmproxy:dbpassword9090@db-reader:5432/litellm"
       STORE_MODEL_IN_DB: "True" # allows adding models to proxy via UI
     env_file:
       - .env # Load local .env file

docker/Dockerfile.alpine

@@ -1,68 +0,0 @@
-# Base image for building
-ARG LITELLM_BUILD_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b80ee99546e749ef82342a419a326620856
-
-# Runtime image
-ARG LITELLM_RUNTIME_IMAGE=python:3.11-alpine@sha256:f07e2ace46f560f09a6eeec7b4913b80ee99546e749ef82342a419a326620856
-ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
-
-FROM $UV_IMAGE AS uvbin
-
-FROM $LITELLM_BUILD_IMAGE AS builder
-
-WORKDIR /app
-
-COPY --from=uvbin /uv /usr/local/bin/uv
-COPY --from=uvbin /uvx /usr/local/bin/uvx
-
-RUN apk add --no-cache gcc python3-dev musl-dev nodejs npm libsndfile
-
-ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
-    UV_PROJECT_ENVIRONMENT=/app/.venv \
-    UV_LINK_MODE=copy \
-    XDG_CACHE_HOME=/app/.cache \
-    PATH="/app/.venv/bin:${PATH}"
-
-# Copy dependency metadata first for layer caching
-COPY pyproject.toml uv.lock ./
-COPY enterprise/pyproject.toml enterprise/
-COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/
-
-# Install third-party dependencies (cached unless pyproject.toml/uv.lock change)
-RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
-    --extra proxy \
-    --extra proxy-runtime \
-    --extra extra_proxy \
-    --extra semantic-router \
-    --python python3
-
-# Copy full source tree
-COPY . .
-
-# Install project and workspace packages (fast - deps already cached)
-RUN uv sync --frozen --no-default-groups --no-editable \
-    --extra proxy \
-    --extra proxy-runtime \
-    --extra extra_proxy \
-    --extra semantic-router \
-    --python python3
-
-RUN prisma generate --schema=./schema.prisma
-
-RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
-    sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
-
-FROM $LITELLM_RUNTIME_IMAGE AS runtime
-
-RUN apk upgrade --no-cache && apk add --no-cache libsndfile nodejs npm
-
-WORKDIR /app
-ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
-    XDG_CACHE_HOME=/app/.cache \
-    PATH="/app/.venv/bin:${PATH}"
-
-COPY --from=builder /app /app
-
-EXPOSE 4000/tcp
-
-ENTRYPOINT ["docker/prod_entrypoint.sh"]
-CMD ["--port", "4000"]

docker/Dockerfile.custom_ui

@@ -1,86 +0,0 @@
-# Use the provided base image
-# NOTE: This is a dev/branch-specific tag. Update digest when the base image is rebuilt.
-FROM ghcr.io/berriai/litellm:litellm_fwd_server_root_path-dev
-
-# Set the working directory to /app
-WORKDIR /app
-
-# Install Node.js and npm (adjust version as needed)
-RUN apt-get update && apt-get upgrade -y \
-        libxml2 \
-        libexpat1 \
-        openssl \
-        libssl3 \
-        git \
-        libkrb5-3 \
-        libglib2.0-0 \
-        wget \
-        libaom3 \
-        libxslt1.1 \
-        libgnutls30 \
-        libc6 && \
-    apt-get install -y --no-install-recommends nodejs npm && \
-    npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
-    GLOBAL="$(npm root -g)" && \
-    find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
-    done && \
-    find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
-    done && \
-    find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
-    done && \
-    find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
-    done && \
-    find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
-        rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
-    done && \
-    find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
-        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
-    npm cache clean --force && \
-    apt-get purge -y npm
-
-# Copy the UI source into the container
-COPY ./ui/litellm-dashboard /app/ui/litellm-dashboard
-
-# Set an environment variable for UI_BASE_PATH
-# This can be overridden at build time
-# set UI_BASE_PATH to "<your server root path>/ui"
-ENV UI_BASE_PATH="/prod/ui"
-
-# Build the UI with the specified UI_BASE_PATH
-WORKDIR /app/ui/litellm-dashboard
-RUN npm ci
-RUN UI_BASE_PATH=$UI_BASE_PATH npm run build
-
-# Create the destination directory
-RUN mkdir -p /app/litellm/proxy/_experimental/out
-
-# Move the built files to the appropriate location
-# Assuming the build output is in ./out directory
-RUN rm -rf /app/litellm/proxy/_experimental/out/* && \
-    mv ./out/* /app/litellm/proxy/_experimental/out/
-
-# Switch back to the main app directory
-WORKDIR /app
-
-# Make sure your docker/entrypoint.sh is executable
-# Convert Windows line endings to Unix for entrypoint scripts
-RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh
-RUN sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
-
-# Run as non-root user
-RUN groupadd --gid 1000 appuser && useradd --uid 1000 --gid 1000 --no-create-home appuser \
-    && chown -R appuser:appuser /app
-USER appuser
-
-# Expose the necessary port
-EXPOSE 4000/tcp
-
-HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
-  CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:4000/health')"]
-
-# Override the CMD instruction with your desired command and arguments
-CMD ["--port", "4000", "--config", "config.yaml", "--detailed_debug"]

docker/Dockerfile.database

@@ -66,7 +66,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
 
 USER root
 
-RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile supervisor && \
+RUN apk add --no-cache bash openssl tzdata nodejs npm python3 libsndfile && \
     npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
     GLOBAL="$(npm root -g)" && \
     find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
@@ -102,7 +102,5 @@ RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
 
 EXPOSE 4000/tcp
 
-COPY docker/supervisord.conf /etc/supervisord.conf
-
 ENTRYPOINT ["docker/prod_entrypoint.sh"]
 CMD ["--port", "4000"]

docker/Dockerfile.dev

@@ -1,121 +0,0 @@
-# Base image for building
-ARG LITELLM_BUILD_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d
-
-# Runtime image
-ARG LITELLM_RUNTIME_IMAGE=python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d
-ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
-
-FROM $UV_IMAGE AS uvbin
-
-FROM $LITELLM_BUILD_IMAGE AS builder
-
-WORKDIR /app
-USER root
-
-COPY --from=uvbin /uv /usr/local/bin/uv
-COPY --from=uvbin /uvx /usr/local/bin/uvx
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    gcc \
-    g++ \
-    python3-dev \
-    libssl-dev \
-    pkg-config \
-    nodejs \
-    npm \
-    && rm -rf /var/lib/apt/lists/*
-
-ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
-    UV_PROJECT_ENVIRONMENT=/app/.venv \
-    UV_LINK_MODE=copy \
-    XDG_CACHE_HOME=/app/.cache \
-    PATH="/app/.venv/bin:${PATH}"
-
-# Copy dependency metadata first for layer caching
-COPY pyproject.toml uv.lock ./
-COPY enterprise/pyproject.toml enterprise/
-COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/
-
-# Install third-party dependencies (cached unless pyproject.toml/uv.lock change)
-RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
-    --extra proxy \
-    --extra proxy-runtime \
-    --extra extra_proxy \
-    --extra semantic-router \
-    --python python
-
-# Copy full source tree
-COPY . .
-
-# Build Admin UI before final sync
-RUN sed -i 's/\r$//' docker/build_admin_ui.sh && chmod +x docker/build_admin_ui.sh && ./docker/build_admin_ui.sh
-
-# Install project and workspace packages (fast - deps already cached)
-RUN uv sync --frozen --no-default-groups --no-editable \
-    --extra proxy \
-    --extra proxy-runtime \
-    --extra extra_proxy \
-    --extra semantic-router \
-    --python python
-
-RUN prisma generate --schema=./schema.prisma
-
-RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
-    sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
-
-FROM $LITELLM_RUNTIME_IMAGE AS runtime
-
-USER root
-
-RUN apt-get update && apt-get upgrade -y \
-        libxml2 \
-        libexpat1 \
-        openssl \
-        libssl3 \
-        git \
-        libkrb5-3 \
-        libglib2.0-0 \
-        wget \
-        libaom3 \
-        libxslt1.1 \
-        libgnutls30 \
-        libc6 \
-    && apt-get install -y --no-install-recommends \
-        libssl3 \
-        libatomic1 \
-        nodejs \
-        npm \
-    && rm -rf /var/lib/apt/lists/* \
-    && npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 \
-    && GLOBAL="$(npm root -g)" \
-    && find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
-          rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
-       done \
-    && find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
-          rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
-       done \
-    && find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
-          rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
-       done \
-    && find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
-          rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
-       done \
-    && find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
-          rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
-       done \
-    && find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
-        sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null \
-    && npm cache clean --force \
-    && apt-get purge -y npm
-
-WORKDIR /app
-ENV PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
-    XDG_CACHE_HOME=/app/.cache \
-    PATH="/app/.venv/bin:${PATH}"
-
-COPY --from=builder /app /app
-
-EXPOSE 4000/tcp
-
-ENTRYPOINT ["docker/prod_entrypoint.sh"]
-CMD ["--port", "4000"]

docker/Dockerfile.health_check

@@ -1,30 +0,0 @@
-ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
-FROM $UV_IMAGE AS uvbin
-
-FROM python:3.13-slim@sha256:739e7213785e88c0f702dcdc12c0973afcbd606dbf021a589cab77d6b00b579d
-
-WORKDIR /app
-
-# Copy the uv binary and the health check script.
-COPY --from=uvbin /uv /usr/local/bin/uv
-COPY pyproject.toml uv.lock /app/
-COPY scripts/health_check/health_check_client.py /app/health_check_client.py
-
-# Resolve and install the health-check dependencies from the project lockfile
-# so the runtime image stays self-contained and reproducible.
-RUN uv export --frozen --no-default-groups --only-group healthcheck --no-emit-project --no-hashes --output-file /tmp/health-check-requirements.txt \
-  && uv pip install --system -r /tmp/health-check-requirements.txt \
-  && rm /tmp/health-check-requirements.txt \
-  && rm /app/pyproject.toml /app/uv.lock \
-  && chmod +x /app/health_check_client.py
-
-# Run as non-root user
-RUN groupadd --gid 1000 appuser && useradd --uid 1000 --gid 1000 --no-create-home appuser
-USER appuser
-
-# Health check
-HEALTHCHECK --interval=30s --timeout=5s --start-period=5s --retries=3 \
-  CMD ["python", "/app/health_check_client.py", "--help"]
-
-# Set entrypoint
-ENTRYPOINT ["python", "/app/health_check_client.py"]

docker/Dockerfile.non_root

@@ -103,13 +103,12 @@ RUN for i in 1 2 3; do \
       apk upgrade --no-cache && break || sleep 5; \
     done && \
     for i in 1 2 3; do \
-      apk add --no-cache python3 bash openssl tzdata supervisor libsndfile nodejs && break || sleep 5; \
+      apk add --no-cache python3 bash openssl tzdata libsndfile nodejs && break || sleep 5; \
     done
 
 COPY --from=builder /app /app
 COPY --from=builder /var/lib/litellm/ui /var/lib/litellm/ui
 COPY --from=builder /var/lib/litellm/assets /var/lib/litellm/assets
-COPY --from=builder /app/docker/supervisord.conf /etc/supervisord.conf
 
 ENV PATH="/app/.venv/bin:${PATH}" \
     PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \

docker/prod_entrypoint.sh

@@ -1,14 +1,8 @@
 #!/bin/sh
 
-if [ "$SEPARATE_HEALTH_APP" = "1" ]; then
-    export LITELLM_ARGS="$@"
-    export SUPERVISORD_STOPWAITSECS="${SUPERVISORD_STOPWAITSECS:-3600}"
-    exec supervisord -c /etc/supervisord.conf
-fi
-
 if [ "$USE_DDTRACE" = "true" ]; then
     export DD_TRACE_OPENAI_ENABLED="False"
     exec ddtrace-run litellm "$@"
 else
     exec litellm "$@"
-fi
+fi

docker/supervisord.conf

@@ -1,46 +0,0 @@
-[supervisord]
-nodaemon=true
-loglevel=info
-logfile=/tmp/supervisord.log
-pidfile=/tmp/supervisord.pid
-
-[group:litellm]
-programs=main,health
-
-[program:main]
-command=sh -c 'if [ "$USE_DDTRACE" = "true" ]; then export DD_TRACE_OPENAI_ENABLED="False"; exec ddtrace-run python -m litellm.proxy.proxy_cli --host 0.0.0.0 --port=4000 $LITELLM_ARGS; else exec python -m litellm.proxy.proxy_cli --host 0.0.0.0 --port=4000 $LITELLM_ARGS; fi'
-autostart=true
-autorestart=true
-startretries=3
-priority=1
-exitcodes=0
-stopasgroup=true
-killasgroup=true
-stopwaitsecs=%(ENV_SUPERVISORD_STOPWAITSECS)s
-stdout_logfile=/dev/stdout
-stderr_logfile=/dev/stderr
-stdout_logfile_maxbytes = 0
-stderr_logfile_maxbytes = 0
-environment=PYTHONUNBUFFERED=true
-
-[program:health]
-command=sh -c '[ "$SEPARATE_HEALTH_APP" = "1" ] && exec uvicorn litellm.proxy.health_endpoints.health_app_factory:build_health_app --factory --host 0.0.0.0 --port=${SEPARATE_HEALTH_PORT:-4001} || exit 0'
-autostart=true
-autorestart=true
-startretries=3
-priority=2
-exitcodes=0
-stopasgroup=true
-killasgroup=true
-stopwaitsecs=%(ENV_SUPERVISORD_STOPWAITSECS)s
-stdout_logfile=/dev/stdout
-stderr_logfile=/dev/stderr
-stdout_logfile_maxbytes = 0
-stderr_logfile_maxbytes = 0
-environment=PYTHONUNBUFFERED=true
-
-[eventlistener:process_monitor]
-command=python -c "from supervisor import childutils; import os, signal; [os.kill(os.getppid(), signal.SIGTERM) for h,p in iter(lambda: childutils.listener.wait(), None) if h['eventname'] in ['PROCESS_STATE_FATAL', 'PROCESS_STATE_EXITED'] and dict([x.split(':') for x in p.split(' ')])['processname'] in ['main', 'health'] or childutils.listener.ok()]"
-events=PROCESS_STATE_EXITED,PROCESS_STATE_FATAL
-autostart=true
-autorestart=true

index.yaml

@@ -1,108 +0,0 @@
-apiVersion: v1
-entries:
-  litellm-helm:
-  - apiVersion: v2
-    appVersion: v1.43.18
-    created: "2024-08-19T23:58:25.331689+08:00"
-    dependencies:
-    - condition: db.deployStandalone
-      name: postgresql
-      repository: oci://registry-1.docker.io/bitnamicharts
-      version: '>=13.3.0'
-    - condition: redis.enabled
-      name: redis
-      repository: oci://registry-1.docker.io/bitnamicharts
-      version: '>=18.0.0'
-    description: Call all LLM APIs using the OpenAI format
-    digest: 0411df3dc42868be8af3ad3e00cb252790e6bd7ad15f5b77f1ca5214573a8531
-    name: litellm-helm
-    type: application
-    urls:
-    - https://berriai.github.io/litellm/litellm-helm-0.2.3.tgz
-    version: 0.2.3
-  postgresql:
-  - annotations:
-      category: Database
-      images: |
-        - name: os-shell
-          image: docker.io/bitnami/os-shell:12-debian-12-r16
-        - name: postgres-exporter
-          image: docker.io/bitnami/postgres-exporter:0.15.0-debian-12-r14
-        - name: postgresql
-          image: docker.io/bitnami/postgresql:16.2.0-debian-12-r6
-      licenses: Apache-2.0
-    apiVersion: v2
-    appVersion: 16.2.0
-    created: "2024-08-19T23:58:25.335716+08:00"
-    dependencies:
-    - name: common
-      repository: oci://registry-1.docker.io/bitnamicharts
-      tags:
-      - bitnami-common
-      version: 2.x.x
-    description: PostgreSQL (Postgres) is an open source object-relational database
-      known for reliability and data integrity. ACID-compliant, it supports foreign
-      keys, joins, views, triggers and stored procedures.
-    digest: 3c8125526b06833df32e2f626db34aeaedb29d38f03d15349db6604027d4a167
-    home: https://bitnami.com
-    icon: https://bitnami.com/assets/stacks/postgresql/img/postgresql-stack-220x234.png
-    keywords:
-    - postgresql
-    - postgres
-    - database
-    - sql
-    - replication
-    - cluster
-    maintainers:
-    - name: VMware, Inc.
-      url: https://github.com/bitnami/charts
-    name: postgresql
-    sources:
-    - https://github.com/bitnami/charts/tree/main/bitnami/postgresql
-    urls:
-    - https://berriai.github.io/litellm/charts/postgresql-14.3.1.tgz
-    version: 14.3.1
-  redis:
-  - annotations:
-      category: Database
-      images: |
-        - name: kubectl
-          image: docker.io/bitnami/kubectl:1.29.2-debian-12-r3
-        - name: os-shell
-          image: docker.io/bitnami/os-shell:12-debian-12-r16
-        - name: redis
-          image: docker.io/bitnami/redis:7.2.4-debian-12-r9
-        - name: redis-exporter
-          image: docker.io/bitnami/redis-exporter:1.58.0-debian-12-r4
-        - name: redis-sentinel
-          image: docker.io/bitnami/redis-sentinel:7.2.4-debian-12-r7
-      licenses: Apache-2.0
-    apiVersion: v2
-    appVersion: 7.2.4
-    created: "2024-08-19T23:58:25.339392+08:00"
-    dependencies:
-    - name: common
-      repository: oci://registry-1.docker.io/bitnamicharts
-      tags:
-      - bitnami-common
-      version: 2.x.x
-    description: Redis(R) is an open source, advanced key-value store. It is often
-      referred to as a data structure server since keys can contain strings, hashes,
-      lists, sets and sorted sets.
-    digest: b2fa1835f673a18002ca864c54fadac3c33789b26f6c5e58e2851b0b14a8f984
-    home: https://bitnami.com
-    icon: https://bitnami.com/assets/stacks/redis/img/redis-stack-220x234.png
-    keywords:
-    - redis
-    - keyvalue
-    - database
-    maintainers:
-    - name: VMware, Inc.
-      url: https://github.com/bitnami/charts
-    name: redis
-    sources:
-    - https://github.com/bitnami/charts/tree/main/bitnami/redis
-    urls:
-    - https://berriai.github.io/litellm/charts/redis-18.19.1.tgz
-    version: 18.19.1
-generated: "2024-08-19T23:58:25.322532+08:00"

litellm-js/proxy/.npmrc

@@ -1,5 +0,0 @@
-# Supply-chain hardening
-# Packages needing lifecycle scripts: npm rebuild <pkg>
-ignore-scripts=true
-# Protects local npm install only — npm ci (used in CI) ignores this
-min-release-age=3

litellm-js/proxy/README.md

@@ -1,8 +0,0 @@
-```
-npm install
-npm run dev
-```
-
-```
-npm run deploy
-```

litellm-js/proxy/package.json

@@ -1,14 +0,0 @@
-{
-  "scripts": {
-    "dev": "wrangler dev src/index.ts",
-    "deploy": "wrangler deploy --minify src/index.ts"
-  },
-  "dependencies": {
-    "hono": "4.12.16",
-    "openai": "4.29.2"
-  },
-  "devDependencies": {
-    "@cloudflare/workers-types": "4.20260501.1",
-    "wrangler": "4.87.0"
-  }
-}

litellm-js/proxy/src/index.ts

@@ -1,59 +0,0 @@
-import { Hono } from 'hono'
-import { Context } from 'hono';
-import { bearerAuth } from 'hono/bearer-auth'
-import OpenAI from "openai";
-
-const openai = new OpenAI({
-  apiKey: "sk-1234",
-  baseURL: "https://openai-endpoint.ishaanjaffer0324.workers.dev"
-});
-
-async function call_proxy() {
-  const completion = await openai.chat.completions.create({
-    messages: [{ role: "system", content: "You are a helpful assistant." }],
-    model: "gpt-3.5-turbo",
-  });
-
-  return completion
-}
-
-const app = new Hono()
-
-// Middleware for API Key Authentication
-const apiKeyAuth = async (c: Context, next: Function) => {
-  const apiKey = c.req.header('Authorization');
-  if (!apiKey || apiKey !== 'Bearer sk-1234') {
-    return c.text('Unauthorized', 401);
-  }
-  await next();
-};
-
-
-app.use('/*', apiKeyAuth)
-
-
-app.get('/', (c) => {
-  return c.text('Hello Hono!')
-})
-
-
-
-
-// Handler for chat completions
-const chatCompletionHandler = async (c: Context) => {
-  // Assuming your logic for handling chat completion goes here
-  // For demonstration, just returning a simple JSON response
-  const response = await call_proxy()
-  return c.json(response);
-};
-
-// Register the above handler for different POST routes with the apiKeyAuth middleware
-app.post('/v1/chat/completions', chatCompletionHandler);
-app.post('/chat/completions', chatCompletionHandler);
-
-// Example showing how you might handle dynamic segments within the URL
-// Here, using ':model*' to capture the rest of the path as a parameter 'model'
-app.post('/openai/deployments/:model*/chat/completions', chatCompletionHandler);
-
-
-export default app

litellm-js/proxy/tsconfig.json

@@ -1,17 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ESNext",
-    "module": "ESNext",
-    "moduleResolution": "Bundler",
-    "strict": true,
-    "lib": [
-      "ESNext"
-    ],
-    "types": [
-      "@cloudflare/workers-types"
-    ],
-    "jsx": "react-jsx",
-    "jsxImportSource": "hono/jsx",
-    "skipLibCheck": true
-  },
-}

litellm-js/proxy/wrangler.toml

@@ -1,18 +0,0 @@
-name = "my-app"
-compatibility_date = "2023-12-01"
-
-# [vars]
-# MY_VAR = "my-variable"
-
-# [[kv_namespaces]]
-# binding = "MY_KV_NAMESPACE"
-# id = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
-
-# [[r2_buckets]]
-# binding = "MY_BUCKET"
-# bucket_name = "my-bucket"
-
-# [[d1_databases]]
-# binding = "DB"
-# database_name = "my-database"
-# database_id = ""

litellm-js/spend-logs/.npmrc

@@ -1,5 +0,0 @@
-# Supply-chain hardening
-# Packages needing lifecycle scripts: npm rebuild <pkg>
-ignore-scripts=true
-# Protects local npm install only — npm ci (used in CI) ignores this
-min-release-age=3

litellm-js/spend-logs/Dockerfile

@@ -1,26 +0,0 @@
-# Use the specific Node.js v20.11.0 image
-FROM node:20.18.1-alpine3.20
-
-# Set the working directory inside the container
-WORKDIR /app
-
-# Copy package.json and package-lock.json to the working directory
-COPY ./litellm-js/spend-logs/package*.json ./
-
-# Install dependencies
-RUN npm ci
-
-# Install Prisma globally
-RUN npm install -g prisma
-
-# Copy the rest of the application code
-COPY ./litellm-js/spend-logs .
-
-# Generate Prisma client
-RUN npx prisma generate
-
-# Expose the port that the Node.js server will run on
-EXPOSE 3000
-
-# Command to run the Node.js app with npm run dev
-CMD ["npm", "run", "dev"]

litellm-js/spend-logs/README.md

@@ -1,8 +0,0 @@
-```
-npm install
-npm run dev
-```
-
-```
-open http://localhost:3000
-```

litellm-js/spend-logs/package-lock.json

@@ -1,597 +0,0 @@
-{
-  "name": "spend-logs",
-  "lockfileVersion": 3,
-  "requires": true,
-  "packages": {
-    "": {
-      "dependencies": {
-        "@hono/node-server": "1.19.13",
-        "hono": "4.12.16"
-      },
-      "devDependencies": {
-        "@types/node": "20.19.25",
-        "tsx": "4.20.6"
-      }
-    },
-    "node_modules/@esbuild/aix-ppc64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.12.tgz",
-      "integrity": "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA==",
-      "cpu": [
-        "ppc64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "aix"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/android-arm": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.12.tgz",
-      "integrity": "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg==",
-      "cpu": [
-        "arm"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "android"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/android-arm64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.12.tgz",
-      "integrity": "sha512-6AAmLG7zwD1Z159jCKPvAxZd4y/VTO0VkprYy+3N2FtJ8+BQWFXU+OxARIwA46c5tdD9SsKGZ/1ocqBS/gAKHg==",
-      "cpu": [
-        "arm64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "android"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/android-x64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.12.tgz",
-      "integrity": "sha512-5jbb+2hhDHx5phYR2By8GTWEzn6I9UqR11Kwf22iKbNpYrsmRB18aX/9ivc5cabcUiAT/wM+YIZ6SG9QO6a8kg==",
-      "cpu": [
-        "x64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "android"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/darwin-arm64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.12.tgz",
-      "integrity": "sha512-N3zl+lxHCifgIlcMUP5016ESkeQjLj/959RxxNYIthIg+CQHInujFuXeWbWMgnTo4cp5XVHqFPmpyu9J65C1Yg==",
-      "cpu": [
-        "arm64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "darwin"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/darwin-x64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.12.tgz",
-      "integrity": "sha512-HQ9ka4Kx21qHXwtlTUVbKJOAnmG1ipXhdWTmNXiPzPfWKpXqASVcWdnf2bnL73wgjNrFXAa3yYvBSd9pzfEIpA==",
-      "cpu": [
-        "x64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "darwin"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/freebsd-arm64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.12.tgz",
-      "integrity": "sha512-gA0Bx759+7Jve03K1S0vkOu5Lg/85dou3EseOGUes8flVOGxbhDDh/iZaoek11Y8mtyKPGF3vP8XhnkDEAmzeg==",
-      "cpu": [
-        "arm64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "freebsd"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/freebsd-x64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.12.tgz",
-      "integrity": "sha512-TGbO26Yw2xsHzxtbVFGEXBFH0FRAP7gtcPE7P5yP7wGy7cXK2oO7RyOhL5NLiqTlBh47XhmIUXuGciXEqYFfBQ==",
-      "cpu": [
-        "x64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "freebsd"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/linux-arm": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.12.tgz",
-      "integrity": "sha512-lPDGyC1JPDou8kGcywY0YILzWlhhnRjdof3UlcoqYmS9El818LLfJJc3PXXgZHrHCAKs/Z2SeZtDJr5MrkxtOw==",
-      "cpu": [
-        "arm"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "linux"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/linux-arm64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.12.tgz",
-      "integrity": "sha512-8bwX7a8FghIgrupcxb4aUmYDLp8pX06rGh5HqDT7bB+8Rdells6mHvrFHHW2JAOPZUbnjUpKTLg6ECyzvas2AQ==",
-      "cpu": [
-        "arm64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "linux"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/linux-ia32": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.12.tgz",
-      "integrity": "sha512-0y9KrdVnbMM2/vG8KfU0byhUN+EFCny9+8g202gYqSSVMonbsCfLjUO+rCci7pM0WBEtz+oK/PIwHkzxkyharA==",
-      "cpu": [
-        "ia32"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "linux"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/linux-loong64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.12.tgz",
-      "integrity": "sha512-h///Lr5a9rib/v1GGqXVGzjL4TMvVTv+s1DPoxQdz7l/AYv6LDSxdIwzxkrPW438oUXiDtwM10o9PmwS/6Z0Ng==",
-      "cpu": [
-        "loong64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "linux"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/linux-mips64el": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.12.tgz",
-      "integrity": "sha512-iyRrM1Pzy9GFMDLsXn1iHUm18nhKnNMWscjmp4+hpafcZjrr2WbT//d20xaGljXDBYHqRcl8HnxbX6uaA/eGVw==",
-      "cpu": [
-        "mips64el"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "linux"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/linux-ppc64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.12.tgz",
-      "integrity": "sha512-9meM/lRXxMi5PSUqEXRCtVjEZBGwB7P/D4yT8UG/mwIdze2aV4Vo6U5gD3+RsoHXKkHCfSxZKzmDssVlRj1QQA==",
-      "cpu": [
-        "ppc64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "linux"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/linux-riscv64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.12.tgz",
-      "integrity": "sha512-Zr7KR4hgKUpWAwb1f3o5ygT04MzqVrGEGXGLnj15YQDJErYu/BGg+wmFlIDOdJp0PmB0lLvxFIOXZgFRrdjR0w==",
-      "cpu": [
-        "riscv64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "linux"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/linux-s390x": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.12.tgz",
-      "integrity": "sha512-MsKncOcgTNvdtiISc/jZs/Zf8d0cl/t3gYWX8J9ubBnVOwlk65UIEEvgBORTiljloIWnBzLs4qhzPkJcitIzIg==",
-      "cpu": [
-        "s390x"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "linux"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/linux-x64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.12.tgz",
-      "integrity": "sha512-uqZMTLr/zR/ed4jIGnwSLkaHmPjOjJvnm6TVVitAa08SLS9Z0VM8wIRx7gWbJB5/J54YuIMInDquWyYvQLZkgw==",
-      "cpu": [
-        "x64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "linux"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/netbsd-arm64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.12.tgz",
-      "integrity": "sha512-xXwcTq4GhRM7J9A8Gv5boanHhRa/Q9KLVmcyXHCTaM4wKfIpWkdXiMog/KsnxzJ0A1+nD+zoecuzqPmCRyBGjg==",
-      "cpu": [
-        "arm64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "netbsd"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/netbsd-x64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.12.tgz",
-      "integrity": "sha512-Ld5pTlzPy3YwGec4OuHh1aCVCRvOXdH8DgRjfDy/oumVovmuSzWfnSJg+VtakB9Cm0gxNO9BzWkj6mtO1FMXkQ==",
-      "cpu": [
-        "x64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "netbsd"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/openbsd-arm64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.12.tgz",
-      "integrity": "sha512-fF96T6KsBo/pkQI950FARU9apGNTSlZGsv1jZBAlcLL1MLjLNIWPBkj5NlSz8aAzYKg+eNqknrUJ24QBybeR5A==",
-      "cpu": [
-        "arm64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "openbsd"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/openbsd-x64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.12.tgz",
-      "integrity": "sha512-MZyXUkZHjQxUvzK7rN8DJ3SRmrVrke8ZyRusHlP+kuwqTcfWLyqMOE3sScPPyeIXN/mDJIfGXvcMqCgYKekoQw==",
-      "cpu": [
-        "x64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "openbsd"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/openharmony-arm64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.25.12.tgz",
-      "integrity": "sha512-rm0YWsqUSRrjncSXGA7Zv78Nbnw4XL6/dzr20cyrQf7ZmRcsovpcRBdhD43Nuk3y7XIoW2OxMVvwuRvk9XdASg==",
-      "cpu": [
-        "arm64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "openharmony"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/sunos-x64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.12.tgz",
-      "integrity": "sha512-3wGSCDyuTHQUzt0nV7bocDy72r2lI33QL3gkDNGkod22EsYl04sMf0qLb8luNKTOmgF/eDEDP5BFNwoBKH441w==",
-      "cpu": [
-        "x64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "sunos"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/win32-arm64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.12.tgz",
-      "integrity": "sha512-rMmLrur64A7+DKlnSuwqUdRKyd3UE7oPJZmnljqEptesKM8wx9J8gx5u0+9Pq0fQQW8vqeKebwNXdfOyP+8Bsg==",
-      "cpu": [
-        "arm64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "win32"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/win32-ia32": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.12.tgz",
-      "integrity": "sha512-HkqnmmBoCbCwxUKKNPBixiWDGCpQGVsrQfJoVGYLPT41XWF8lHuE5N6WhVia2n4o5QK5M4tYr21827fNhi4byQ==",
-      "cpu": [
-        "ia32"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "win32"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@esbuild/win32-x64": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.12.tgz",
-      "integrity": "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA==",
-      "cpu": [
-        "x64"
-      ],
-      "dev": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "win32"
-      ],
-      "engines": {
-        "node": ">=18"
-      }
-    },
-    "node_modules/@hono/node-server": {
-      "version": "1.19.13",
-      "resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-1.19.13.tgz",
-      "integrity": "sha512-TsQLe4i2gvoTtrHje625ngThGBySOgSK3Xo2XRYOdqGN1teR8+I7vchQC46uLJi8OF62YTYA3AhSpumtkhsaKQ==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=18.14.1"
-      },
-      "peerDependencies": {
-        "hono": "^4"
-      }
-    },
-    "node_modules/@types/node": {
-      "version": "20.19.25",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.25.tgz",
-      "integrity": "sha512-ZsJzA5thDQMSQO788d7IocwwQbI8B5OPzmqNvpf3NY/+MHDAS759Wo0gd2WQeXYt5AAAQjzcrTVC6SKCuYgoCQ==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "undici-types": "~6.21.0"
-      }
-    },
-    "node_modules/esbuild": {
-      "version": "0.25.12",
-      "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz",
-      "integrity": "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg==",
-      "dev": true,
-      "hasInstallScript": true,
-      "license": "MIT",
-      "bin": {
-        "esbuild": "bin/esbuild"
-      },
-      "engines": {
-        "node": ">=18"
-      },
-      "optionalDependencies": {
-        "@esbuild/aix-ppc64": "0.25.12",
-        "@esbuild/android-arm": "0.25.12",
-        "@esbuild/android-arm64": "0.25.12",
-        "@esbuild/android-x64": "0.25.12",
-        "@esbuild/darwin-arm64": "0.25.12",
-        "@esbuild/darwin-x64": "0.25.12",
-        "@esbuild/freebsd-arm64": "0.25.12",
-        "@esbuild/freebsd-x64": "0.25.12",
-        "@esbuild/linux-arm": "0.25.12",
-        "@esbuild/linux-arm64": "0.25.12",
-        "@esbuild/linux-ia32": "0.25.12",
-        "@esbuild/linux-loong64": "0.25.12",
-        "@esbuild/linux-mips64el": "0.25.12",
-        "@esbuild/linux-ppc64": "0.25.12",
-        "@esbuild/linux-riscv64": "0.25.12",
-        "@esbuild/linux-s390x": "0.25.12",
-        "@esbuild/linux-x64": "0.25.12",
-        "@esbuild/netbsd-arm64": "0.25.12",
-        "@esbuild/netbsd-x64": "0.25.12",
-        "@esbuild/openbsd-arm64": "0.25.12",
-        "@esbuild/openbsd-x64": "0.25.12",
-        "@esbuild/openharmony-arm64": "0.25.12",
-        "@esbuild/sunos-x64": "0.25.12",
-        "@esbuild/win32-arm64": "0.25.12",
-        "@esbuild/win32-ia32": "0.25.12",
-        "@esbuild/win32-x64": "0.25.12"
-      }
-    },
-    "node_modules/fsevents": {
-      "version": "2.3.3",
-      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
-      "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==",
-      "dev": true,
-      "hasInstallScript": true,
-      "license": "MIT",
-      "optional": true,
-      "os": [
-        "darwin"
-      ],
-      "engines": {
-        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
-      }
-    },
-    "node_modules/get-tsconfig": {
-      "version": "4.13.0",
-      "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.13.0.tgz",
-      "integrity": "sha512-1VKTZJCwBrvbd+Wn3AOgQP/2Av+TfTCOlE4AcRJE72W1ksZXbAx8PPBR9RzgTeSPzlPMHrbANMH3LbltH73wxQ==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "resolve-pkg-maps": "^1.0.0"
-      },
-      "funding": {
-        "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1"
-      }
-    },
-    "node_modules/hono": {
-      "version": "4.12.16",
-      "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.16.tgz",
-      "integrity": "sha512-jN0ZewiNAWSe5khM3EyCmBb250+b40wWbwNILNfEvq84VREWwOIkuUsFONk/3i3nqkz7Oe1PcpM2mwQEK2L9Kg==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=16.9.0"
-      }
-    },
-    "node_modules/resolve-pkg-maps": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz",
-      "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==",
-      "dev": true,
-      "license": "MIT",
-      "funding": {
-        "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1"
-      }
-    },
-    "node_modules/tsx": {
-      "version": "4.20.6",
-      "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.20.6.tgz",
-      "integrity": "sha512-ytQKuwgmrrkDTFP4LjR0ToE2nqgy886GpvRSpU0JAnrdBYppuY5rLkRUYPU1yCryb24SsKBTL/hlDQAEFVwtZg==",
-      "dev": true,
-      "license": "MIT",
-      "dependencies": {
-        "esbuild": "~0.25.0",
-        "get-tsconfig": "^4.7.5"
-      },
-      "bin": {
-        "tsx": "dist/cli.mjs"
-      },
-      "engines": {
-        "node": ">=18.0.0"
-      },
-      "optionalDependencies": {
-        "fsevents": "~2.3.3"
-      }
-    },
-    "node_modules/undici-types": {
-      "version": "6.21.0",
-      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",
-      "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==",
-      "dev": true,
-      "license": "MIT"
-    }
-  }
-}

litellm-js/spend-logs/package.json

@@ -1,13 +0,0 @@
-{
-  "scripts": {
-    "dev": "tsx watch src/index.ts"
-  },
-  "dependencies": {
-    "@hono/node-server": "1.19.13",
-    "hono": "4.12.16"
-  },
-  "devDependencies": {
-    "@types/node": "20.19.25",
-    "tsx": "4.20.6"
-  }
-}

litellm-js/spend-logs/schema.prisma

@@ -1,29 +0,0 @@
-generator client {
-  provider        = "prisma-client-js"
-}
-
-datasource client {
-  provider = "postgresql"
-  url      = env("DATABASE_URL")
-}
-
-model LiteLLM_SpendLogs {
-  request_id        String   @id
-  call_type         String
-  api_key           String   @default("")
-  spend             Float    @default(0.0)
-  total_tokens      Int      @default(0)
-  prompt_tokens     Int      @default(0)
-  completion_tokens Int      @default(0)
-  startTime         DateTime
-  endTime           DateTime
-  model             String   @default("")
-  api_base          String   @default("")
-  user              String   @default("")
-  metadata          Json     @default("{}")
-  cache_hit         String   @default("")
-  cache_key         String   @default("")
-  request_tags      Json     @default("[]")
-  team_id           String?
-  end_user          String?
-}

litellm-js/spend-logs/src/_types.ts

@@ -1,32 +0,0 @@
-export type LiteLLM_IncrementSpend = {
-    key_transactions: Array<LiteLLM_IncrementObject>, // [{"key": spend},..]
-    user_transactions: Array<LiteLLM_IncrementObject>, 
-    team_transactions: Array<LiteLLM_IncrementObject>,
-    spend_logs_transactions: Array<LiteLLM_SpendLogs>
-}
-
-export type LiteLLM_IncrementObject = {
-    key: string,
-    spend: number
-}
-
-export type LiteLLM_SpendLogs = {
-    request_id: string; // @id means it's a unique identifier
-    call_type: string;
-    api_key: string; // @default("") means it defaults to an empty string if not provided
-    spend: number; // Float in Prisma corresponds to number in TypeScript
-    total_tokens: number; // Int in Prisma corresponds to number in TypeScript
-    prompt_tokens: number;
-    completion_tokens: number;
-    startTime: Date; // DateTime in Prisma corresponds to Date in TypeScript
-    endTime: Date;
-    model: string; // @default("") means it defaults to an empty string if not provided
-    api_base: string;
-    user: string;
-    metadata: any; // Json type in Prisma is represented by any in TypeScript; could also use a more specific type if the structure of JSON is known
-    cache_hit: string;
-    cache_key: string;
-    request_tags: any; // Similarly, this could be an array or a more specific type depending on the expected structure
-    team_id?: string | null; // ? indicates it's optional and can be undefined, but could also be null if not provided
-    end_user?: string | null;
-};

litellm-js/spend-logs/src/index.ts

@@ -1,84 +0,0 @@
-import { serve } from '@hono/node-server'
-import { Hono } from 'hono'
-import { PrismaClient } from '@prisma/client'
-import {LiteLLM_SpendLogs, LiteLLM_IncrementSpend, LiteLLM_IncrementObject} from './_types'
-
-const app = new Hono()
-const prisma = new PrismaClient()
-// In-memory storage for logs
-let spend_logs: LiteLLM_SpendLogs[] = [];
-const key_logs: LiteLLM_IncrementObject[] = [];
-const user_logs: LiteLLM_IncrementObject[] = [];
-const transaction_logs: LiteLLM_IncrementObject[] = [];
-
-
-app.get('/', (c) => {
-  return c.text('Hello Hono!')
-})
-
-const MIN_LOGS = 1; // Minimum number of logs needed to initiate a flush
-const FLUSH_INTERVAL = 5000; // Time in ms to wait before trying to flush again
-const BATCH_SIZE = 100; // Preferred size of each batch to write to the database
-const MAX_LOGS_PER_INTERVAL = 1000; // Maximum number of logs to flush in a single interval
-
-const flushLogsToDb = async () => {
-  if (spend_logs.length >= MIN_LOGS) {
-    // Limit the logs to process in this interval to MAX_LOGS_PER_INTERVAL or less
-    const logsToProcess = spend_logs.slice(0, MAX_LOGS_PER_INTERVAL);
-  
-    for (let i = 0; i < logsToProcess.length; i += BATCH_SIZE) {
-      // Create subarray for current batch, ensuring it doesn't exceed the BATCH_SIZE
-      const batch = logsToProcess.slice(i, i + BATCH_SIZE);
-
-      // Convert datetime strings to Date objects
-      const batchWithDates = batch.map(entry => ({
-        ...entry,
-        startTime: new Date(entry.startTime),
-        endTime: new Date(entry.endTime),
-        // Repeat for any other DateTime fields you may have
-      }));
-
-      await prisma.liteLLM_SpendLogs.createMany({
-        data: batchWithDates,
-      });
-
-      console.log(`Flushed ${batch.length} logs to the DB.`);
-    }
-
-    // Remove the processed logs from spend_logs
-    spend_logs = spend_logs.slice(logsToProcess.length);
-    
-    console.log(`${logsToProcess.length} logs processed. Remaining in queue: ${spend_logs.length}`);
-  } else {
-    // This will ensure it doesn't falsely claim "No logs to flush." when it's merely below the MIN_LOGS threshold.
-    if(spend_logs.length > 0) {
-      console.log(`Accumulating logs. Currently at ${spend_logs.length}, waiting for at least ${MIN_LOGS}.`);
-    } else {
-      console.log("No logs to flush.");
-    }
-  }
-};
-
-// Setup interval for attempting to flush the logs
-setInterval(flushLogsToDb, FLUSH_INTERVAL);
-
-// Route to receive log messages
-app.post('/spend/update', async (c) => {
-  const incomingLogs = await c.req.json<LiteLLM_SpendLogs[]>();
-  
-  spend_logs.push(...incomingLogs);
-
-  console.log(`Received and stored ${incomingLogs.length} logs. Total logs in memory: ${spend_logs.length}`);
-  
-  return c.json({ message: `Successfully stored ${incomingLogs.length} logs` });
-});
-
-
-
-const port = 3000
-console.log(`Server is running on port ${port}`)
-
-serve({
-  fetch: app.fetch,
-  port
-})

litellm-js/spend-logs/tsconfig.json

@@ -1,13 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ESNext",
-    "module": "ESNext",
-    "moduleResolution": "Bundler",
-    "strict": true,
-    "types": [
-      "node"
-    ],
-    "jsx": "react-jsx",
-    "jsxImportSource": "hono/jsx",
-  }
-}

litellm/__init__.py

@@ -206,6 +206,7 @@ add_user_information_to_llm_headers: Optional[bool] = (
 )
 store_audit_logs = False  # Enterprise feature, allow users to see audit logs
 skip_system_message_in_guardrail: bool = False
+skip_tool_message_in_guardrail: bool = False
 ### end of callbacks #############
 
 email: Optional[str] = (
@@ -388,6 +389,7 @@ anthropic_beta_headers_url: str = os.getenv(
 suppress_debug_info = False
 dynamodb_table_name: Optional[str] = None
 s3_callback_params: Optional[Dict] = None
+s3_audit_callback_params: Optional[Dict] = None
 datadog_llm_observability_params: Optional[Union[DatadogLLMObsInitParams, Dict]] = None
 datadog_params: Optional[Union[DatadogInitParams, Dict]] = None
 aws_sqs_callback_params: Optional[Dict] = None
@@ -414,6 +416,9 @@ custom_prometheus_metadata_labels: List[str] = []
 custom_prometheus_tags: List[str] = []
 prometheus_metrics_config: Optional[List] = None
 prometheus_emit_stream_label: bool = False
+prometheus_end_user_metrics_max_series_per_metric: Optional[int] = 10000
+prometheus_end_user_metrics_ttl_seconds: Optional[float] = 3600.0
+prometheus_end_user_metrics_cleanup_interval_seconds: Optional[float] = 60.0
 disable_add_prefix_to_prompt: bool = (
     False  # used by anthropic, to disable adding prefix to prompt
 )

litellm/_redis.py

@@ -19,6 +19,7 @@ import redis.asyncio as async_redis  # type: ignore
 
 from litellm import get_secret, get_secret_str
 from litellm._redis_credential_provider import (
+    AzureADCredentialProvider,
     GCPIAMCredentialProvider,
     _generate_gcp_iam_access_token,
 )
@@ -27,6 +28,8 @@ from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
 
 from ._logging import verbose_logger
 
+AZURE_REDIS_SCOPE = "https://redis.azure.com/.default"
+
 
 def _get_redis_kwargs():
     arg_spec = inspect.getfullargspec(redis.Redis)
@@ -43,6 +46,10 @@ def _get_redis_kwargs():
         "redis_connect_func",
         "gcp_service_account",
         "gcp_ssl_ca_certs",
+        "azure_redis_ad_token",
+        "azure_client_id",
+        "azure_tenant_id",
+        "azure_client_secret",
     ]
 
     available_args = [x for x in arg_spec.args if x not in exclude_args] + include_args
@@ -89,6 +96,10 @@ def _get_redis_cluster_kwargs(client=None):
     )  # Needed for sync clusters and IAM detection
     available_args.append("gcp_service_account")
     available_args.append("gcp_ssl_ca_certs")
+    available_args.append("azure_redis_ad_token")
+    available_args.append("azure_client_id")
+    available_args.append("azure_tenant_id")
+    available_args.append("azure_client_secret")
     available_args.append("max_connections")
 
     return available_args
@@ -155,6 +166,125 @@ def create_gcp_iam_redis_connect_func(
     return iam_connect
 
 
+def _build_azure_credential(
+    azure_client_id: Optional[str] = None,
+    azure_tenant_id: Optional[str] = None,
+    azure_client_secret: Optional[str] = None,
+):
+    """
+    Build a long-lived Azure credential object.
+
+    Azure SDK credentials cache tokens internally and handle expiry/refresh
+    transparently, so this should be called once and the result reused.
+    """
+    try:
+        from azure.identity import (
+            ClientSecretCredential,
+            DefaultAzureCredential,
+            ManagedIdentityCredential,
+        )
+    except ImportError:
+        raise ImportError(
+            "azure-identity is required for Azure AD Redis authentication. "
+            "Install it with: pip install azure-identity"
+        )
+
+    _client_id = azure_client_id or os.environ.get("AZURE_CLIENT_ID")
+    _tenant_id = azure_tenant_id or os.environ.get("AZURE_TENANT_ID")
+    _client_secret = azure_client_secret or os.environ.get("AZURE_CLIENT_SECRET")
+
+    if _client_id and _tenant_id and _client_secret:
+        return ClientSecretCredential(
+            client_id=_client_id,
+            tenant_id=_tenant_id,
+            client_secret=_client_secret,
+        )
+    elif _client_id:
+        return ManagedIdentityCredential(client_id=_client_id)
+    else:
+        return DefaultAzureCredential()
+
+
+def _generate_azure_ad_redis_token(
+    azure_client_id: Optional[str] = None,
+    azure_tenant_id: Optional[str] = None,
+    azure_client_secret: Optional[str] = None,
+) -> str:
+    """
+    One-shot helper that builds a credential and fetches a single Azure AD
+    access token for Redis. Each call rebuilds the credential and performs a
+    network round-trip, so it should not be used in steady-state Redis flows
+    — the sync (``create_azure_ad_redis_connect_func``) and async paths
+    (``AzureADCredentialProvider``) keep the credential alive across
+    connections so the Azure SDK's internal cache + silent refresh apply.
+    """
+    credential = _build_azure_credential(
+        azure_client_id=azure_client_id,
+        azure_tenant_id=azure_tenant_id,
+        azure_client_secret=azure_client_secret,
+    )
+    token = credential.get_token(AZURE_REDIS_SCOPE)
+    return token.token
+
+
+def create_azure_ad_redis_connect_func(
+    azure_client_id: Optional[str] = None,
+    azure_tenant_id: Optional[str] = None,
+    azure_client_secret: Optional[str] = None,
+) -> Callable:
+    """
+    Creates a custom Redis connection function for Azure AD authentication.
+
+    Used for sync Redis clients. The credential is created once (captured by the
+    closure) and reused across connections — the Azure SDK handles token caching
+    and silent renewal internally. Only ``get_token`` is called per connection.
+    """
+    credential = _build_azure_credential(
+        azure_client_id=azure_client_id,
+        azure_tenant_id=azure_tenant_id,
+        azure_client_secret=azure_client_secret,
+    )
+
+    def ad_connect(self):
+        """Initialize the connection and authenticate using Azure AD"""
+        from redis.exceptions import (
+            AuthenticationError,
+            AuthenticationWrongNumberOfArgsError,
+        )
+        from redis.utils import str_if_bytes
+
+        self._parser.on_connect(self)
+
+        access_token = credential.get_token(AZURE_REDIS_SCOPE).token
+
+        # Only include username when explicitly set — sending AUTH "" <token>
+        # is invalid for most ACL-configured Azure Redis instances.
+        username = os.environ.get("REDIS_USERNAME", "")
+        if username:
+            auth_args = (username, access_token)
+        else:
+            auth_args = (access_token,)
+
+        self.send_command("AUTH", *auth_args, check_health=False)
+
+        try:
+            auth_response = self.read_response()
+        except AuthenticationWrongNumberOfArgsError:
+            # Fallback: try with just the token (Redis < 6 / no ACL)
+            self.send_command("AUTH", access_token, check_health=False)
+            auth_response = self.read_response()
+
+        if str_if_bytes(auth_response) != "OK":
+            raise AuthenticationError("Azure AD authentication failed for Redis")
+
+    # Attach the live credential object so async paths can wrap it in
+    # AzureADCredentialProvider for refresh-aware token retrieval. The raw
+    # client_id/tenant_id/secret are intentionally NOT exposed here — the
+    # credential closure already holds them.
+    ad_connect._azure_credential = credential  # type: ignore[attr-defined]
+    return ad_connect
+
+
 def get_redis_url_from_environment():
     if "REDIS_URL" in os.environ:
         return os.environ["REDIS_URL"]
@@ -179,7 +309,7 @@ def get_redis_url_from_environment():
     return f"{redis_protocol}://{auth_part}{os.environ['REDIS_HOST']}:{os.environ['REDIS_PORT']}"
 
 
-def _get_redis_client_logic(**env_overrides):
+def _get_redis_client_logic(**env_overrides):  # noqa: PLR0915
     """
     Common functionality across sync + async redis client implementations
     """
@@ -253,6 +383,52 @@ def _get_redis_client_logic(**env_overrides):
         if _gcp_ssl_ca_certs and redis_kwargs.get("ssl", False):
             redis_kwargs["ssl_ca_certs"] = _gcp_ssl_ca_certs
 
+    # Handle Azure AD authentication (after GCP IAM block)
+    _azure_redis_ad_token = redis_kwargs.get("azure_redis_ad_token") or get_secret(
+        "REDIS_AZURE_AD_TOKEN"
+    )
+
+    _azure_ad_enabled = (
+        _azure_redis_ad_token is not None
+        and str(_azure_redis_ad_token).lower() == "true"
+    )
+
+    if _azure_ad_enabled and _gcp_service_account is not None:
+        verbose_logger.warning(
+            "Both GCP IAM (gcp_service_account) and Azure AD (azure_redis_ad_token) are configured for Redis. "
+            "Using GCP IAM. Remove one to avoid misconfiguration."
+        )
+
+    if _azure_ad_enabled and _gcp_service_account is None:
+        _azure_client_id = redis_kwargs.get("azure_client_id") or get_secret_str(
+            "AZURE_CLIENT_ID"
+        )
+        _azure_tenant_id = redis_kwargs.get("azure_tenant_id") or get_secret_str(
+            "AZURE_TENANT_ID"
+        )
+        _azure_client_secret = redis_kwargs.get(
+            "azure_client_secret"
+        ) or get_secret_str("AZURE_CLIENT_SECRET")
+
+        verbose_logger.debug("Setting up Azure AD authentication for Redis.")
+        redis_kwargs["redis_connect_func"] = create_azure_ad_redis_connect_func(
+            azure_client_id=_azure_client_id,
+            azure_tenant_id=_azure_tenant_id,
+            azure_client_secret=_azure_client_secret,
+        )
+        # Marker for async paths to detect Azure AD auth. The live credential
+        # object is attached separately as `_azure_credential` by
+        # `create_azure_ad_redis_connect_func`; the raw client_id/tenant_id/secret
+        # are intentionally NOT exposed on the function to avoid leaking
+        # credentials via inspection or logging.
+        redis_kwargs["redis_connect_func"]._azure_redis_ad_token = True  # type: ignore[attr-defined]
+
+    # Always remove Azure-specific kwargs that shouldn't be passed to Redis client
+    redis_kwargs.pop("azure_redis_ad_token", None)
+    redis_kwargs.pop("azure_client_id", None)
+    redis_kwargs.pop("azure_tenant_id", None)
+    redis_kwargs.pop("azure_client_secret", None)
+
     if "url" in redis_kwargs and redis_kwargs["url"] is not None:
         # Only strip host/port/db/password when not routing to a cluster.
         # When startup_nodes is also present the cluster path takes priority and
@@ -373,7 +549,7 @@ def get_redis_client(**env_overrides):
     return redis.Redis(**redis_kwargs)
 
 
-def get_redis_async_client(
+def get_redis_async_client(  # noqa: PLR0915
     connection_pool: Optional[async_redis.BlockingConnectionPool] = None,
     **env_overrides,
 ) -> Union[async_redis.Redis, async_redis.RedisCluster]:
@@ -398,6 +574,14 @@ def get_redis_async_client(
             cluster_kwargs["credential_provider"] = GCPIAMCredentialProvider(
                 redis_connect_func._gcp_service_account
             )
+        # Handle Azure AD authentication for async clusters via CredentialProvider
+        # so the credential's internal cache + silent refresh runs per connection
+        # (mirrors GCP IAM above; avoids static-token-baked-in-pool expiry).
+        elif redis_connect_func and hasattr(redis_connect_func, "_azure_credential"):
+            cluster_kwargs["credential_provider"] = AzureADCredentialProvider(
+                redis_connect_func._azure_credential,
+                username=os.environ.get("REDIS_USERNAME") or None,
+            )
 
         new_startup_nodes: List[ClusterNode] = []
 
@@ -431,6 +615,22 @@ def get_redis_async_client(
     # Check for Redis Sentinel
     if "sentinel_nodes" in redis_kwargs and "service_name" in redis_kwargs:
         return _init_async_redis_sentinel(redis_kwargs)
+
+    # Wrap GCP / Azure AD auth in a CredentialProvider for the standard async
+    # Redis client. The async client doesn't support redis_connect_func, but it
+    # does honour credential_provider — which is called per connection, so the
+    # underlying SDK can refresh tokens silently before they expire.
+    redis_connect_func = redis_kwargs.pop("redis_connect_func", None)
+    if redis_connect_func and hasattr(redis_connect_func, "_azure_credential"):
+        redis_kwargs["credential_provider"] = AzureADCredentialProvider(
+            redis_connect_func._azure_credential,
+            username=os.environ.get("REDIS_USERNAME") or None,
+        )
+    elif redis_connect_func and hasattr(redis_connect_func, "_gcp_service_account"):
+        redis_kwargs["credential_provider"] = GCPIAMCredentialProvider(
+            redis_connect_func._gcp_service_account
+        )
+
     _pretty_print_redis_config(redis_kwargs=redis_kwargs)
 
     if connection_pool is not None:
@@ -464,6 +664,21 @@ def get_redis_connection_pool(
                     redis_kwargs["max_connections"],
                 )
         return async_redis.BlockingConnectionPool.from_url(**pool_kwargs)
+
+    # Wrap GCP / Azure AD auth in a CredentialProvider so pool-managed
+    # connections re-fetch tokens via the SDK's internal cache + silent refresh
+    # rather than reusing a single token captured at pool creation.
+    redis_connect_func = redis_kwargs.pop("redis_connect_func", None)
+    if redis_connect_func and hasattr(redis_connect_func, "_azure_credential"):
+        redis_kwargs["credential_provider"] = AzureADCredentialProvider(
+            redis_connect_func._azure_credential,
+            username=os.environ.get("REDIS_USERNAME") or None,
+        )
+    elif redis_connect_func and hasattr(redis_connect_func, "_gcp_service_account"):
+        redis_kwargs["credential_provider"] = GCPIAMCredentialProvider(
+            redis_connect_func._gcp_service_account
+        )
+
     connection_class = async_redis.Connection
     if "ssl" in redis_kwargs:
         connection_class = async_redis.SSLConnection

litellm/_redis_credential_provider.py

@@ -1,10 +1,13 @@
 import asyncio
 import threading
 import time
-from typing import Dict, Tuple
+from typing import Any, Dict, Optional, Tuple, Union
 
 from redis.credentials import CredentialProvider  # type: ignore[attr-defined]
 
+# Azure AD scope for Redis Cache for Azure.
+AZURE_REDIS_SCOPE = "https://redis.azure.com/.default"
+
 # GCP IAM tokens are valid for 1 hour. Cache for 55 minutes to refresh before expiry.
 _GCP_IAM_TOKEN_TTL_SECONDS = 3300
 
@@ -101,3 +104,33 @@ class GCPIAMCredentialProvider(CredentialProvider):
             _get_cached_gcp_iam_token, self._gcp_service_account
         )
         return (token,)
+
+
+class AzureADCredentialProvider(CredentialProvider):
+    """
+    redis.credentials.CredentialProvider implementation that supplies Azure AD
+    tokens for Redis authentication.
+
+    Wraps an azure-identity credential object so the Azure SDK's internal token
+    cache and silent refresh are honoured on every Redis connection. This avoids
+    the static-token-baked-in-pool issue where pool-managed connections would
+    fail authentication after the initial token expired (~1 hour TTL).
+    """
+
+    def __init__(self, credential: Any, username: Optional[str] = None) -> None:
+        self._credential = credential
+        self._username = username
+
+    def get_credentials(self) -> Union[Tuple[str], Tuple[str, str]]:
+        token = self._credential.get_token(AZURE_REDIS_SCOPE).token
+        if self._username:
+            return (self._username, token)
+        return (token,)
+
+    async def get_credentials_async(self) -> Union[Tuple[str], Tuple[str, str]]:
+        token_obj = await asyncio.to_thread(
+            self._credential.get_token, AZURE_REDIS_SCOPE
+        )
+        if self._username:
+            return (self._username, token_obj.token)
+        return (token_obj.token,)

litellm/constants.py

@@ -161,6 +161,11 @@ MCP_STDIO_ALLOWED_COMMANDS: frozenset = frozenset(
     | (set(_MCP_STDIO_EXTRA_COMMANDS.split(",")) - {""})
 )
 
+# MCP OAuth2 Token Exchange (OBO) Defaults
+MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE = int(
+    os.getenv("MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE", "500")
+)
+
 LITELLM_UI_ALLOW_HEADERS = [
     "x-litellm-semantic-filter",
     "x-litellm-semantic-filter-tools",
@@ -1457,6 +1462,12 @@ KEY_ROTATION_JOB_NAME = "litellm_key_rotation_job"
 EXPIRED_UI_SESSION_KEY_CLEANUP_JOB_NAME = "litellm_expired_ui_session_key_cleanup_job"
 SPEND_LOG_RUN_LOOPS = int(os.getenv("SPEND_LOG_RUN_LOOPS", 500))
 SPEND_LOG_CLEANUP_BATCH_SIZE = int(os.getenv("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000))
+SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES = int(
+    os.getenv("SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3)
+)
+SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS = float(
+    os.getenv("SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.5)
+)
 SPEND_LOG_QUEUE_SIZE_THRESHOLD = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100))
 SPEND_LOG_QUEUE_POLL_INTERVAL = float(os.getenv("SPEND_LOG_QUEUE_POLL_INTERVAL", 2.0))
 SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE = int(

litellm/experimental_mcp_client/client.py

@@ -366,6 +366,8 @@ class MCPClient:
                     headers["Authorization"] = f"Bearer {self._mcp_auth_value}"
                 elif self.auth_type == MCPAuth.token:
                     headers["Authorization"] = f"token {self._mcp_auth_value}"
+                elif self.auth_type == MCPAuth.oauth2_token_exchange:
+                    headers["Authorization"] = f"Bearer {self._mcp_auth_value}"
             elif isinstance(self._mcp_auth_value, dict):
                 headers.update(self._mcp_auth_value)
         # Note: aws_sigv4 auth is not handled here — SigV4 requires per-request

litellm/integrations/azure_sentinel/azure_sentinel.py

@@ -14,16 +14,18 @@ For batching specific details see CustomBatchLogger class
 
 import asyncio
 import os
+import time
 import traceback
-from typing import List, Optional
+from typing import List, Optional, Union
 
 from litellm._logging import verbose_logger
 from litellm.integrations.custom_batch_logger import CustomBatchLogger
+from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.llms.custom_httpx.http_handler import (
     get_async_httpx_client,
     httpxSpecialProvider,
 )
-from litellm.types.utils import StandardLoggingPayload
+from litellm.types.utils import StandardAuditLogPayload, StandardLoggingPayload
 
 
 class AzureSentinelLogger(CustomBatchLogger):
@@ -39,6 +41,7 @@ class AzureSentinelLogger(CustomBatchLogger):
         tenant_id: Optional[str] = None,
         client_id: Optional[str] = None,
         client_secret: Optional[str] = None,
+        audit_stream_name: Optional[str] = None,
         **kwargs,
     ):
         """
@@ -57,57 +60,77 @@ class AzureSentinelLogger(CustomBatchLogger):
                 If not provided, will use AZURE_SENTINEL_CLIENT_ID or AZURE_CLIENT_ID env var.
             client_secret (str, optional): Azure Client Secret for OAuth2 authentication.
                 If not provided, will use AZURE_SENTINEL_CLIENT_SECRET or AZURE_CLIENT_SECRET env var.
+            audit_stream_name (str, optional): Stream name from DCR for audit logs.
+                If not provided, audit logs use the standard stream name.
         """
         self.async_httpx_client = get_async_httpx_client(
             llm_provider=httpxSpecialProvider.LoggingCallback
         )
 
-        self.dcr_immutable_id = dcr_immutable_id or os.getenv(
+        resolved_dcr_immutable_id = dcr_immutable_id or os.getenv(
             "AZURE_SENTINEL_DCR_IMMUTABLE_ID"
         )
-        self.stream_name = stream_name or os.getenv(
-            "AZURE_SENTINEL_STREAM_NAME", "Custom-LiteLLM"
+        resolved_stream_name = (
+            stream_name or os.getenv("AZURE_SENTINEL_STREAM_NAME") or "Custom-LiteLLM"
         )
-        self.endpoint = endpoint or os.getenv("AZURE_SENTINEL_ENDPOINT")
-        self.tenant_id = (
+        resolved_audit_stream_name = audit_stream_name or resolved_stream_name
+        resolved_endpoint = endpoint or os.getenv("AZURE_SENTINEL_ENDPOINT")
+        resolved_tenant_id = (
             tenant_id
             or os.getenv("AZURE_SENTINEL_TENANT_ID")
             or os.getenv("AZURE_TENANT_ID")
         )
-        self.client_id = (
+        resolved_client_id = (
             client_id
             or os.getenv("AZURE_SENTINEL_CLIENT_ID")
             or os.getenv("AZURE_CLIENT_ID")
         )
-        self.client_secret = (
+        resolved_client_secret = (
             client_secret
             or os.getenv("AZURE_SENTINEL_CLIENT_SECRET")
             or os.getenv("AZURE_CLIENT_SECRET")
         )
 
-        if not self.dcr_immutable_id:
+        if not resolved_dcr_immutable_id:
             raise ValueError(
                 "AZURE_SENTINEL_DCR_IMMUTABLE_ID is required. Set it as an environment variable or pass dcr_immutable_id parameter."
             )
-        if not self.endpoint:
+        if not resolved_endpoint:
             raise ValueError(
                 "AZURE_SENTINEL_ENDPOINT is required. Set it as an environment variable or pass endpoint parameter."
             )
-        if not self.tenant_id:
+        if not resolved_tenant_id:
             raise ValueError(
                 "AZURE_SENTINEL_TENANT_ID or AZURE_TENANT_ID is required. Set it as an environment variable or pass tenant_id parameter."
             )
-        if not self.client_id:
+        if not resolved_client_id:
             raise ValueError(
                 "AZURE_SENTINEL_CLIENT_ID or AZURE_CLIENT_ID is required. Set it as an environment variable or pass client_id parameter."
             )
-        if not self.client_secret:
+        if not resolved_client_secret:
             raise ValueError(
                 "AZURE_SENTINEL_CLIENT_SECRET or AZURE_CLIENT_SECRET is required. Set it as an environment variable or pass client_secret parameter."
             )
 
+        self.dcr_immutable_id = resolved_dcr_immutable_id
+        self.stream_name = resolved_stream_name
+        self.audit_stream_name = resolved_audit_stream_name
+        self.endpoint = resolved_endpoint
+        self.tenant_id = resolved_tenant_id
+        self.client_id = resolved_client_id
+        self.client_secret = resolved_client_secret
+
         # Build API endpoint: {Endpoint}/dataCollectionRules/{DCR Immutable ID}/streams/{Stream Name}?api-version=2023-01-01
-        self.api_endpoint = f"{self.endpoint.rstrip('/')}/dataCollectionRules/{self.dcr_immutable_id}/streams/{self.stream_name}?api-version=2023-01-01"
+        self.api_endpoint = self._build_api_endpoint(
+            endpoint=resolved_endpoint,
+            dcr_immutable_id=resolved_dcr_immutable_id,
+            stream_name=resolved_stream_name,
+        )
+        self.audit_api_endpoint = self._build_api_endpoint(
+            endpoint=resolved_endpoint,
+            dcr_immutable_id=resolved_dcr_immutable_id,
+            stream_name=resolved_audit_stream_name,
+        )
 
         # OAuth2 scope for Azure Monitor
         self.oauth_scope = "https://monitor.azure.com/.default"
@@ -118,6 +141,13 @@ class AzureSentinelLogger(CustomBatchLogger):
         super().__init__(**kwargs, flush_lock=self.flush_lock)
         asyncio.create_task(self.periodic_flush())
         self.log_queue: List[StandardLoggingPayload] = []
+        self.audit_log_queue: List[StandardAuditLogPayload] = []
+
+    @staticmethod
+    def _build_api_endpoint(
+        endpoint: str, dcr_immutable_id: str, stream_name: str
+    ) -> str:
+        return f"{endpoint.rstrip('/')}/dataCollectionRules/{dcr_immutable_id}/streams/{stream_name}?api-version=2023-01-01"
 
     async def _get_oauth_token(self) -> str:
         """
@@ -126,9 +156,6 @@ class AzureSentinelLogger(CustomBatchLogger):
         Returns:
             Bearer token string
         """
-        # Check if we have a valid cached token
-        import time
-
         if (
             self.oauth_token
             and self.oauth_token_expires_at
@@ -170,9 +197,6 @@ class AzureSentinelLogger(CustomBatchLogger):
         if not self.oauth_token:
             raise Exception("OAuth2 token response did not contain access_token")
 
-        # Cache token expiry time
-        import time
-
         self.oauth_token_expires_at = time.time() + expires_in
 
         return self.oauth_token
@@ -246,6 +270,34 @@ class AzureSentinelLogger(CustomBatchLogger):
             )
             pass
 
+    async def async_log_audit_log_event(
+        self, audit_log: StandardAuditLogPayload
+    ) -> None:
+        """
+        Async log LiteLLM audit log events to Azure Sentinel.
+
+        Audit logs are queued separately from standard LLM logs so mixed callback
+        usage never sends schema-mismatched records in the same ingestion batch.
+        """
+        try:
+            verbose_logger.debug(
+                "Azure Sentinel: Logging audit event id=%s action=%s table=%s",
+                audit_log.get("id"),
+                audit_log.get("action"),
+                audit_log.get("table_name"),
+            )
+
+            self.audit_log_queue.append(audit_log)
+
+            if len(self.audit_log_queue) >= self.batch_size:
+                await self.async_send_audit_batch()
+
+        except Exception as e:
+            verbose_logger.exception(
+                f"Azure Sentinel Audit Log Layer Error - {str(e)}\n{traceback.format_exc()}"
+            )
+            pass
+
     async def async_send_batch(self):
         """
         Sends the batch of logs to Azure Monitor Logs Ingestion API
@@ -253,22 +305,42 @@ class AzureSentinelLogger(CustomBatchLogger):
         Raises:
             Raises a NON Blocking verbose_logger.exception if an error occurs
         """
+        await self._async_send_batch_to_api(
+            log_queue=self.log_queue,
+            api_endpoint=self.api_endpoint,
+            log_type="logs",
+        )
+
+    async def async_send_audit_batch(self):
+        """
+        Sends the batch of audit logs to Azure Monitor Logs Ingestion API
+        """
+        await self._async_send_batch_to_api(
+            log_queue=self.audit_log_queue,
+            api_endpoint=self.audit_api_endpoint,
+            log_type="audit logs",
+        )
+
+    async def _async_send_batch_to_api(
+        self,
+        log_queue: List[Union[StandardLoggingPayload, StandardAuditLogPayload]],
+        api_endpoint: str,
+        log_type: str,
+    ) -> None:
         try:
-            if not self.log_queue:
+            if not log_queue:
                 return
 
             verbose_logger.debug(
-                "Azure Sentinel - about to flush %s events", len(self.log_queue)
+                "Azure Sentinel - about to flush %s %s", len(log_queue), log_type
             )
 
-            from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
-
             # Get OAuth2 token
             bearer_token = await self._get_oauth_token()
 
             # Convert log queue to JSON array format expected by Logs Ingestion API
             # Each log entry should be a JSON object in the array
-            body = safe_dumps(self.log_queue)
+            body = safe_dumps(log_queue)
 
             # Set headers for Logs Ingestion API
             headers = {
@@ -278,7 +350,7 @@ class AzureSentinelLogger(CustomBatchLogger):
 
             # Send the request
             response = await self.async_httpx_client.post(
-                url=self.api_endpoint, data=body.encode("utf-8"), headers=headers
+                url=api_endpoint, data=body.encode("utf-8"), headers=headers
             )
 
             if response.status_code not in [200, 204]:
@@ -301,4 +373,15 @@ class AzureSentinelLogger(CustomBatchLogger):
                 f"Azure Sentinel Error sending batch API - {str(e)}\n{traceback.format_exc()}"
             )
         finally:
-            self.log_queue.clear()
+            log_queue.clear()
+
+    async def flush_queue(self):
+        if self.flush_lock is None:
+            return
+
+        async with self.flush_lock:
+            if self.log_queue:
+                await self.async_send_batch()
+            if self.audit_log_queue:
+                await self.async_send_audit_batch()
+            self.last_flush_time = time.time()

litellm/integrations/opentelemetry.py

@@ -57,6 +57,17 @@ LITELLM_PROXY_REQUEST_SPAN_NAME = "Received Proxy Server Request"
 RAW_REQUEST_SPAN_NAME = "raw_gen_ai_request"
 LITELLM_REQUEST_SPAN_NAME = "litellm_request"
 
+CAPTURE_MODE_NO_CONTENT = "NO_CONTENT"
+CAPTURE_MODE_SPAN_ONLY = "SPAN_ONLY"
+CAPTURE_MODE_EVENT_ONLY = "EVENT_ONLY"
+CAPTURE_MODE_SPAN_AND_EVENT = "SPAN_AND_EVENT"
+_VALID_CAPTURE_MODES = {
+    CAPTURE_MODE_NO_CONTENT,
+    CAPTURE_MODE_SPAN_ONLY,
+    CAPTURE_MODE_EVENT_ONLY,
+    CAPTURE_MODE_SPAN_AND_EVENT,
+}
+
 
 @dataclass
 class OpenTelemetryConfig:
@@ -71,6 +82,9 @@ class OpenTelemetryConfig:
     ignore_context_propagation: Optional[bool] = None
     # When True, create a private TracerProvider instead of reusing or setting the global one.
     skip_set_global: bool = False
+    # Programmatic override for OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT.
+    # One of NO_CONTENT, SPAN_ONLY, EVENT_ONLY, SPAN_AND_EVENT (or "true" as legacy alias).
+    capture_message_content: Optional[str] = None
 
     def __post_init__(self) -> None:
         # If endpoint is specified but exporter is still the default "console",
@@ -182,6 +196,9 @@ class OpenTelemetry(CustomLogger):
         super().__init__(**kwargs)
         self._init_metrics(meter_provider)
         self._init_logs(logger_provider)
+        # Sample env-var / config / message_logging at init so subsequent
+        # _capture_in_span / _capture_in_event calls are deterministic.
+        self._capture_mode_cached = self._compute_capture_mode_from_init_state()
         self._init_otel_logger_on_litellm_proxy()
 
     @staticmethod
@@ -306,6 +323,62 @@ class OpenTelemetry(CustomLogger):
             hasattr(self, "callback_name") and self.callback_name == "langfuse_otel"
         )
 
+    def _compute_capture_mode_from_init_state(self) -> Optional[str]:
+        """Sample explicit settings at init. Returns the resolved mode or
+        None if nothing explicit is set (in which case the legacy
+        ``self.message_logging`` flag is consulted dynamically per request).
+
+        ``"true"``/``"1"`` map to ``EVENT_ONLY`` per the contrib convention.
+        ``"false"``/``"0"`` map to ``NO_CONTENT``.
+        Unknown values are ignored.
+        """
+        explicit = self.config.capture_message_content or os.getenv(
+            "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"
+        )
+        if not explicit:
+            return None
+        normalized = explicit.upper()
+        if normalized in ("TRUE", "1"):
+            return CAPTURE_MODE_EVENT_ONLY
+        if normalized in ("FALSE", "0"):
+            return CAPTURE_MODE_NO_CONTENT
+        if normalized in _VALID_CAPTURE_MODES:
+            return normalized
+        return None
+
+    def _resolve_capture_mode(self) -> str:
+        """Return the active capture mode for this request.
+
+        Precedence:
+          1. ``litellm.turn_off_message_logging=True`` forces ``NO_CONTENT``
+             (kill-switch checked dynamically).
+          2. Explicit setting sampled at init from
+             ``OpenTelemetryConfig.capture_message_content`` or
+             ``OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT``.
+          3. Legacy ``self.message_logging`` (checked dynamically).
+        """
+        if litellm.turn_off_message_logging:
+            return CAPTURE_MODE_NO_CONTENT
+        if self._capture_mode_cached is not None:
+            return self._capture_mode_cached
+        return (
+            CAPTURE_MODE_SPAN_AND_EVENT
+            if self.message_logging
+            else CAPTURE_MODE_NO_CONTENT
+        )
+
+    def _capture_in_span(self) -> bool:
+        return self._resolve_capture_mode() in (
+            CAPTURE_MODE_SPAN_ONLY,
+            CAPTURE_MODE_SPAN_AND_EVENT,
+        )
+
+    def _capture_in_event(self) -> bool:
+        return self._resolve_capture_mode() in (
+            CAPTURE_MODE_EVENT_ONLY,
+            CAPTURE_MODE_SPAN_AND_EVENT,
+        )
+
     def _init_tracing(self, tracer_provider):
         from opentelemetry import trace
         from opentelemetry.sdk.trace import TracerProvider
@@ -825,8 +898,7 @@ class OpenTelemetry(CustomLogger):
         from opentelemetry import trace
         from opentelemetry.trace import Status, StatusCode
 
-        # only log raw LLM request/response if message_logging is on and not globally turned off
-        if litellm.turn_off_message_logging or not self.message_logging:
+        if not self._capture_in_span():
             return
 
         litellm_params = kwargs.get("litellm_params", {})
@@ -1117,9 +1189,14 @@ class OpenTelemetry(CustomLogger):
             }
             if role == "tool" and msg.get("id"):
                 attrs["id"] = msg["id"]
-            if self.message_logging and msg.get("content"):
+            capture_event_content = self._capture_in_event()
+            if capture_event_content and msg.get("content"):
                 attrs["gen_ai.prompt"] = msg["content"]
 
+            body = msg.copy()
+            if not capture_event_content:
+                body.pop("content", None)
+
             log_record = SdkLogRecord(
                 timestamp=self._to_ns(datetime.now()),
                 trace_id=parent_ctx.trace_id,
@@ -1127,7 +1204,7 @@ class OpenTelemetry(CustomLogger):
                 trace_flags=parent_ctx.trace_flags,
                 severity_number=SeverityNumber.INFO,
                 severity_text="INFO",
-                body=msg.copy(),
+                body=body,
                 attributes=attrs,
             )
             otel_logger.emit(log_record)
@@ -1141,14 +1218,15 @@ class OpenTelemetry(CustomLogger):
                 "finish_reason": choice.get("finish_reason"),
             }
             body_msg = choice.get("message", {})
-            if self.message_logging and body_msg.get("content"):
+            capture_event_content = self._capture_in_event()
+            if capture_event_content and body_msg.get("content"):
                 attrs["message.content"] = body_msg["content"]
             body = {
                 "index": idx,
                 "finish_reason": choice.get("finish_reason"),
                 "message": {"role": body_msg.get("role", "assistant")},
             }
-            if self.message_logging and body_msg.get("content"):
+            if capture_event_content and body_msg.get("content"):
                 body["message"]["content"] = body_msg["content"]
 
             log_record = SdkLogRecord(
@@ -1674,9 +1752,7 @@ class OpenTelemetry(CustomLogger):
             ########## LLM Request Medssages / tools / content Attributes ###########
             #########################################################################
 
-            if litellm.turn_off_message_logging is True:
-                return
-            if self.message_logging is not True:
+            if not self._capture_in_span():
                 return
 
             if optional_params.get("tools"):
@@ -1695,17 +1771,41 @@ class OpenTelemetry(CustomLogger):
                     value=safe_dumps(transformed_messages),
                 )
 
-            if kwargs.get("system_instructions"):
-                transformed_system_instructions = (
-                    self._transform_messages_to_otel_semantic_conventions(
-                        kwargs.get("system_instructions")
+            # Coalesce the different kwarg names that carry the system
+            # prompt depending on the call path:
+            #   - "system_instructions" — Vertex AI Gemini chat-completion
+            #   - "instructions"        — OpenAI Responses API
+            #   - "system"              — Anthropic Messages API
+            # Use `is not None` rather than truthiness to avoid falsy
+            # values (e.g. []) falling through to the wrong kwarg.
+            system_instructions = (
+                kwargs.get("system_instructions")
+                if kwargs.get("system_instructions") is not None
+                else (
+                    kwargs.get("instructions")
+                    if kwargs.get("instructions") is not None
+                    else kwargs.get("system")
+                )
+            )
+            if system_instructions:
+                if isinstance(system_instructions, str):
+                    # Plain text system prompt — no transformation needed
+                    self.safe_set_attribute(
+                        span=span,
+                        key=SpanAttributes.GEN_AI_SYSTEM_INSTRUCTIONS.value,
+                        value=system_instructions,
+                    )
+                else:
+                    transformed_system_instructions = (
+                        self._transform_messages_to_otel_semantic_conventions(
+                            system_instructions
+                        )
+                    )
+                    self.safe_set_attribute(
+                        span=span,
+                        key=SpanAttributes.GEN_AI_SYSTEM_INSTRUCTIONS.value,
+                        value=safe_dumps(transformed_system_instructions),
                     )
-                )
-                self.safe_set_attribute(
-                    span=span,
-                    key=SpanAttributes.GEN_AI_SYSTEM_INSTRUCTIONS.value,
-                    value=safe_dumps(transformed_system_instructions),
-                )
 
             self.safe_set_attribute(
                 span=span,
@@ -1764,6 +1864,57 @@ class OpenTelemetry(CustomLogger):
                                         value=value,
                                     )
 
+                elif response_obj.get("output"):
+                    # Responses API: ResponsesAPIResponse has an "output"
+                    # list instead of "choices".  Each item with
+                    # type="message" contains a "content" list of
+                    # OutputText objects (type="output_text").
+                    output_items = response_obj.get("output")
+                    output_messages = self._transform_responses_api_output_to_otel(
+                        output_items
+                    )
+                    if output_messages:
+                        self.safe_set_attribute(
+                            span=span,
+                            key=SpanAttributes.GEN_AI_OUTPUT_MESSAGES.value,
+                            value=safe_dumps(output_messages),
+                        )
+
+                    # Emit per-tool-call span attributes (parity with
+                    # the choices branch that calls _tool_calls_kv_pair).
+                    # Convert Responses API function_call items to the
+                    # ChatCompletionMessageToolCall format expected by
+                    # _tool_calls_kv_pair.
+                    tool_calls = []
+                    for out_item in output_items:
+                        item_d = self._to_dict(out_item)
+                        if item_d and item_d.get("type") == "function_call":
+                            tool_calls.append(
+                                {
+                                    "function": {
+                                        "name": item_d.get("name", ""),
+                                        "arguments": item_d.get("arguments", ""),
+                                    }
+                                }
+                            )
+                    if tool_calls:
+                        kv_pairs = OpenTelemetry._tool_calls_kv_pair(tool_calls)  # type: ignore
+                        for key, value in kv_pairs.items():
+                            self.safe_set_attribute(
+                                span=span,
+                                key=key,
+                                value=value,
+                            )
+
+                    # Extract finish reason from ResponsesAPIResponse.status
+                    status = response_obj.get("status")
+                    if status:
+                        self.safe_set_attribute(
+                            span=span,
+                            key=SpanAttributes.GEN_AI_RESPONSE_FINISH_REASONS.value,
+                            value=safe_dumps([status]),
+                        )
+
         except Exception as e:
             self.handle_callback_failure(
                 callback_name=self.callback_name or "opentelemetry"
@@ -1859,6 +2010,78 @@ class OpenTelemetry(CustomLogger):
             transformed.append(transformed_msg)
         return transformed
 
+    @staticmethod
+    def _to_dict(obj) -> Optional[dict]:
+        """Normalize an object to a plain dict.
+
+        Handles three forms that appear in practice:
+
+        1. Plain ``dict`` — returned as-is.
+        2. LiteLLM's ``BaseLiteLLMOpenAIResponseObject`` — exposes a
+           ``.get()`` method that delegates to ``__dict__``.
+        3. Raw Pydantic v2 models from the ``openai`` SDK (e.g.
+           ``ResponseOutputMessage``, ``ResponseOutputText``) — these do
+           **not** have ``.get()`` but do have ``.model_dump()``.
+
+        Returns ``None`` for anything else so callers can skip it.
+        """
+        if isinstance(obj, dict):
+            return obj
+        if hasattr(obj, "get"):
+            # BaseLiteLLMOpenAIResponseObject duck-type
+            return obj  # type: ignore[return-value]
+        if hasattr(obj, "model_dump"):
+            # Raw Pydantic v2 model (e.g. openai SDK types)
+            return obj.model_dump()  # type: ignore[union-attr]
+        return None
+
+    def _transform_responses_api_output_to_otel(self, output: List) -> List[dict]:
+        """
+        Transform Responses API output items into OTEL GenAI 1.38 format.
+
+        The Responses API returns output as a list of items, each with a
+        ``type`` field.  Message items (``type="message"``) contain a
+        ``content`` list of ``OutputText`` objects with ``type="output_text"``
+        and ``text`` fields.
+
+        Items may be plain dicts, LiteLLM wrapper objects (with ``.get()``),
+        or raw Pydantic v2 models from the ``openai`` SDK (with
+        ``.model_dump()``).  We normalize each item to a dict via
+        ``_to_dict`` before processing.
+
+        This method converts them to the same ``{"role": ..., "parts": [...]}``
+        format used by ``_transform_choices_to_otel_semantic_conventions``.
+        """
+        transformed = []
+        for raw_item in output:
+            item = self._to_dict(raw_item)
+            if item is None:
+                continue
+            if item.get("type") == "message":
+                role = item.get("role", "assistant")
+                parts = []
+                for raw_content in item.get("content", []):
+                    content = self._to_dict(raw_content)
+                    if content is None:
+                        continue
+                    if content.get("type") == "output_text":
+                        text = content.get("text", "")
+                        if text:
+                            parts.append({"type": "text", "content": text})
+                if parts:
+                    transformed.append({"role": role, "parts": parts})
+            elif item.get("type") == "function_call":
+                # Surface tool calls from Responses API output
+                part: dict = {
+                    "type": "tool_call",
+                    "name": item.get("name", ""),
+                    "arguments": item.get("arguments", ""),
+                }
+                if item.get("call_id"):
+                    part["id"] = item["call_id"]
+                transformed.append({"role": "assistant", "parts": [part]})
+        return transformed
+
     def set_raw_request_attributes(self, span: Span, kwargs, response_obj):
         try:
             # Only set provider-specific raw payload attributes on this span.

litellm/integrations/prometheus.py

@@ -25,6 +25,9 @@ from typing import (
 import litellm
 from litellm._logging import print_verbose, verbose_logger
 from litellm.integrations.custom_logger import CustomLogger
+from litellm.integrations.prometheus_helpers.bounded_prometheus_series_tracker import (
+    BoundedPrometheusSeriesTracker,
+)
 from litellm.integrations.prometheus_helpers import (
     PrometheusLabelFactoryContext,
     _get_cached_end_user_id_for_cost_tracking,
@@ -81,6 +84,7 @@ class PrometheusLogger(CustomLogger):
                 if _custom_buckets is not None
                 else LATENCY_BUCKETS
             )
+            self._bounded_prometheus_series_tracker = BoundedPrometheusSeriesTracker()
 
             # Create metric factory functions
             self._counter_factory = self._create_metric_factory(Counter)
@@ -984,6 +988,40 @@ class PrometheusLogger(CustomLogger):
 
         return filtered_labels
 
+    def _track_end_user_metric_series(
+        self,
+        metric: Any,
+        metric_name: DEFINED_PROMETHEUS_METRICS,
+        labels: Dict[str, Optional[str]],
+    ) -> None:
+        """
+        Cap the cardinality of metrics that include the ``end_user`` label.
+
+        Called *after* ``metric.labels(...).inc()/observe()`` so the emission is
+        recorded in prometheus-client's child map before any eviction runs.
+        Series that get evicted before the next scrape lose updates accrued
+        since the last scrape — this is inherent to any cardinality cap.
+        """
+        labelnames = self.get_labels_for_metric(metric_name)
+        if UserAPIKeyLabelNames.END_USER.value not in labelnames:
+            return
+        if labels.get(UserAPIKeyLabelNames.END_USER.value) is None:
+            return
+
+        max_series = litellm.prometheus_end_user_metrics_max_series_per_metric
+        ttl_seconds = litellm.prometheus_end_user_metrics_ttl_seconds
+        if max_series is None and ttl_seconds is None:
+            return
+
+        self._bounded_prometheus_series_tracker.track_series(
+            metric=metric,
+            metric_name=metric_name,
+            label_values=tuple(labels.get(label) for label in labelnames),
+            max_series=max_series,
+            ttl_seconds=ttl_seconds,
+            cleanup_interval_seconds=litellm.prometheus_end_user_metrics_cleanup_interval_seconds,
+        )
+
     def _inc_labeled_counter(
         self,
         counter: Any,
@@ -998,6 +1036,7 @@ class PrometheusLogger(CustomLogger):
             label_context=label_context,
         )
         counter.labels(**_labels).inc(amount)
+        self._track_end_user_metric_series(counter, metric_name, _labels)
 
     async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
         # Define prometheus client
@@ -1404,12 +1443,12 @@ class PrometheusLogger(CustomLogger):
         )
         remaining_tokens_variable_name = f"litellm-key-remaining-tokens-{model_group}"
 
-        remaining_requests = (
-            metadata.get(remaining_requests_variable_name, sys.maxsize) or sys.maxsize
-        )
-        remaining_tokens = (
-            metadata.get(remaining_tokens_variable_name, sys.maxsize) or sys.maxsize
-        )
+        remaining_requests = metadata.get(remaining_requests_variable_name)
+        if remaining_requests is None:
+            remaining_requests = sys.maxsize
+        remaining_tokens = metadata.get(remaining_tokens_variable_name)
+        if remaining_tokens is None:
+            remaining_tokens = sys.maxsize
 
         enum_values = UserAPIKeyLabelValues(
             hashed_api_key=user_api_key,
@@ -1479,6 +1518,11 @@ class PrometheusLogger(CustomLogger):
             self.litellm_llm_api_time_to_first_token_metric.labels(
                 **_ttft_labels
             ).observe(time_to_first_token_seconds)
+            self._track_end_user_metric_series(
+                self.litellm_llm_api_time_to_first_token_metric,
+                "litellm_llm_api_time_to_first_token_metric",
+                _ttft_labels,
+            )
         else:
             verbose_logger.debug(
                 "Time to first token metric not emitted, stream option in model_parameters is not True"
@@ -1499,6 +1543,11 @@ class PrometheusLogger(CustomLogger):
             self.litellm_llm_api_latency_metric.labels(**_labels).observe(
                 api_call_total_time_seconds
             )
+            self._track_end_user_metric_series(
+                self.litellm_llm_api_latency_metric,
+                "litellm_llm_api_latency_metric",
+                _labels,
+            )
 
         # total request latency
         total_time_seconds = self._safe_duration_seconds(
@@ -1516,6 +1565,11 @@ class PrometheusLogger(CustomLogger):
             self.litellm_request_total_latency_metric.labels(**_labels).observe(
                 total_time_seconds
             )
+            self._track_end_user_metric_series(
+                self.litellm_request_total_latency_metric,
+                "litellm_request_total_latency_metric",
+                _labels,
+            )
 
         # request queue time (time from arrival to processing start)
         _litellm_params = kwargs.get("litellm_params", {}) or {}
@@ -1533,6 +1587,11 @@ class PrometheusLogger(CustomLogger):
             self.litellm_request_queue_time_metric.labels(**_labels).observe(
                 queue_time_seconds
             )
+            self._track_end_user_metric_series(
+                self.litellm_request_queue_time_metric,
+                "litellm_request_queue_time_seconds",
+                _labels,
+            )
 
     async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
         verbose_logger.debug(

litellm/integrations/prometheus_helpers/bounded_prometheus_series_tracker.py

@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import time
+from collections import OrderedDict
+from threading import RLock
+from typing import Any, Dict, Optional
+
+
+class BoundedPrometheusSeriesTracker:
+    """
+    Tracks Prometheus child series and removes stale/excess labelsets.
+
+    The tracker is label-agnostic: callers decide which series should be tracked
+    and pass the full label tuple used by the Prometheus metric.
+    """
+
+    def __init__(self) -> None:
+        self._series: Dict[str, OrderedDict[tuple[Optional[str], ...], float]] = {}
+        self._last_ttl_cleanup: Dict[str, float] = {}
+        self.lock = RLock()
+
+    def track_series(
+        self,
+        metric: Any,
+        metric_name: str,
+        label_values: tuple[Optional[str], ...],
+        max_series: Optional[int],
+        ttl_seconds: Optional[float],
+        cleanup_interval_seconds: Optional[float],
+    ) -> None:
+        if max_series is None and ttl_seconds is None:
+            return
+
+        now = time.monotonic()
+
+        with self.lock:
+            series = self._series.setdefault(metric_name, OrderedDict())
+            series[label_values] = now
+            series.move_to_end(label_values)
+
+            if ttl_seconds is not None and self._should_run_ttl_cleanup(
+                metric_name=metric_name,
+                now=now,
+                cleanup_interval_seconds=cleanup_interval_seconds,
+            ):
+                expired_label_values = [
+                    tracked_label_values
+                    for tracked_label_values, last_seen in series.items()
+                    if now - last_seen > ttl_seconds
+                ]
+                for tracked_label_values in expired_label_values:
+                    self._remove_metric_series(metric, series, tracked_label_values)
+
+            # max_series <= 0 is treated as "unlimited" so a misconfigured zero
+            # value cannot silently drop every emission for this metric.
+            if max_series is not None and max_series > 0:
+                while len(series) > max_series:
+                    tracked_label_values = next(iter(series))
+                    if not self._remove_metric_child(metric, tracked_label_values):
+                        break
+                    del series[tracked_label_values]
+
+    def _should_run_ttl_cleanup(
+        self,
+        metric_name: str,
+        now: float,
+        cleanup_interval_seconds: Optional[float],
+    ) -> bool:
+        if cleanup_interval_seconds is None or cleanup_interval_seconds <= 0:
+            self._last_ttl_cleanup[metric_name] = now
+            return True
+
+        last_cleanup = self._last_ttl_cleanup.get(metric_name)
+        if last_cleanup is None or now - last_cleanup >= cleanup_interval_seconds:
+            self._last_ttl_cleanup[metric_name] = now
+            return True
+        return False
+
+    def _remove_metric_series(
+        self,
+        metric: Any,
+        series: OrderedDict[tuple[Optional[str], ...], float],
+        label_values: tuple[Optional[str], ...],
+    ) -> None:
+        if self._remove_metric_child(metric, label_values):
+            series.pop(label_values, None)
+
+    @staticmethod
+    def _remove_metric_child(
+        metric: Any, label_values: tuple[Optional[str], ...]
+    ) -> bool:
+        """
+        Remove the Prometheus child for ``label_values`` and report whether the
+        tracker should commit the matching state change.
+
+        Returns ``True`` when the child is no longer present in Prometheus
+        (either it was just removed or it was already gone), and ``False`` when
+        ``metric.remove()`` raised an unexpected error and the child likely
+        still exists.
+        """
+        try:
+            metric.remove(*label_values)
+            return True
+        except KeyError:
+            return True
+        except (AttributeError, ValueError):
+            return False

litellm/integrations/s3_v2.py

@@ -16,6 +16,7 @@ from litellm._logging import print_verbose, verbose_logger
 from litellm.constants import DEFAULT_S3_BATCH_SIZE, DEFAULT_S3_FLUSH_INTERVAL_SECONDS
 from litellm.integrations.s3 import get_s3_object_key
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
+from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
 from litellm.llms.custom_httpx.http_handler import (
     _get_httpx_client,
@@ -53,15 +54,25 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
         s3_strip_base64_files: bool = False,
         s3_use_key_prefix: bool = False,
         s3_use_virtual_hosted_style: bool = False,
+        s3_callback_params_override: Optional[dict] = None,
         **kwargs,
     ):
         try:
-            verbose_logger.debug(
-                f"in init s3 logger - s3_callback_params {litellm.s3_callback_params}"
-            )
+            _masker = SensitiveDataMasker()
+            if s3_callback_params_override is not None:
+                verbose_logger.debug(
+                    f"in init s3 logger (audit override) - "
+                    f"{_masker.mask_dict(dict(s3_callback_params_override))}"
+                )
+            else:
+                verbose_logger.debug(
+                    f"in init s3 logger - s3_callback_params "
+                    f"{_masker.mask_dict(dict(litellm.s3_callback_params or {}))}"
+                )
 
             # Initialize S3 params first to get the correct s3_verify value
             self._init_s3_params(
+                params_source=s3_callback_params_override,
                 s3_bucket_name=s3_bucket_name,
                 s3_region_name=s3_region_name,
                 s3_api_version=s3_api_version,
@@ -139,94 +150,85 @@ class S3Logger(CustomBatchLogger, BaseAWSLLM):
         s3_strip_base64_files: bool = False,
         s3_use_key_prefix: bool = False,
         s3_use_virtual_hosted_style: bool = False,
+        params_source: Optional[dict] = None,
     ):
         """
-        Initialize the s3 params for this logging callback
+        Initialize the s3 params for this logging callback. Reads from
+        `params_source` if given (e.g. `s3_audit_callback_params` for the
+        audit-log instance), otherwise falls back to `litellm.s3_callback_params`.
+        Resolves `os.environ/X` markers into a local dict; never mutates the source.
         """
-        litellm.s3_callback_params = litellm.s3_callback_params or {}
-        # read in .env variables - example os.environ/AWS_BUCKET_NAME
-        for key, value in litellm.s3_callback_params.items():
-            if isinstance(value, str) and value.startswith("os.environ/"):
-                litellm.s3_callback_params[key] = litellm.get_secret(value)
+        if params_source is None:
+            params_source = litellm.s3_callback_params or {}
+        params: dict = {
+            key: (
+                litellm.get_secret(value)
+                if isinstance(value, str) and value.startswith("os.environ/")
+                else value
+            )
+            for key, value in params_source.items()
+        }
 
-        self.s3_bucket_name = (
-            litellm.s3_callback_params.get("s3_bucket_name") or s3_bucket_name
-        )
-        self.s3_region_name = (
-            litellm.s3_callback_params.get("s3_region_name") or s3_region_name
-        )
-        self.s3_api_version = (
-            litellm.s3_callback_params.get("s3_api_version") or s3_api_version
-        )
+        self.s3_bucket_name = params.get("s3_bucket_name") or s3_bucket_name
+        self.s3_region_name = params.get("s3_region_name") or s3_region_name
+        self.s3_api_version = params.get("s3_api_version") or s3_api_version
         self.s3_use_ssl = (
-            litellm.s3_callback_params.get("s3_use_ssl", True)
-            if litellm.s3_callback_params.get("s3_use_ssl") is not None
+            params.get("s3_use_ssl", True)
+            if params.get("s3_use_ssl") is not None
             else s3_use_ssl
         )
         self.s3_verify = (
-            litellm.s3_callback_params.get("s3_verify")
-            if litellm.s3_callback_params.get("s3_verify") is not None
+            params.get("s3_verify")
+            if params.get("s3_verify") is not None
             else s3_verify
         )
-        self.s3_endpoint_url = (
-            litellm.s3_callback_params.get("s3_endpoint_url") or s3_endpoint_url
-        )
+        self.s3_endpoint_url = params.get("s3_endpoint_url") or s3_endpoint_url
         self.s3_aws_access_key_id = (
-            litellm.s3_callback_params.get("s3_aws_access_key_id")
-            or s3_aws_access_key_id
+            params.get("s3_aws_access_key_id") or s3_aws_access_key_id
         )
 
         self.s3_aws_secret_access_key = (
-            litellm.s3_callback_params.get("s3_aws_secret_access_key")
-            or s3_aws_secret_access_key
+            params.get("s3_aws_secret_access_key") or s3_aws_secret_access_key
         )
 
         self.s3_aws_session_token = (
-            litellm.s3_callback_params.get("s3_aws_session_token")
-            or s3_aws_session_token
+            params.get("s3_aws_session_token") or s3_aws_session_token
         )
 
         self.s3_aws_session_name = (
-            litellm.s3_callback_params.get("s3_aws_session_name") or s3_aws_session_name
+            params.get("s3_aws_session_name") or s3_aws_session_name
         )
 
         self.s3_aws_profile_name = (
-            litellm.s3_callback_params.get("s3_aws_profile_name") or s3_aws_profile_name
+            params.get("s3_aws_profile_name") or s3_aws_profile_name
         )
 
-        self.s3_aws_role_name = (
-            litellm.s3_callback_params.get("s3_aws_role_name") or s3_aws_role_name
-        )
+        self.s3_aws_role_name = params.get("s3_aws_role_name") or s3_aws_role_name
 
         self.s3_aws_web_identity_token = (
-            litellm.s3_callback_params.get("s3_aws_web_identity_token")
-            or s3_aws_web_identity_token
+            params.get("s3_aws_web_identity_token") or s3_aws_web_identity_token
         )
 
         self.s3_aws_sts_endpoint = (
-            litellm.s3_callback_params.get("s3_aws_sts_endpoint") or s3_aws_sts_endpoint
+            params.get("s3_aws_sts_endpoint") or s3_aws_sts_endpoint
         )
 
-        self.s3_config = litellm.s3_callback_params.get("s3_config") or s3_config
-        self.s3_path = litellm.s3_callback_params.get("s3_path") or s3_path
-        # done reading litellm.s3_callback_params
+        self.s3_config = params.get("s3_config") or s3_config
+        self.s3_path = params.get("s3_path") or s3_path
         self.s3_use_team_prefix = (
-            bool(litellm.s3_callback_params.get("s3_use_team_prefix", False))
-            or s3_use_team_prefix
+            bool(params.get("s3_use_team_prefix", False)) or s3_use_team_prefix
         )
 
         self.s3_use_key_prefix = (
-            bool(litellm.s3_callback_params.get("s3_use_key_prefix", False))
-            or s3_use_key_prefix
+            bool(params.get("s3_use_key_prefix", False)) or s3_use_key_prefix
         )
 
         self.s3_strip_base64_files = (
-            bool(litellm.s3_callback_params.get("s3_strip_base64_files", False))
-            or s3_strip_base64_files
+            bool(params.get("s3_strip_base64_files", False)) or s3_strip_base64_files
         )
 
         self.s3_use_virtual_hosted_style = (
-            bool(litellm.s3_callback_params.get("s3_use_virtual_hosted_style", False))
+            bool(params.get("s3_use_virtual_hosted_style", False))
             or s3_use_virtual_hosted_style
         )
 

litellm/litellm_core_utils/custom_logger_registry.py

@@ -14,6 +14,7 @@ from litellm import _custom_logger_compatible_callbacks_literal
 from litellm.integrations.agentops import AgentOps
 from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
 from litellm.integrations.argilla import ArgillaLogger
+from litellm.integrations.azure_sentinel.azure_sentinel import AzureSentinelLogger
 from litellm.integrations.azure_storage.azure_storage import AzureBlobStorageLogger
 from litellm.integrations.bitbucket import BitBucketPromptManager
 from litellm.integrations.braintrust_logging import BraintrustLogger
@@ -73,6 +74,7 @@ class CustomLoggerRegistry:
         "opik": OpikLogger,
         "argilla": ArgillaLogger,
         "opentelemetry": OpenTelemetry,
+        "azure_sentinel": AzureSentinelLogger,
         "azure_storage": AzureBlobStorageLogger,
         "humanloop": HumanloopLogger,
         # OTEL compatible loggers

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -436,12 +436,21 @@ def update_messages_with_model_file_ids(
     """
     Updates messages with model file ids.
 
+    For managed files (unified file IDs), uses model_file_id_mapping if it
+    resolves the id, otherwise decodes the base64-encoded unified file ID
+    and extracts the llm_output_file_id directly. Mirrors the Responses-API
+    sibling `update_responses_input_with_model_file_ids`.
+
     model_file_id_mapping: Dict[str, Dict[str, str]] = {
         "litellm_proxy/file_id": {
             "model_id": "provider_file_id"
         }
     }
     """
+    from litellm.proxy.openai_files_endpoints.common_utils import (
+        _is_base64_encoded_unified_file_id,
+        convert_b64_uid_to_unified_uid,
+    )
 
     for message in messages:
         if message.get("role") == "user":
@@ -450,7 +459,13 @@ def update_messages_with_model_file_ids(
                 if isinstance(content, str):
                     continue
                 for c in content:
-                    if c["type"] == "file":
+                    if not isinstance(c, dict):
+                        # Content list items aren't always dicts. e.g.
+                        # text_completion forwards a token-ids list/list-of-
+                        # lists through this path. Skip non-dict items
+                        # instead of indexing into them.
+                        continue
+                    if c.get("type") == "file":
                         file_object = cast(ChatCompletionFileObject, c)
                         file_object_file_field = file_object.get("file")
                         if not isinstance(file_object_file_field, dict):
@@ -468,9 +483,23 @@ def update_messages_with_model_file_ids(
                         if file_id:
                             provider_file_id = (
                                 model_file_id_mapping.get(file_id, {}).get(model_id)
-                                or file_id
+                                if model_file_id_mapping
+                                else None
+                            )
+                            if (
+                                not provider_file_id
+                                and _is_base64_encoded_unified_file_id(file_id)
+                            ):
+                                unified_file_id = convert_b64_uid_to_unified_uid(
+                                    file_id
+                                )
+                                if "llm_output_file_id," in unified_file_id:
+                                    provider_file_id = unified_file_id.split(
+                                        "llm_output_file_id,"
+                                    )[1].split(";")[0]
+                            file_object_file_field["file_id"] = (
+                                provider_file_id or file_id
                             )
-                            file_object_file_field["file_id"] = provider_file_id
                         if format:
                             file_object_file_field["format"] = format
     return messages

litellm/llms/anthropic/chat/guardrail_translation/handler.py

@@ -23,7 +23,9 @@ from litellm.llms.anthropic.experimental_pass_through.adapters.transformation im
 from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
 from litellm.llms.base_llm.guardrail_translation.utils import (
     effective_skip_system_message_for_guardrail,
+    effective_skip_tool_message_for_guardrail,
     openai_messages_without_system,
+    openai_messages_without_tool,
 )
 from litellm.proxy.pass_through_endpoints.llm_provider_handlers.anthropic_passthrough_logging_handler import (
     AnthropicPassthroughLoggingHandler,
@@ -108,6 +110,7 @@ class AnthropicMessagesHandler(BaseTranslation):
             return data
 
         skip_system = effective_skip_system_message_for_guardrail(guardrail_to_apply)
+        skip_tool = effective_skip_tool_message_for_guardrail(guardrail_to_apply)
 
         chat_completion_compatible_request = self._translate_to_openai(data)
 
@@ -117,6 +120,8 @@ class AnthropicMessagesHandler(BaseTranslation):
         )
         if skip_system:
             structured_messages = openai_messages_without_system(structured_messages)
+        if skip_tool:
+            structured_messages = openai_messages_without_tool(structured_messages)
 
         texts_to_check: List[str] = []
         images_to_check: List[str] = []
@@ -134,6 +139,7 @@ class AnthropicMessagesHandler(BaseTranslation):
                 images_to_check=images_to_check,
                 task_mappings=task_mappings,
                 skip_system_message=skip_system,
+                skip_tool_message=skip_tool,
             )
 
         # Step 2: Apply guardrail to all texts in batch
@@ -198,13 +204,17 @@ class AnthropicMessagesHandler(BaseTranslation):
         images_to_check: List[str],
         task_mappings: List[Tuple[int, Optional[int]]],
         skip_system_message: bool = False,
+        skip_tool_message: bool = False,
     ) -> None:
         """
         Extract text content and images from a message.
 
         Override this method to customize text/image extraction logic.
         """
-        if skip_system_message and str(message.get("role") or "").lower() == "system":
+        role = str(message.get("role") or "").lower()
+        if skip_system_message and role == "system":
+            return
+        if skip_tool_message and role == "tool":
             return
 
         content = message.get("content", None)

litellm/llms/anthropic/chat/handler.py

@@ -579,6 +579,10 @@ class ModelResponseIterator:
         # Accumulate compaction blocks for multi-turn reconstruction
         self.compaction_blocks: List[Dict[str, Any]] = []
 
+        # Accumulate streamed thinking text so final usage can split reasoning
+        # tokens from regular output tokens.
+        self.reasoning_content_chunks: List[str] = []
+
         # Track server tool use inputs and results for code_interpreter_results
         self._server_tool_inputs: Dict[str, Any] = {}
         self.tool_results: List[Dict[str, Any]] = []
@@ -609,9 +613,14 @@ class ModelResponseIterator:
         return False
 
     def _handle_usage(self, anthropic_usage_chunk: Union[dict, UsageDelta]) -> Usage:
+        reasoning_content = (
+            "".join(self.reasoning_content_chunks)
+            if self.reasoning_content_chunks
+            else None
+        )
         return AnthropicConfig().calculate_usage(
             usage_object=cast(dict, anthropic_usage_chunk),
-            reasoning_content=None,
+            reasoning_content=reasoning_content,
             speed=self.speed,
         )
 
@@ -658,10 +667,13 @@ class ModelResponseIterator:
             "thinking" in content_block["delta"]
             or "signature" in content_block["delta"]
         ):
+            thinking_content = content_block["delta"].get("thinking")
+            if isinstance(thinking_content, str) and thinking_content:
+                self.reasoning_content_chunks.append(thinking_content)
             thinking_blocks = [
                 ChatCompletionThinkingBlock(
                     type="thinking",
-                    thinking=content_block["delta"].get("thinking") or "",
+                    thinking=thinking_content or "",
                     signature=str(content_block["delta"].get("signature") or ""),
                 )
             ]

litellm/llms/anthropic/chat/transformation.py

@@ -2156,8 +2156,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         speed: Optional[str] = None,
     ) -> Usage:
         # NOTE: Sometimes the usage object has None set explicitly for token counts, meaning .get() & key access returns None, and we need to account for this
-        prompt_tokens = usage_object.get("input_tokens", 0) or 0
-        completion_tokens = usage_object.get("output_tokens", 0) or 0
+        raw_prompt_tokens = usage_object.get("input_tokens", 0) or 0
+        prompt_tokens: int = (
+            int(raw_prompt_tokens) if isinstance(raw_prompt_tokens, (int, float)) else 0
+        )
+        raw_completion_tokens = usage_object.get("output_tokens", 0) or 0
+        completion_tokens: int = (
+            int(raw_completion_tokens)
+            if isinstance(raw_completion_tokens, (int, float))
+            else 0
+        )
         _usage = usage_object
         cache_creation_input_tokens: int = 0
         cache_read_input_tokens: int = 0
@@ -2226,11 +2234,12 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
             text_tokens=raw_input_tokens,
         )
         # Always populate completion_token_details, not just when there's reasoning_content
-        reasoning_tokens = (
+        estimated_reasoning_tokens = (
             token_counter(text=reasoning_content, count_response_tokens=True)
             if reasoning_content
             else 0
         )
+        reasoning_tokens = min(estimated_reasoning_tokens, completion_tokens)
         completion_token_details = CompletionTokensDetailsWrapper(
             reasoning_tokens=reasoning_tokens if reasoning_tokens > 0 else 0,
             text_tokens=(

litellm/llms/base_llm/guardrail_translation/utils.py

@@ -14,7 +14,22 @@ def effective_skip_system_message_for_guardrail(guardrail_to_apply: Any) -> bool
     return bool(getattr(litellm, "skip_system_message_in_guardrail", False))
 
 
+def effective_skip_tool_message_for_guardrail(guardrail_to_apply: Any) -> bool:
+    per = getattr(guardrail_to_apply, "skip_tool_message_in_guardrail", None)
+    if per is not None:
+        return bool(per)
+    import litellm
+
+    return bool(getattr(litellm, "skip_tool_message_in_guardrail", False))
+
+
 def openai_messages_without_system(
     messages: List[AllMessageValues],
 ) -> List[AllMessageValues]:
     return [m for m in messages if str((m or {}).get("role") or "").lower() != "system"]
+
+
+def openai_messages_without_tool(
+    messages: List[AllMessageValues],
+) -> List[AllMessageValues]:
+    return [m for m in messages if str((m or {}).get("role") or "").lower() != "tool"]

litellm/llms/bedrock/chat/invoke_agent/transformation.py

@@ -299,29 +299,9 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM):
             )
 
     def _get_response_stream_shape(self):
-        """Get the response stream shape for parsing, reusing existing logic."""
-        try:
-            # Try to reuse the cached shape from the existing decoder
-            from litellm.llms.bedrock.chat.invoke_handler import (
-                get_response_stream_shape,
-            )
+        from litellm.llms.bedrock.common_utils import BEDROCK_RESPONSE_STREAM_SHAPE
 
-            return get_response_stream_shape()
-        except ImportError:
-            # Fallback: create our own shape
-            try:
-                from botocore.loaders import Loader
-                from botocore.model import ServiceModel
-
-                loader = Loader()
-                bedrock_service_dict = loader.load_service_model(
-                    "bedrock-runtime", "service-2"
-                )
-                bedrock_service_model = ServiceModel(bedrock_service_dict)
-                return bedrock_service_model.shape_for("ResponseStream")
-            except Exception as e:
-                verbose_logger.warning(f"Could not load response stream shape: {e}")
-                return None
+        return BEDROCK_RESPONSE_STREAM_SHAPE
 
     def _extract_response_content(self, events: InvokeAgentEventList) -> str:
         """Extract the final response content from parsed events."""

litellm/llms/bedrock/chat/invoke_handler.py

@@ -67,9 +67,13 @@ from litellm.types.utils import (
 from litellm.utils import CustomStreamWrapper, get_secret
 
 from ..base_aws_llm import BaseAWSLLM
-from ..common_utils import BedrockError, ModelResponseIterator, get_bedrock_tool_name
+from ..common_utils import (
+    BEDROCK_RESPONSE_STREAM_SHAPE,
+    BedrockError,
+    ModelResponseIterator,
+    get_bedrock_tool_name,
+)
 
-_response_stream_shape_cache = None
 bedrock_tool_name_mappings: InMemoryCache = InMemoryCache(
     max_size_in_memory=50, default_ttl=600
 )
@@ -1391,20 +1395,6 @@ class BedrockLLM(BaseAWSLLM):
         return None
 
 
-def get_response_stream_shape():
-    global _response_stream_shape_cache
-    if _response_stream_shape_cache is None:
-        from botocore.loaders import Loader
-        from botocore.model import ServiceModel
-
-        loader = Loader()
-        bedrock_service_dict = loader.load_service_model("bedrock-runtime", "service-2")
-        bedrock_service_model = ServiceModel(bedrock_service_dict)
-        _response_stream_shape_cache = bedrock_service_model.shape_for("ResponseStream")
-
-    return _response_stream_shape_cache
-
-
 class AWSEventStreamDecoder:
     def __init__(self, model: str, json_mode: Optional[bool] = False) -> None:
         from botocore.parsers import EventStreamJSONParser
@@ -1838,8 +1828,18 @@ class AWSEventStreamDecoder:
                     yield self._chunk_parser(chunk_data=_data)
 
     def _parse_message_from_event(self, event) -> Optional[str]:
+        if BEDROCK_RESPONSE_STREAM_SHAPE is None:
+            raise BedrockError(
+                status_code=500,
+                message=(
+                    "Bedrock event-stream shape could not be loaded from botocore. "
+                    "Ensure botocore is correctly installed."
+                ),
+            )
         response_dict = event.to_response_dict()
-        parsed_response = self.parser.parse(response_dict, get_response_stream_shape())
+        parsed_response = self.parser.parse(
+            response_dict, BEDROCK_RESPONSE_STREAM_SHAPE
+        )
 
         if response_dict["status_code"] != 200:
             decoded_body = response_dict["body"].decode()

litellm/llms/bedrock/common_utils.py

@@ -14,6 +14,7 @@ if TYPE_CHECKING:
 import httpx
 
 import litellm
+from litellm import verbose_logger
 from litellm.llms.base_llm.anthropic_messages.transformation import (
     BaseAnthropicMessagesConfig,
 )
@@ -917,38 +918,57 @@ def get_bedrock_chat_config(model: str):
         return litellm.AmazonInvokeConfig()
 
 
+def _load_bedrock_response_stream_shape():
+    """
+    Load the ResponseStream shape from botocore's bundled bedrock-runtime schema.
+
+    Called once at module import time; the result is stored in
+    ``BEDROCK_RESPONSE_STREAM_SHAPE`` and reused for the process lifetime.
+    Returns ``None`` if botocore is unavailable or the service model cannot be
+    loaded, so the module still imports cleanly.
+    """
+    try:
+        from botocore.loaders import Loader
+        from botocore.model import ServiceModel
+
+        loader = Loader()
+        service_dict = loader.load_service_model("bedrock-runtime", "service-2")
+        return ServiceModel(service_dict).shape_for("ResponseStream")
+    except Exception as e:
+        verbose_logger.warning(
+            "litellm: could not pre-load bedrock-runtime response stream shape "
+            "— Bedrock event-stream decoding will be unavailable. Error: %s",
+            e,
+        )
+        return None
+
+
+# Eagerly resolved once per process — avoids per-instance or per-request disk I/O.
+BEDROCK_RESPONSE_STREAM_SHAPE = _load_bedrock_response_stream_shape()
+
+
 class BedrockEventStreamDecoderBase:
     """
     Base class for event stream decoding for Bedrock
     """
 
-    _response_stream_shape_cache = None
-
     def __init__(self):
         from botocore.parsers import EventStreamJSONParser
 
         self.parser = EventStreamJSONParser()
 
-    def get_response_stream_shape(self):
-        if self._response_stream_shape_cache is None:
-            from botocore.loaders import Loader
-            from botocore.model import ServiceModel
-
-            loader = Loader()
-            bedrock_service_dict = loader.load_service_model(
-                "bedrock-runtime", "service-2"
-            )
-            bedrock_service_model = ServiceModel(bedrock_service_dict)
-            self._response_stream_shape_cache = bedrock_service_model.shape_for(
-                "ResponseStream"
-            )
-
-        return self._response_stream_shape_cache
-
     def _parse_message_from_event(self, event) -> Optional[str]:
+        if BEDROCK_RESPONSE_STREAM_SHAPE is None:
+            raise BedrockError(
+                status_code=500,
+                message=(
+                    "Bedrock event-stream shape could not be loaded from botocore. "
+                    "Ensure botocore is correctly installed."
+                ),
+            )
         response_dict = event.to_response_dict()
         parsed_response = self.parser.parse(
-            response_dict, self.get_response_stream_shape()
+            response_dict, BEDROCK_RESPONSE_STREAM_SHAPE
         )
 
         if response_dict["status_code"] != 200:

litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py

@@ -408,6 +408,47 @@ class AmazonAnthropicClaudeMessagesConfig(
             if self._supports_tool_search_on_bedrock(model):
                 beta_set.add("tool-search-tool-2025-10-19")
 
+    @staticmethod
+    def _filter_context_management_for_bedrock_invoke(
+        anthropic_messages_request: Dict,
+        beta_set: set,
+    ) -> None:
+        """
+        Bedrock InvokeModel accepts ``context_management`` only when it carries
+        ``compact_20260112`` edits paired with the ``compact-2026-01-12``
+        anthropic-beta header. Other edit types (notably ``clear_thinking_20251015``,
+        which Claude Code sends on every request) are LiteLLM-internal and would
+        cause Bedrock to 400 with ``"context_management: Extra inputs are not
+        permitted"``.
+
+        Filter the edits list to the supported subset, add the beta header when
+        compact edits remain, and drop ``context_management`` entirely when no
+        supported edits are left so the safety-net allowlist can pass it through.
+
+        Ref: https://github.com/BerriAI/litellm/issues/27532
+        """
+        cm = anthropic_messages_request.get("context_management")
+        if not isinstance(cm, dict):
+            return
+        edits = cm.get("edits")
+        if not isinstance(edits, list):
+            anthropic_messages_request.pop("context_management", None)
+            return
+
+        compact_edits = [
+            e
+            for e in edits
+            if isinstance(e, dict) and e.get("type") == "compact_20260112"
+        ]
+        if compact_edits:
+            beta_set.add("compact-2026-01-12")
+            anthropic_messages_request["context_management"] = {
+                **cm,
+                "edits": compact_edits,
+            }
+        else:
+            anthropic_messages_request.pop("context_management", None)
+
     def _convert_output_format_to_inline_schema(
         self,
         output_format: Dict,
@@ -551,6 +592,11 @@ class AmazonAnthropicClaudeMessagesConfig(
         if injected_thinking_for_clear_thinking:
             beta_set.add("interleaved-thinking-2025-05-14")
 
+        self._filter_context_management_for_bedrock_invoke(
+            anthropic_messages_request=anthropic_messages_request,
+            beta_set=beta_set,
+        )
+
         self._get_tool_search_beta_header_for_bedrock(
             model=model,
             tool_search_used=tool_search_used,
@@ -597,8 +643,9 @@ class AmazonAnthropicClaudeMessagesConfig(
             anthropic_messages_request.pop("output_config", None)
 
         # 7. Final safety net: filter top-level fields to the Bedrock Invoke allowlist.
-        # Catches Anthropic-only extensions (context_management, output_config, speed,
-        # mcp_servers, ...) and any future additions Claude Code may start sending.
+        # Catches Anthropic-only extensions (output_config, speed, mcp_servers, ...)
+        # and any future additions Claude Code may start sending. ``context_management``
+        # has already been pre-filtered to its Bedrock-supported subset above.
         allowed = self.BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS
         stripped = sorted(k for k in anthropic_messages_request if k not in allowed)
         if stripped:

litellm/llms/custom_httpx/http_handler.py

@@ -1,4 +1,5 @@
 import asyncio
+import concurrent.futures
 import inspect
 import os
 import socket
@@ -133,6 +134,11 @@ _DEFAULT_TIMEOUT = httpx.Timeout(
     timeout=COMPLETION_HTTP_FALLBACK_SECONDS,
     connect=HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS,
 )
+_STREAMING_ERROR_BODY_READ_TIMEOUT_SECONDS = 5.0
+_STREAMING_ERROR_BODY_READ_EXECUTOR = concurrent.futures.ThreadPoolExecutor(
+    max_workers=50,
+    thread_name_prefix="litellm-streaming-error-body-read",
+)
 
 
 def _prepare_request_data_and_content(
@@ -386,17 +392,30 @@ def _safe_get_response_text(response: httpx.Response) -> str:
         return ""
 
 
-async def _safe_aread_response(response: httpx.Response) -> bytes:
+async def _safe_aread_response(
+    response: httpx.Response, timeout: Optional[float] = None
+) -> bytes:
     """Safely read async response body, falling back to empty bytes on errors."""
     try:
+        if timeout is not None:
+            return await asyncio.wait_for(response.aread(), timeout=timeout)
         return await response.aread()
     except Exception:
         return b""
 
 
-def _safe_read_response(response: httpx.Response) -> bytes:
+def _safe_read_response(
+    response: httpx.Response, timeout: Optional[float] = None
+) -> bytes:
     """Safely read sync response body, falling back to empty bytes on errors."""
     try:
+        if timeout is not None:
+            future = _STREAMING_ERROR_BODY_READ_EXECUTOR.submit(response.read)
+            try:
+                return future.result(timeout=timeout)
+            except Exception:
+                response.close()
+                return b""
         return response.read()
     except Exception:
         return b""
@@ -405,8 +424,19 @@ def _safe_read_response(response: httpx.Response) -> bytes:
 def _raise_masked_sync_error(e: httpx.HTTPStatusError, stream: bool) -> None:
     """Raise a MaskedHTTPStatusError for sync HTTP handlers."""
     if stream:
-        _body = mask_sensitive_info(_safe_read_response(e.response))
-        raise MaskedHTTPStatusError(e, message=_body, text=_body) from None
+        try:
+            _body = mask_sensitive_info(
+                _safe_read_response(
+                    e.response,
+                    timeout=_STREAMING_ERROR_BODY_READ_TIMEOUT_SECONDS,
+                )
+            )
+            raise MaskedHTTPStatusError(e, message=_body, text=_body) from None
+        finally:
+            try:
+                e.response.close()
+            except Exception:
+                pass
     _text = mask_sensitive_info(_safe_get_response_text(e.response))
     raise MaskedHTTPStatusError(e, message=_text, text=_text) from None
 
@@ -414,8 +444,19 @@ def _raise_masked_sync_error(e: httpx.HTTPStatusError, stream: bool) -> None:
 async def _raise_masked_async_error(e: httpx.HTTPStatusError, stream: bool) -> None:
     """Raise a MaskedHTTPStatusError for async HTTP handlers."""
     if stream:
-        _body = mask_sensitive_info(await _safe_aread_response(e.response))
-        raise MaskedHTTPStatusError(e, message=_body, text=_body) from None
+        try:
+            _body = mask_sensitive_info(
+                await _safe_aread_response(
+                    e.response,
+                    timeout=_STREAMING_ERROR_BODY_READ_TIMEOUT_SECONDS,
+                )
+            )
+            raise MaskedHTTPStatusError(e, message=_body, text=_body) from None
+        finally:
+            try:
+                await e.response.aclose()
+            except Exception:
+                pass
     _text = mask_sensitive_info(_safe_get_response_text(e.response))
     raise MaskedHTTPStatusError(e, message=_text, text=_text) from None
 

litellm/llms/openai/chat/guardrail_translation/handler.py

@@ -21,7 +21,9 @@ from litellm._logging import verbose_proxy_logger
 from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
 from litellm.llms.base_llm.guardrail_translation.utils import (
     effective_skip_system_message_for_guardrail,
+    effective_skip_tool_message_for_guardrail,
     openai_messages_without_system,
+    openai_messages_without_tool,
 )
 from litellm.main import stream_chunk_builder
 from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam
@@ -73,6 +75,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
             return data
 
         skip_system = effective_skip_system_message_for_guardrail(guardrail_to_apply)
+        skip_tool = effective_skip_tool_message_for_guardrail(guardrail_to_apply)
 
         texts_to_check: List[str] = []
         images_to_check: List[str] = []
@@ -91,6 +94,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
                 text_task_mappings=text_task_mappings,
                 tool_call_task_mappings=tool_call_task_mappings,
                 skip_system_message=skip_system,
+                skip_tool_message=skip_tool,
             )
 
         # Step 2: Apply guardrail to all texts and tool calls in batch
@@ -102,11 +106,15 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
                 inputs["tool_calls"] = tool_calls_to_check  # type: ignore
             structured_messages = self.get_structured_messages(data)
             if structured_messages:
-                inputs["structured_messages"] = (
-                    openai_messages_without_system(structured_messages)
-                    if skip_system
-                    else structured_messages
-                )
+                if skip_system:
+                    structured_messages = openai_messages_without_system(
+                        structured_messages
+                    )
+                if skip_tool:
+                    structured_messages = openai_messages_without_tool(
+                        structured_messages
+                    )
+                inputs["structured_messages"] = structured_messages
             # Pass tools (function definitions) to the guardrail
             tools = data.get("tools")
             if tools:
@@ -176,13 +184,17 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         text_task_mappings: List[Tuple[int, Optional[int]]],
         tool_call_task_mappings: List[Tuple[int, int]],
         skip_system_message: bool = False,
+        skip_tool_message: bool = False,
     ) -> None:
         """
         Extract text content, images, and tool calls from a message.
 
         Override this method to customize text/image/tool call extraction logic.
         """
-        if skip_system_message and str(message.get("role") or "").lower() == "system":
+        role = str(message.get("role") or "").lower()
+        if skip_system_message and role == "system":
+            return
+        if skip_tool_message and role == "tool":
             return
 
         content = message.get("content", None)

litellm/llms/ovhcloud/audio_transcription/transformation.py

@@ -156,5 +156,17 @@ class OVHCloudAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
         text = response_json.get("text") or response_json.get("transcript") or ""
         response = TranscriptionResponse(text=text)
 
+        # OVHCloud field migration (deadline: 2026-05-11):
+        # `duration` is replaced by `seconds` in STT responses.
+        # Prefer `seconds`, fall back to `duration`, normalize to `duration`
+        # so downstream consumers see a consistent key.
+        duration = (
+            response_json["seconds"]
+            if "seconds" in response_json and response_json["seconds"] is not None
+            else response_json.get("duration")
+        )
+        if duration is not None:
+            response_json["duration"] = duration
+
         response._hidden_params = response_json
         return response

litellm/llms/ovhcloud/chat/transformation.py

@@ -13,6 +13,7 @@ from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
 from litellm.llms.ovhcloud.utils import OVHCloudException
 from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
+
 from litellm.types.llms.openai import AllMessageValues
 
 
@@ -98,10 +99,16 @@ class OVHCloudChatCompletionStreamingHandler(BaseModelResponseIterator):
 
             new_choices = []
             for choice in chunk["choices"]:
-                if "delta" in choice and "reasoning" in choice["delta"]:
-                    choice["delta"]["reasoning_content"] = choice["delta"].get(
-                        "reasoning"
-                    )
+                if "delta" in choice:
+                    delta = choice["delta"]
+                    # OVHCloud field migration (deadline: 2026-05-11):
+                    # `reasoning_content` is replaced by `reasoning`.
+                    # Normalise to `reasoning_content` so downstream consumers
+                    # see a consistent key during the transition window.
+                    reasoning_new = delta.get("reasoning")
+                    reasoning_legacy = delta.get("reasoning_content")
+                    if reasoning_new is not None and reasoning_legacy is None:
+                        delta["reasoning_content"] = reasoning_new
                 new_choices.append(choice)
 
             return ModelResponseStream(

litellm/llms/sagemaker/common_utils.py

@@ -9,7 +9,27 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.types.utils import GenericStreamingChunk as GChunk
 from litellm.types.utils import StreamingChatCompletionChunk
 
-_response_stream_shape_cache = None
+
+def _load_sagemaker_response_stream_shape():
+    try:
+        from botocore.loaders import Loader
+        from botocore.model import ServiceModel
+
+        loader = Loader()
+        service_dict = loader.load_service_model("sagemaker-runtime", "service-2")
+        return ServiceModel(service_dict).shape_for(
+            "InvokeEndpointWithResponseStreamOutput"
+        )
+    except Exception as e:
+        verbose_logger.warning(
+            "litellm: could not pre-load sagemaker-runtime response stream shape "
+            "— SageMaker event-stream decoding will be unavailable. Error: %s",
+            e,
+        )
+        return None
+
+
+SAGEMAKER_RESPONSE_STREAM_SHAPE = _load_sagemaker_response_stream_shape()
 
 
 class SagemakerError(BaseLLMException):
@@ -187,8 +207,18 @@ class AWSEventStreamDecoder:
                 verbose_logger.error(f"Final error parsing accumulated JSON: {e}")
 
     def _parse_message_from_event(self, event) -> Optional[str]:
+        if SAGEMAKER_RESPONSE_STREAM_SHAPE is None:
+            raise SagemakerError(
+                status_code=500,
+                message=(
+                    "SageMaker event-stream shape could not be loaded from botocore. "
+                    "Ensure botocore is correctly installed."
+                ),
+            )
         response_dict = event.to_response_dict()
-        parsed_response = self.parser.parse(response_dict, get_response_stream_shape())
+        parsed_response = self.parser.parse(
+            response_dict, SAGEMAKER_RESPONSE_STREAM_SHAPE
+        )
 
         if response_dict["status_code"] != 200:
             raise ValueError(f"Bad response code, expected 200: {response_dict}")
@@ -204,20 +234,3 @@ class AWSEventStreamDecoder:
                 return None
 
             return chunk.decode()  # type: ignore[no-any-return]
-
-
-def get_response_stream_shape():
-    global _response_stream_shape_cache
-    if _response_stream_shape_cache is None:
-        from botocore.loaders import Loader
-        from botocore.model import ServiceModel
-
-        loader = Loader()
-        sagemaker_service_dict = loader.load_service_model(
-            "sagemaker-runtime", "service-2"
-        )
-        sagemaker_service_model = ServiceModel(sagemaker_service_dict)
-        _response_stream_shape_cache = sagemaker_service_model.shape_for(
-            "InvokeEndpointWithResponseStreamOutput"
-        )
-    return _response_stream_shape_cache

litellm/main.py

@@ -1,3 +1,5 @@
+# LiteLLM main module: public completion, embedding, streaming, and moderation entrypoints.
+#
 # +-----------------------------------------------+
 # |                                               |
 # |           Give Feedback / Get Help            |
@@ -1459,14 +1461,14 @@ def completion(  # type: ignore # noqa: PLR0915
             if eos_token:
                 custom_prompt_dict[model]["eos_token"] = eos_token
 
-        if kwargs.get("model_file_id_mapping"):
-            messages = update_messages_with_model_file_ids(
-                messages=messages,
-                model_id=kwargs.get("model_info", {}).get("id", None),
-                model_file_id_mapping=cast(
-                    Dict[str, Dict[str, str]], kwargs.get("model_file_id_mapping")
-                ),
-            )
+        messages = update_messages_with_model_file_ids(
+            messages=messages,
+            model_id=kwargs.get("model_info", {}).get("id", None),
+            model_file_id_mapping=cast(
+                Dict[str, Dict[str, str]],
+                kwargs.get("model_file_id_mapping") or {},
+            ),
+        )
 
         provider_config: Optional[BaseConfig] = None
         if custom_llm_provider is not None and custom_llm_provider in [

litellm/model_prices_and_context_window_backup.json

@@ -27187,6 +27187,20 @@
         "supports_reasoning": true,
         "supports_tool_choice": true
     },
+    "openrouter/qwen/qwen3.6-plus": {
+        "input_cost_per_token": 3.25e-07,
+        "litellm_provider": "openrouter",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 65536,
+        "max_tokens": 65536,
+        "mode": "chat",
+        "output_cost_per_token": 1.95e-06,
+        "source": "https://openrouter.ai/qwen/qwen3.6-plus",
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
     "openrouter/qwen/qwen3.5-35b-a3b": {
         "input_cost_per_token": 2.5e-07,
         "litellm_provider": "openrouter",
@@ -28874,6 +28888,19 @@
         "mode": "chat",
         "output_cost_per_token": 0.0
     },
+    "sambanova/MiniMax-M2.7": {
+        "input_cost_per_token": 3e-07,
+        "litellm_provider": "sambanova",
+        "max_input_tokens": 204800,
+        "max_output_tokens": 131072,
+        "max_tokens": 131072,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-06,
+        "source": "https://cloud.sambanova.ai/plans/pricing",
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },
     "sambanova/DeepSeek-R1": {
         "input_cost_per_token": 5e-06,
         "litellm_provider": "sambanova",
@@ -34927,6 +34954,48 @@
         "supports_vision": true,
         "supports_web_search": true
     },
+    "xai/grok-4.3": {
+        "cache_read_input_token_cost": 2e-07,
+        "cache_read_input_token_cost_above_200k_tokens": 4e-07,
+        "input_cost_per_token": 1.25e-06,
+        "input_cost_per_token_above_200k_tokens": 2.5e-06,
+        "litellm_provider": "xai",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 1000000,
+        "max_tokens": 1000000,
+        "mode": "chat",
+        "output_cost_per_token": 2.5e-06,
+        "output_cost_per_token_above_200k_tokens": 5e-06,
+        "source": "https://docs.x.ai/docs/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
+    "xai/grok-4.3-latest": {
+        "cache_read_input_token_cost": 2e-07,
+        "cache_read_input_token_cost_above_200k_tokens": 4e-07,
+        "input_cost_per_token": 1.25e-06,
+        "input_cost_per_token_above_200k_tokens": 2.5e-06,
+        "litellm_provider": "xai",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 1000000,
+        "max_tokens": 1000000,
+        "mode": "chat",
+        "output_cost_per_token": 2.5e-06,
+        "output_cost_per_token_above_200k_tokens": 5e-06,
+        "source": "https://docs.x.ai/docs/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_web_search": true
+    },
     "xai/grok-beta": {
         "input_cost_per_token": 5e-06,
         "litellm_provider": "xai",

litellm/proxy/_experimental/mcp_server/auth/token_exchange.py

@@ -0,0 +1,196 @@
+"""
+OAuth 2.0 Token Exchange (RFC 8693) handler for MCP servers.
+
+Exchanges a user's incoming JWT (subject_token) for a scoped access token
+at an IDP's token exchange endpoint.  The exchanged token is then used to
+authenticate requests to the upstream MCP server.
+
+See: https://datatracker.ietf.org/doc/html/rfc8693
+"""
+
+import asyncio
+import hashlib
+import weakref
+from typing import TYPE_CHECKING, Dict, Tuple
+
+import httpx
+
+from litellm._logging import verbose_logger
+from litellm.caching.in_memory_cache import InMemoryCache
+from litellm.constants import (
+    MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL,
+    MCP_OAUTH2_TOKEN_CACHE_MIN_TTL,
+    MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS,
+    MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE,
+)
+from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
+from litellm.types.llms.custom_http import httpxSpecialProvider
+
+if TYPE_CHECKING:
+    from litellm.types.mcp_server.mcp_server_manager import MCPServer
+
+# RFC 8693 grant type constant
+TOKEN_EXCHANGE_GRANT_TYPE = "urn:ietf:params:oauth:grant-type:token-exchange"
+
+DEFAULT_SUBJECT_TOKEN_TYPE = "urn:ietf:params:oauth:token-type:access_token"
+
+
+class TokenExchangeHandler:
+    """Handles OAuth 2.0 Token Exchange (RFC 8693) for MCP servers.
+
+    Caches exchanged tokens keyed by ``hash(subject_token + server_id)`` so
+    repeated calls with the same user token skip the IDP round-trip.
+    """
+
+    def __init__(self) -> None:
+        self._cache = InMemoryCache(
+            max_size_in_memory=MCP_TOKEN_EXCHANGE_CACHE_MAX_SIZE,
+            default_ttl=MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL,
+        )
+        # WeakValueDictionary so locks are GC'd once no coroutine holds a reference,
+        # preventing unbounded growth with many rotating user tokens.
+        self._locks: weakref.WeakValueDictionary[str, asyncio.Lock] = (
+            weakref.WeakValueDictionary()
+        )
+
+    def _get_lock(self, cache_key: str) -> asyncio.Lock:
+        lock = self._locks.get(cache_key)
+        if lock is None:
+            lock = asyncio.Lock()
+            self._locks[cache_key] = lock
+        return lock
+
+    @staticmethod
+    def _cache_key(subject_token: str, server_id: str) -> str:
+        raw = f"{subject_token}:{server_id}"
+        return hashlib.sha256(raw.encode()).hexdigest()
+
+    async def exchange_token(
+        self,
+        subject_token: str,
+        server: "MCPServer",
+    ) -> str:
+        """Exchange *subject_token* for a scoped access token.
+
+        Returns the exchanged ``access_token`` string (suitable for a
+        ``Bearer`` header).
+
+        Raises ``ValueError`` on configuration or IDP errors.
+        """
+        cache_key = self._cache_key(subject_token, server.server_id)
+
+        # Fast path
+        cached = self._cache.get_cache(cache_key)
+        if cached is not None:
+            return cached
+
+        # Slow path — one exchange at a time per (user, server) pair
+        async with self._get_lock(cache_key):
+            cached = self._cache.get_cache(cache_key)
+            if cached is not None:
+                return cached
+
+            token, ttl = await self._do_exchange(subject_token, server)
+            self._cache.set_cache(cache_key, token, ttl=ttl)
+            return token
+
+    async def _do_exchange(
+        self,
+        subject_token: str,
+        server: "MCPServer",
+    ) -> Tuple[str, int]:
+        """POST to the token exchange endpoint with RFC 8693 parameters.
+
+        Returns ``(access_token, ttl_seconds)``.
+        """
+        endpoint = server.token_exchange_endpoint or server.token_url
+        if not endpoint:
+            raise ValueError(
+                f"MCP server '{server.server_id}' has auth_type=oauth2_token_exchange "
+                f"but no token_exchange_endpoint or token_url configured"
+            )
+        if not server.client_id or not server.client_secret:
+            raise ValueError(
+                f"MCP server '{server.server_id}' has auth_type=oauth2_token_exchange "
+                f"but missing client_id or client_secret"
+            )
+
+        data: Dict[str, str] = {
+            "grant_type": TOKEN_EXCHANGE_GRANT_TYPE,
+            "subject_token": subject_token,
+            "subject_token_type": server.subject_token_type
+            or DEFAULT_SUBJECT_TOKEN_TYPE,
+            "client_id": server.client_id,
+            "client_secret": server.client_secret,
+        }
+        if server.audience:
+            data["audience"] = server.audience
+        if server.scopes:
+            data["scope"] = " ".join(server.scopes)
+
+        verbose_logger.debug(
+            "Exchanging token for MCP server %s at %s (audience=%s)",
+            server.server_id,
+            endpoint,
+            server.audience,
+        )
+
+        client = get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP)
+        try:
+            response = await client.post(endpoint, data=data)
+            response.raise_for_status()
+        except httpx.HTTPStatusError as exc:
+            verbose_logger.debug(
+                "Token exchange IDP error for MCP server %s (status %d)",
+                server.server_id,
+                exc.response.status_code,
+            )
+            raise ValueError(
+                f"Token exchange for MCP server '{server.server_id}' "
+                f"failed with status {exc.response.status_code}"
+            ) from exc
+
+        body = response.json()
+        if not isinstance(body, dict):
+            raise ValueError(
+                f"Token exchange response for MCP server '{server.server_id}' "
+                f"returned non-object JSON (got {type(body).__name__})"
+            )
+
+        access_token = body.get("access_token")
+        if not access_token:
+            raise ValueError(
+                f"Token exchange response for MCP server '{server.server_id}' "
+                f"missing 'access_token'"
+            )
+
+        raw_expires_in = body.get("expires_in")
+        try:
+            expires_in = (
+                int(raw_expires_in)
+                if raw_expires_in is not None
+                else MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL
+            )
+        except (TypeError, ValueError):
+            expires_in = MCP_OAUTH2_TOKEN_CACHE_DEFAULT_TTL
+
+        ttl = max(
+            expires_in - MCP_OAUTH2_TOKEN_EXPIRY_BUFFER_SECONDS,
+            MCP_OAUTH2_TOKEN_CACHE_MIN_TTL,
+        )
+
+        verbose_logger.info(
+            "Token exchange succeeded for MCP server %s (expires in %ds)",
+            server.server_id,
+            expires_in,
+        )
+        return access_token, ttl
+
+    def invalidate(self, subject_token: str, server_id: str) -> None:
+        """Remove a cached exchanged token (e.g. after a 401)."""
+        cache_key = self._cache_key(subject_token, server_id)
+        self._cache.delete_cache(cache_key)
+
+
+# Module-level singleton
+mcp_token_exchange_handler = TokenExchangeHandler()

litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py

@@ -12,7 +12,8 @@ from litellm.llms.custom_httpx.http_handler import (
 )
 from litellm.proxy._experimental.mcp_server.oauth_utils import (
     TOKEN_NO_CACHE_HEADERS,
-    validate_loopback_redirect_uri,
+    get_request_base_url,
+    validate_trusted_redirect_uri,
 )
 from litellm.proxy.auth.ip_address_utils import IPAddressUtils
 from litellm.proxy.common_utils.encrypt_decrypt_utils import (
@@ -29,51 +30,6 @@ router = APIRouter(
 )
 
 
-def get_request_base_url(request: Request) -> str:
-    """
-    Get the base URL for the request, considering X-Forwarded-* headers.
-
-    X-Forwarded-Proto / X-Forwarded-Host / X-Forwarded-Port are only honoured
-    when the request comes from a configured trusted proxy
-    (``use_x_forwarded_for`` enabled AND caller in ``mcp_trusted_proxy_ranges``).
-    Otherwise the request's literal ``base_url`` is returned, so an
-    untrusted caller cannot poison OAuth-discovery / redirect_uri values
-    by injecting headers.
-
-    Args:
-        request: FastAPI Request object
-
-    Returns:
-        The reconstructed base URL (e.g., "https://proxy.example.com")
-    """
-    base_url = str(request.base_url).rstrip("/")
-    parsed = urlparse(base_url)
-
-    if not IPAddressUtils.is_request_from_trusted_proxy(request):
-        return base_url
-
-    x_forwarded_proto = request.headers.get("X-Forwarded-Proto")
-    x_forwarded_host = request.headers.get("X-Forwarded-Host")
-    x_forwarded_port = request.headers.get("X-Forwarded-Port")
-
-    scheme = x_forwarded_proto if x_forwarded_proto else parsed.scheme
-
-    if x_forwarded_host:
-        # X-Forwarded-Host may already include port (e.g., "example.com:8080")
-        if ":" in x_forwarded_host and not x_forwarded_host.startswith("["):
-            netloc = x_forwarded_host
-        elif x_forwarded_port:
-            netloc = f"{x_forwarded_host}:{x_forwarded_port}"
-        else:
-            netloc = x_forwarded_host
-    else:
-        netloc = parsed.netloc
-        if x_forwarded_port and ":" not in netloc:
-            netloc = f"{netloc}:{x_forwarded_port}"
-
-    return urlunparse((scheme, netloc, parsed.path, "", "", ""))
-
-
 def encode_state_with_base_url(
     base_url: str,
     original_state: str,
@@ -127,12 +83,14 @@ def decode_state_hash(encrypted_state: str) -> dict:
     return state_data
 
 
-def _get_validated_client_redirect_uri(state_data: Dict[str, Any]) -> str:
-    """Return a loopback client redirect URI from OAuth state."""
+def _get_validated_client_redirect_uri(
+    request: Request, state_data: Dict[str, Any]
+) -> str:
+    """Return a trusted (same-origin or loopback) client redirect URI from OAuth state."""
     redirect_uri = state_data.get("client_redirect_uri") or state_data.get("base_url")
     if not redirect_uri or not isinstance(redirect_uri, str):
         raise HTTPException(status_code=400, detail="Invalid redirect URI")
-    validate_loopback_redirect_uri(redirect_uri)
+    validate_trusted_redirect_uri(request, redirect_uri)
     return redirect_uri
 
 
@@ -338,12 +296,12 @@ async def authorize_with_server(
             status_code=400, detail="MCP server authorization url is not set"
         )
 
-    # Loopback-only redirect_uri. The URI is encrypted into the OAuth
-    # state and decoded on /callback to redirect the user back; a non-
-    # loopback URI would be an open-redirect + code-theft primitive
-    # (VERIA-57 root cause B). MCP clients are native apps — loopback is
-    # the spec-compliant callback pattern.
-    validate_loopback_redirect_uri(redirect_uri)
+    # Loopback OR same-origin redirect_uri. The URI is encrypted into the
+    # OAuth state and decoded on /callback to redirect the user back;
+    # restricting to trusted origins blocks the open-redirect +
+    # code-theft primitive (VERIA-57 root cause B). Loopback supports
+    # native MCP clients; same-origin supports the proxy's own UI callback.
+    validate_trusted_redirect_uri(request, redirect_uri)
     parsed = urlparse(redirect_uri)
     base_url = urlunparse(parsed._replace(query=""))
     request_base_url = get_request_base_url(request)
@@ -660,17 +618,18 @@ async def token_endpoint(
 
 
 @router.get("/callback")
-async def callback(code: str, state: str):
+async def callback(request: Request, code: str, state: str):
     try:
         state_data = decode_state_hash(state)
         original_state = state_data["original_state"]
 
-        # Re-validate loopback at the sink. /authorize rejects non-loopback
+        # Re-validate at the sink. /authorize rejects untrusted
         # redirect_uri before encoding into state, but encrypted states
         # minted before that check was added have no expiry and remain
-        # valid indefinitely. Validating here blocks the open-redirect +
-        # code-theft primitive even for pre-fix states.
-        redirect_uri = _get_validated_client_redirect_uri(state_data)
+        # valid indefinitely. Validating here (same-origin OR loopback)
+        # blocks the open-redirect + code-theft primitive even for pre-fix
+        # states while allowing the UI's same-origin callback to work.
+        redirect_uri = _get_validated_client_redirect_uri(request, state_data)
 
         params = {"code": code, "state": original_state}
         complete_returned_url = _append_query_params(redirect_uri, params)

litellm/proxy/_experimental/mcp_server/mcp_server_manager.py

@@ -411,6 +411,15 @@ class MCPServerManager:
                 aws_role_name=server_config.get("aws_role_name", None),
                 aws_session_name=server_config.get("aws_session_name", None),
                 instructions=server_config.get("instructions", None),
+                # Token Exchange (OBO) fields
+                token_exchange_endpoint=server_config.get(
+                    "token_exchange_endpoint", None
+                ),
+                audience=server_config.get("audience", None),
+                subject_token_type=server_config.get(
+                    "subject_token_type",
+                    "urn:ietf:params:oauth:token-type:access_token",
+                ),
             )
             self._assign_unique_short_prefix(new_server)
             self.config_mcp_servers[server_id] = new_server
@@ -497,7 +506,8 @@ class MCPServerManager:
             # Add any static headers from server config.
             #
             # Note: `extra_headers` on MCPServer is a List[str] of header names to forward
-            # from the client request (not available in this OpenAPI tool generation step).
+            # from each client MCP request; values are applied at call time via
+            # `_request_extra_headers` in server.py (not baked in here).
             # `static_headers` is a dict of concrete headers to always send.
             headers = (
                 merge_mcp_headers(
@@ -765,10 +775,23 @@ class MCPServerManager:
             aws_role_name=aws_creds.get("aws_role_name"),
             aws_session_name=aws_creds.get("aws_session_name"),
             instructions=mcp_server.instructions,
+            # Token Exchange (OBO) fields — read from credentials JSON blob
+            token_exchange_endpoint=(
+                credentials_dict.get("token_exchange_endpoint")
+                if credentials_dict
+                else None
+            ),
+            audience=(credentials_dict.get("audience") if credentials_dict else None),
+            subject_token_type=(
+                credentials_dict.get("subject_token_type") if credentials_dict else None
+            )
+            or "urn:ietf:params:oauth:token-type:access_token",
         )
         return new_server
 
-    async def _maybe_register_openapi_tools(self, server: MCPServer):
+    async def _maybe_register_openapi_tools(
+        self, server: MCPServer, *, initialize_mapping: bool = True
+    ):
         """Register OpenAPI tools if the server has a spec_path configured."""
         if server.spec_path:
             verbose_logger.info(
@@ -779,7 +802,8 @@ class MCPServerManager:
                 server=server,
                 base_url=server.url or "",
             )
-            self.initialize_tool_name_to_mcp_server_name_mapping()
+            if initialize_mapping:
+                self.initialize_tool_name_to_mcp_server_name_mapping()
 
     async def add_server(self, mcp_server: LiteLLM_MCPServerTable):
         try:
@@ -1136,6 +1160,29 @@ class MCPServerManager:
     #########################################################
     # Methods that call the upstream MCP servers
     #########################################################
+    @staticmethod
+    def _extract_bearer_token(
+        oauth2_headers: Optional[Dict[str, str]],
+        raw_headers: Optional[Dict[str, str]],
+    ) -> Optional[str]:
+        """Extract the bare Bearer token from oauth2_headers or raw_headers.
+
+        Returns the token string without the ``Bearer `` prefix, or ``None``
+        if no Authorization header is found.
+        """
+        auth_value: Optional[str] = None
+        if oauth2_headers and "Authorization" in oauth2_headers:
+            auth_value = oauth2_headers["Authorization"]
+        elif raw_headers:
+            # raw_headers may have lowercase keys depending on the ASGI server
+            normalized = {k.lower(): v for k, v in raw_headers.items()}
+            auth_value = normalized.get("authorization")
+        if auth_value:
+            if auth_value.startswith("Bearer "):
+                return auth_value[len("Bearer ") :]
+            return auth_value
+        return None
+
     def _build_stdio_env(
         self,
         server: MCPServer,
@@ -1169,25 +1216,30 @@ class MCPServerManager:
         mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None,
         extra_headers: Optional[Dict[str, str]] = None,
         stdio_env: Optional[Dict[str, str]] = None,
+        subject_token: Optional[str] = None,
     ) -> MCPClient:
         """
         Create an MCPClient instance for the given server.
 
         Auth resolution (single place for all auth logic):
         1. ``mcp_auth_header`` — per-request/per-user override
-        2. OAuth2 client_credentials token — auto-fetched and cached
-        3. ``server.authentication_token`` — static token from config/DB
+        2. OAuth2 Token Exchange (OBO) — exchange user token for scoped token
+        3. OAuth2 client_credentials token — auto-fetched and cached
+        4. ``server.authentication_token`` — static token from config/DB
 
         Args:
             server: The server configuration.
             mcp_auth_header: Optional per-request auth override.
             extra_headers: Additional headers to forward.
             stdio_env: Environment variables for stdio transport.
+            subject_token: Optional user JWT for token exchange (OBO) flow.
 
         Returns:
             Configured MCP client instance.
         """
-        auth_value = await resolve_mcp_auth(server, mcp_auth_header)
+        auth_value = await resolve_mcp_auth(
+            server, mcp_auth_header, subject_token=subject_token
+        )
 
         transport = server.transport or MCPTransport.sse
 
@@ -1978,7 +2030,11 @@ class MCPServerManager:
 
     _SHORT_PREFIX_MAX_REHASH_ATTEMPTS = 1024
 
-    def _assign_unique_short_prefix(self, server: MCPServer) -> None:
+    def _assign_unique_short_prefix(
+        self,
+        server: MCPServer,
+        registry: Optional[Dict[str, MCPServer]] = None,
+    ) -> None:
         """Resolve and cache a collision-free short tool prefix on ``server``.
 
         Called at registration time for every MCP server entering the
@@ -2002,7 +2058,8 @@ class MCPServerManager:
             return
 
         used: Dict[str, str] = {}
-        for other in self.get_registry().values():
+        registry_for_collision_check = registry or self.get_registry()
+        for other in registry_for_collision_check.values():
             if other.server_id == server.server_id:
                 continue
             if other.short_prefix:
@@ -2534,9 +2591,12 @@ class MCPServerManager:
         if server_auth_header is None:
             server_auth_header = mcp_auth_header
 
-        # oauth2 headers
+        # Extract subject token for OAuth2 Token Exchange (OBO) flow
+        subject_token: Optional[str] = None
         extra_headers: Optional[Dict[str, str]] = None
-        if mcp_server.auth_type == MCPAuth.oauth2:
+        if mcp_server.auth_type == MCPAuth.oauth2_token_exchange:
+            subject_token = self._extract_bearer_token(oauth2_headers, raw_headers)
+        elif mcp_server.auth_type == MCPAuth.oauth2:
             if mcp_server.has_client_credentials:
                 # For M2M OAuth servers, Authorization must come from token fetch.
                 extra_headers = None
@@ -2604,6 +2664,7 @@ class MCPServerManager:
             mcp_auth_header=server_auth_header,
             extra_headers=extra_headers,
             stdio_env=stdio_env,
+            subject_token=subject_token,
         )
 
         call_tool_params = MCPCallToolRequestParams(
@@ -2916,46 +2977,72 @@ class MCPServerManager:
         # against the *full* set so dedup is deterministic regardless of
         # iteration order.
         for server in db_mcp_servers:
-            existing_server = previous_registry.get(server.server_id)
+            try:
+                existing_server = previous_registry.get(server.server_id)
 
-            if (
-                existing_server is not None
-                and existing_server.updated_at is not None
-                and server.updated_at is not None
-                and existing_server.updated_at == server.updated_at
-            ):
-                # Re-use existing server instance to avoid re-running build_mcp_server_from_table()
-                # which can perform network discovery for OAuth2 servers.
-                new_registry[server.server_id] = existing_server
-                continue
+                if (
+                    existing_server is not None
+                    and existing_server.updated_at is not None
+                    and server.updated_at is not None
+                    and existing_server.updated_at == server.updated_at
+                ):
+                    # Re-use existing server instance to avoid re-running build_mcp_server_from_table()
+                    # which can perform network discovery for OAuth2 servers.
+                    new_registry[server.server_id] = existing_server
+                    continue
 
-            _warn_on_server_name_fields(
-                server_id=server.server_id,
-                alias=getattr(server, "alias", None),
-                server_name=getattr(server, "server_name", None),
-            )
-            verbose_logger.debug(
-                f"Building server from DB: {server.server_id} ({server.server_name})"
-            )
-            new_server = await self.build_mcp_server_from_table(server)
-            # Carry the cached short_prefix from the previous registry entry
-            # (if any) so the prefix is stable across reloads.
-            if existing_server is not None and existing_server.short_prefix:
-                new_server.short_prefix = existing_server.short_prefix
-            new_registry[server.server_id] = new_server
+                _warn_on_server_name_fields(
+                    server_id=server.server_id,
+                    alias=getattr(server, "alias", None),
+                    server_name=getattr(server, "server_name", None),
+                )
+                verbose_logger.debug(
+                    f"Building server from DB: {server.server_id} ({server.server_name})"
+                )
+                new_server = await self.build_mcp_server_from_table(server)
+                # Carry the cached short_prefix from the previous registry entry
+                # (if any) so the prefix is stable across reloads.
+                if existing_server is not None and existing_server.short_prefix:
+                    new_server.short_prefix = existing_server.short_prefix
+                new_registry[server.server_id] = new_server
+            except Exception as e:
+                verbose_logger.exception(
+                    "Skipping MCP server %s (%s) during DB reload: %s",
+                    server.server_id,
+                    getattr(server, "alias", None),
+                    e,
+                )
 
-        # Swap in the new registry first so _assign_unique_short_prefix
-        # sees the complete set when checking for collisions.
-        self.registry = new_registry
-        for new_server in new_registry.values():
-            self._assign_unique_short_prefix(new_server)
-            # Register OpenAPI tools *after* the final short prefix is assigned
-            # so the tools are stored in the global registry under the same
-            # prefix that lookups will use.
-            await self._maybe_register_openapi_tools(new_server)
+        # Assign short prefixes against the full candidate set without
+        # publishing the staged registry to concurrent callers.
+        registered_registry: Dict[str, MCPServer] = {}
+        registered_openapi_tools = False
+        for server_id, new_server in new_registry.items():
+            try:
+                self._assign_unique_short_prefix(new_server, registry=new_registry)
+                # Register OpenAPI tools *after* the final short prefix is assigned
+                # so the tools are stored in the global registry under the same
+                # prefix that lookups will use.
+                await self._maybe_register_openapi_tools(
+                    new_server, initialize_mapping=False
+                )
+                registered_registry[server_id] = new_server
+                if new_server.spec_path:
+                    registered_openapi_tools = True
+            except Exception as e:
+                verbose_logger.exception(
+                    "Skipping MCP server %s (%s) during DB reload: %s",
+                    new_server.server_id,
+                    getattr(new_server, "alias", None),
+                    e,
+                )
+
+        self.registry = registered_registry
+        if registered_openapi_tools:
+            self.initialize_tool_name_to_mcp_server_name_mapping()
 
         verbose_logger.debug(
-            "MCP registry refreshed (%s servers in registry)", len(new_registry)
+            "MCP registry refreshed (%s servers in registry)", len(registered_registry)
         )
 
     def get_mcp_servers_from_ids(self, server_ids: List[str]) -> List[MCPServer]:

litellm/proxy/_experimental/mcp_server/oauth2_token_cache.py

@@ -26,6 +26,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import (
     decrypt_value_helper,
     encrypt_value_helper,
 )
+from litellm.proxy._experimental.mcp_server.auth import token_exchange
 from litellm.types.llms.custom_http import httpxSpecialProvider
 
 if TYPE_CHECKING:
@@ -50,12 +51,23 @@ class MCPOAuth2TokenCache(InMemoryCache):
     def _get_lock(self, server_id: str) -> asyncio.Lock:
         return self._locks.setdefault(server_id, asyncio.Lock())
 
-    async def async_get_token(self, server: "MCPServer") -> Optional[str]:
+    @staticmethod
+    def _has_client_credentials_config(server: "MCPServer") -> bool:
+        return bool(server.client_id and server.client_secret and server.token_url)
+
+    async def async_get_token(
+        self,
+        server: "MCPServer",
+        *,
+        require_client_credentials_flow: bool = True,
+    ) -> Optional[str]:
         """Return a valid access token, fetching or refreshing as needed.
 
         Returns ``None`` when the server lacks client credentials config.
         """
-        if not server.has_client_credentials:
+        if require_client_credentials_flow and not server.has_client_credentials:
+            return None
+        if not self._has_client_credentials_config(server):
             return None
 
         server_id = server.server_id
@@ -263,16 +275,38 @@ mcp_per_user_token_cache = MCPPerUserTokenCache()
 async def resolve_mcp_auth(
     server: "MCPServer",
     mcp_auth_header: Optional[Union[str, Dict[str, str]]] = None,
+    subject_token: Optional[str] = None,
 ) -> Optional[Union[str, Dict[str, str]]]:
     """Resolve the auth value for an MCP server.
 
     Priority:
     1. ``mcp_auth_header`` — per-request/per-user override
-    2. OAuth2 client_credentials token — auto-fetched and cached
-    3. ``server.authentication_token`` — static token from config/DB
+    2. OAuth2 Token Exchange (OBO / RFC 8693) — exchange user token for scoped token
+    3. OAuth2 client_credentials token — auto-fetched and cached
+    4. ``server.authentication_token`` — static token from config/DB
     """
     if mcp_auth_header:
         return mcp_auth_header
+    if server.has_token_exchange_config:
+        if subject_token:
+            return await token_exchange.mcp_token_exchange_handler.exchange_token(
+                subject_token, server
+            )
+        # No subject_token — fall back to client_credentials using the same client
+        # credentials and token_url so M2M scenarios still work.
+        if server.client_id and server.client_secret and server.token_url:
+            return await mcp_oauth2_token_cache.async_get_token(
+                server,
+                require_client_credentials_flow=False,
+            )
+        # OBO configured but no subject_token and missing client credentials — warn
+        # rather than silently proceeding unauthenticated.
+        verbose_logger.warning(
+            "MCP server '%s' is configured for token exchange (OBO) but no subject_token "
+            "was provided and client credentials (client_id/client_secret/token_url) are "
+            "incomplete. The request will proceed without authentication.",
+            server.server_id,
+        )
     if server.has_client_credentials:
         return await mcp_oauth2_token_cache.async_get_token(server)
     return server.authentication_token

litellm/proxy/_experimental/mcp_server/oauth_utils.py

@@ -2,15 +2,63 @@
 (BYOK + discoverable / pass-through OAuth proxy)."""
 
 from ipaddress import ip_address
-from urllib.parse import urlparse
+from urllib.parse import urlparse, urlunparse
 
-from fastapi import HTTPException
+from fastapi import HTTPException, Request
+
+from litellm._logging import verbose_logger
+from litellm.proxy.auth.ip_address_utils import IPAddressUtils
 
 # RFC 6749 §5.1 / OAuth 2.1 draft-15 §4.1.3: token-endpoint responses
 # must not be cached — both success and error bodies may reveal secrets.
 TOKEN_NO_CACHE_HEADERS = {"Cache-Control": "no-store", "Pragma": "no-cache"}
 
 
+def get_request_base_url(request: Request) -> str:
+    """
+    Get the base URL for the request, considering X-Forwarded-* headers.
+
+    X-Forwarded-Proto / X-Forwarded-Host / X-Forwarded-Port are only honoured
+    when the request comes from a configured trusted proxy
+    (``use_x_forwarded_for`` enabled AND caller in ``mcp_trusted_proxy_ranges``).
+    Otherwise the request's literal ``base_url`` is returned, so an
+    untrusted caller cannot poison OAuth-discovery / redirect_uri values
+    by injecting headers.
+
+    Args:
+        request: FastAPI Request object
+
+    Returns:
+        The reconstructed base URL (e.g., "https://proxy.example.com")
+    """
+    base_url = str(request.base_url).rstrip("/")
+    parsed = urlparse(base_url)
+
+    if not IPAddressUtils.is_request_from_trusted_proxy(request):
+        return base_url
+
+    x_forwarded_proto = request.headers.get("X-Forwarded-Proto")
+    x_forwarded_host = request.headers.get("X-Forwarded-Host")
+    x_forwarded_port = request.headers.get("X-Forwarded-Port")
+
+    scheme = x_forwarded_proto if x_forwarded_proto else parsed.scheme
+
+    if x_forwarded_host:
+        # X-Forwarded-Host may already include port (e.g., "example.com:8080")
+        if ":" in x_forwarded_host and not x_forwarded_host.startswith("["):
+            netloc = x_forwarded_host
+        elif x_forwarded_port:
+            netloc = f"{x_forwarded_host}:{x_forwarded_port}"
+        else:
+            netloc = x_forwarded_host
+    else:
+        netloc = parsed.netloc
+        if x_forwarded_port and ":" not in netloc:
+            netloc = f"{netloc}:{x_forwarded_port}"
+
+    return urlunparse((scheme, netloc, parsed.path, "", "", ""))
+
+
 def validate_loopback_redirect_uri(redirect_uri: str) -> None:
     """Require a loopback ``redirect_uri`` (OAuth 2.1 §4.1.2.1 + RFC 8252
     §7.3 native-app pattern). MCP clients are native apps that listen on
@@ -46,3 +94,60 @@ def validate_loopback_redirect_uri(redirect_uri: str) -> None:
         # don't let it bubble up as a 500.
         pass
     raise HTTPException(status_code=400, detail="invalid_request")
+
+
+def validate_trusted_redirect_uri(request: Request, redirect_uri: str) -> None:
+    """Accept same-origin (proxy's own origin) OR loopback ``redirect_uri``.
+
+    Same-origin is required for the LiteLLM UI's OAuth flow: the UI
+    redirects to ``<proxy>/ui/mcp/oauth/callback`` which is not loopback
+    but is on the proxy's own trusted HTTPS origin. An attacker cannot
+    host content on the proxy's own origin without already owning the
+    proxy, so the open-redirect / code-theft primitive that motivated
+    :func:`validate_loopback_redirect_uri` does not apply here.
+
+    Loopback continues to be accepted for native MCP clients (per
+    OAuth 2.1 §4.1.2.1 + RFC 8252 §7.3).
+
+    Use this in the discoverable OAuth proxy endpoints that serve both
+    native clients and the proxy's own UI. BYOK endpoints that only
+    support native clients should keep
+    :func:`validate_loopback_redirect_uri`.
+    """
+    try:
+        parsed = urlparse(redirect_uri)
+    except ValueError:
+        raise HTTPException(status_code=400, detail="invalid_request")
+    if parsed.scheme not in ("http", "https"):
+        raise HTTPException(status_code=400, detail="invalid_request")
+    if parsed.fragment:
+        raise HTTPException(status_code=400, detail="invalid_request")
+
+    # Same-origin: scheme + netloc (host[:port]) must match the proxy's
+    # own base URL at this request (honouring trusted X-Forwarded-*).
+    try:
+        proxy_base = urlparse(get_request_base_url(request))
+        if (
+            parsed.netloc
+            and parsed.scheme == proxy_base.scheme
+            and parsed.netloc.lower() == proxy_base.netloc.lower()
+        ):
+            return
+    except Exception as exc:
+        # If we can't determine the proxy's origin, fall through to
+        # loopback. Log so the failure is diagnosable in production.
+        verbose_logger.warning(
+            "validate_trusted_redirect_uri: could not determine proxy origin, "
+            "falling back to loopback-only check. error=%s",
+            exc,
+        )
+
+    host = (parsed.hostname or "").lower()
+    if host == "localhost":
+        return
+    try:
+        if ip_address(host).is_loopback:
+            return
+    except ValueError:
+        pass
+    raise HTTPException(status_code=400, detail="invalid_request")

litellm/proxy/_experimental/mcp_server/openapi_to_mcp_generator.py

@@ -55,6 +55,13 @@ _request_auth_header: contextvars.ContextVar[Optional[str]] = contextvars.Contex
     "_request_auth_header", default=None
 )
 
+# Per-request extra headers forwarded from the client request.
+# Populated from MCPServer.extra_headers names matched against raw request
+# headers in server.py before dispatching to a local/OpenAPI tool handler.
+_request_extra_headers: contextvars.ContextVar[Optional[Dict[str, str]]] = (
+    contextvars.ContextVar("_request_extra_headers", default=None)
+)
+
 
 def _sanitize_path_parameter_value(param_value: Any, param_name: str) -> str:
     """Ensure path params cannot introduce directory traversal."""
@@ -297,6 +304,46 @@ def build_input_schema(operation: Dict[str, Any]) -> Dict[str, Any]:
     }
 
 
+def _merge_openapi_tool_request_headers(
+    static_headers: Dict[str, str]
+) -> Dict[str, str]:
+    """Merge static closure headers with per-request ContextVar overrides.
+
+    Precedence (highest to lowest):
+        1. ``_request_auth_header`` — BYOK override of ``Authorization``
+        2. ``static_headers`` — operator-configured headers baked into the
+           tool closure at registration time
+        3. ``_request_extra_headers`` — per-request headers forwarded from
+           the MCP caller (allowlisted by ``MCPServer.extra_headers``)
+
+    This matches the existing MCP invariant in
+    :func:`litellm.proxy._experimental.mcp_server.utils.merge_mcp_headers`
+    and the managed MCP path, where ``static_headers`` always wins over
+    caller-forwarded headers. Keeping the same precedence here prevents an
+    authenticated caller from overriding an operator-configured value
+    (e.g. a tenant id or upstream API key) by sending the same header name.
+
+    Header names are compared case-insensitively so different casing cannot
+    bypass the precedence rules.
+    """
+    request_extra = _request_extra_headers.get() or {}
+    static = static_headers or {}
+
+    static_lower_names = {k.lower() for k in static}
+    effective_headers: Dict[str, str] = {
+        k: v for k, v in request_extra.items() if k.lower() not in static_lower_names
+    }
+    effective_headers.update(static)
+
+    override_auth = _request_auth_header.get()
+    if override_auth:
+        for existing in [k for k in effective_headers if k.lower() == "authorization"]:
+            del effective_headers[existing]
+        effective_headers["Authorization"] = override_auth
+
+    return effective_headers
+
+
 def create_tool_function(
     path: str,
     method: str,
@@ -334,14 +381,7 @@ def create_tool_function(
         The function safely handles parameter names that aren't valid Python identifiers
         by using **kwargs instead of named parameters.
         """
-        # Allow per-request auth override (e.g. BYOK credential set via ContextVar).
-        # The ContextVar holds the full Authorization header value, including the
-        # correct prefix (Bearer / ApiKey / Basic) formatted by the caller in
-        # server.py based on the server's configured auth_type.
-        effective_headers = dict(headers)
-        override_auth = _request_auth_header.get()
-        if override_auth:
-            effective_headers["Authorization"] = override_auth
+        effective_headers = _merge_openapi_tool_request_headers(headers)
 
         # Build URL from base_url and path
         url = base_url + path

litellm/proxy/_experimental/mcp_server/server.py

@@ -158,6 +158,7 @@ if MCP_AVAILABLE:
     )
     from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import (
         _request_auth_header,
+        _request_extra_headers,
     )
     from litellm.proxy._experimental.mcp_server.sse_transport import SseServerTransport
     from litellm.proxy._experimental.mcp_server.tool_registry import (
@@ -2195,11 +2196,40 @@ if MCP_AVAILABLE:
                     auth_header_value = f"Basic {mcp_auth_header}"
                 else:
                     auth_header_value = f"Bearer {mcp_auth_header}"
+
+            # Forward named client headers to OpenAPI tool upstream requests.
+            # MCPServer.extra_headers lists header names to copy from raw_headers.
+            # OAuth2 M2M: never take Authorization from the caller (matches
+            # _prepare_mcp_server_headers for managed MCP).
+            forwarded_headers: Optional[Dict[str, str]] = None
+            if mcp_server and mcp_server.extra_headers and raw_headers:
+                normalized_raw = {
+                    str(k).lower(): v
+                    for k, v in raw_headers.items()
+                    if isinstance(k, str)
+                }
+                skip_caller_authorization = bool(mcp_server.has_client_credentials)
+                for header_name in mcp_server.extra_headers:
+                    if not isinstance(header_name, str):
+                        continue
+                    if (
+                        skip_caller_authorization
+                        and header_name.lower() == "authorization"
+                    ):
+                        continue
+                    value = normalized_raw.get(header_name.lower())
+                    if value is not None:
+                        if forwarded_headers is None:
+                            forwarded_headers = {}
+                        forwarded_headers[header_name] = value
+
             _auth_token = _request_auth_header.set(auth_header_value)
+            _extra_token = _request_extra_headers.set(forwarded_headers)
             try:
                 local_content = await _handle_local_mcp_tool(name, arguments)
             finally:
                 _request_auth_header.reset(_auth_token)
+                _request_extra_headers.reset(_extra_token)
             response = CallToolResult(content=cast(Any, local_content), isError=False)
 
         # Try managed MCP server tool (pass the full prefixed name)

litellm/proxy/_types.py

@@ -239,6 +239,7 @@ class KeyManagementRoutes(str, enum.Enum):
     KEY_BLOCK = "/key/block"
     KEY_UNBLOCK = "/key/unblock"
     KEY_BULK_UPDATE = "/key/bulk_update"
+    TEAM_KEY_BULK_UPDATE = "/team/key/bulk_update"
     KEY_RESET_SPEND = "/key/{key_id}/reset_spend"
 
     # info and health routes
@@ -353,8 +354,10 @@ class LiteLLMRoutes(enum.Enum):
         # realtime
         "/realtime",
         "/v1/realtime",
+        "/openai/v1/realtime",
         "/realtime?{model}",
         "/v1/realtime?{model}",
+        "/openai/v1/realtime?{model}",
         # responses API
         "/responses",
         "/v1/responses",
@@ -538,6 +541,7 @@ class LiteLLMRoutes(enum.Enum):
         KeyManagementRoutes.KEY_BLOCK.value,
         KeyManagementRoutes.KEY_UNBLOCK.value,
         KeyManagementRoutes.KEY_BULK_UPDATE.value,
+        KeyManagementRoutes.TEAM_KEY_BULK_UPDATE.value,
         KeyManagementRoutes.TEAM_DAILY_ACTIVITY.value,
         KeyManagementRoutes.SPEND_LOGS.value,
         KeyManagementRoutes.KEY_RESET_SPEND.value,
@@ -656,6 +660,13 @@ class LiteLLMRoutes(enum.Enum):
         "/health/services",
     ] + info_routes
 
+    # Stateless validators on caller-supplied log data; source logs are
+    # already accessible via spend_tracking_routes, so no scope expansion.
+    compliance_check_routes = [
+        "/compliance/eu-ai-act",
+        "/compliance/gdpr",
+    ]
+
     # Routes in `global_spend_tracking_routes` return proxy-wide spend across
     # every team, customer, and api_key. They are intentionally NOT included
     # here — non-admin roles must not see other tenants' spend. Admin roles go
@@ -679,14 +690,19 @@ class LiteLLMRoutes(enum.Enum):
         ]
         + spend_tracking_routes
         + key_management_routes
+        + compliance_check_routes
     )
 
-    internal_user_view_only_routes = spend_tracking_routes + [
-        # Tag usage endpoints scope internal viewers to tags produced by
-        # their own keys in tag_management_endpoints.py.
-        "/tag/daily/activity",
-        "/tag/list",
-    ]
+    internal_user_view_only_routes = (
+        spend_tracking_routes
+        + compliance_check_routes
+        + [
+            # Tag usage endpoints scope internal viewers to tags produced by
+            # their own keys in tag_management_endpoints.py.
+            "/tag/daily/activity",
+            "/tag/list",
+        ]
+    )
 
     self_managed_routes = [
         "/team/member_add",
@@ -708,6 +724,8 @@ class LiteLLMRoutes(enum.Enum):
         # Project read routes - endpoint scopes results to caller's teams (non-admin)
         "/project/list",
         "/project/info",
+        # Endpoint enforces proxy-admin vs team-admin model access itself.
+        "/health/test_connection",
         # Invitation routes - org/team admins checked in endpoint via _user_has_admin_privileges
         "/invitation/new",
         "/invitation/delete",
@@ -3357,6 +3375,19 @@ class AllCallbacks(LiteLLMPydanticObjectBase):
         ],
     )
 
+    azure_sentinel: CallbackOnUI = CallbackOnUI(
+        litellm_callback_name="azure_sentinel",
+        ui_callback_name="Azure Sentinel",
+        litellm_callback_params=[
+            "AZURE_SENTINEL_DCR_IMMUTABLE_ID",
+            "AZURE_SENTINEL_ENDPOINT",
+            "AZURE_SENTINEL_TENANT_ID",
+            "AZURE_SENTINEL_CLIENT_ID",
+            "AZURE_SENTINEL_CLIENT_SECRET",
+            "AZURE_SENTINEL_STREAM_NAME",
+        ],
+    )
+
     openmeter: CallbackOnUI = CallbackOnUI(
         litellm_callback_name="openmeter",
         ui_callback_name="OpenMeter",
@@ -4330,10 +4361,16 @@ class JWTRoutingOverride(BaseModel):
 
     A rule matches when all provided selectors match token claims.
     If matched, request is routed to the configured auth path.
+
+    Wildcard selectors use shell-style patterns (* and ?) and are matched with
+    case-sensitive semantics; use the same casing your IdP emits in JWT claims.
+    Space-delimited tokenization applies only to the ``scope`` claim (OAuth/OIDC
+    scope strings), not to ``iss``, ``aud``, or ``client_id``.
     """
 
     iss: Union[str, List[str]]
     client_id: Optional[Union[str, List[str]]] = None
+    scope: Optional[Union[str, List[str]]] = None
     aud: Optional[Union[str, List[str]]] = None
     path: Literal["oauth2"] = "oauth2"
 

litellm/proxy/auth/auth_checks.py

@@ -2849,7 +2849,7 @@ def _can_object_call_model(
             object_type=object_type
         ),
         param="model",
-        code=status.HTTP_401_UNAUTHORIZED,
+        code=status.HTTP_403_FORBIDDEN,
     )
 
 
@@ -3082,7 +3082,7 @@ async def can_user_call_model(
             message=f"User not allowed to access model. No default model access, only team models allowed. Tried to access {model}",
             type=ProxyErrorTypes.key_model_access_denied,
             param="model",
-            code=status.HTTP_401_UNAUTHORIZED,
+            code=status.HTTP_403_FORBIDDEN,
         )
 
     return _can_object_call_model(
@@ -3516,7 +3516,6 @@ async def _check_team_member_budget(
     if (
         team_object is not None
         and team_object.team_id is not None
-        and user_object is not None
         and valid_token is not None
         and valid_token.user_id is not None
     ):
@@ -3619,13 +3618,14 @@ async def _check_team_member_model_access(
             llm_router=llm_router,
             models=member_allowed_models,
             object_type="team",
+            team_id=team_object.team_id,
         )
     except ProxyException:
         raise ProxyException(
             message=f"Team member not allowed to access model. User={valid_token.user_id}, Team={team_object.team_id}, Model={model}. Allowed member models = {member_allowed_models}",
             type=ProxyErrorTypes.team_model_access_denied,
             param="model",
-            code=status.HTTP_401_UNAUTHORIZED,
+            code=status.HTTP_403_FORBIDDEN,
         )
 
 

litellm/proxy/auth/auth_exception_handler.py

@@ -123,7 +123,7 @@ class UserAPIKeyAuthExceptionHandler:
                     message=e.message,
                     type=ProxyErrorTypes.budget_exceeded,
                     param=None,
-                    code=400,
+                    code=getattr(e, "status_code", status.HTTP_429_TOO_MANY_REQUESTS),
                 )
             if isinstance(e, HTTPException):
                 raise ProxyException(

litellm/proxy/auth/handle_jwt.py

@@ -224,6 +224,41 @@ class JWTHandler:
 
         return []
 
+    def get_all_jwt_team_ids(self, token: dict) -> List[str]:
+        """
+        Return team IDs from both the plural ``team_ids_jwt_field`` and the
+        singular ``team_id_jwt_field`` claim (string or list of strings), as a
+        deduplicated list preserving plural-first order.
+
+        Membership-reconciliation paths (SSO callback, JWT-bearer sync) need
+        to consider both claim shapes. Reading only the plural field — as
+        callers historically did — silently dropped users whose IdP populates
+        the singular field, which is what Okta and Auth0 default to when a
+        user has a single primary team.
+
+        This intentionally does NOT consult ``team_id_default``: that fallback
+        is a property of how the JWT-bearer auth flow resolves a single
+        request-bound team, not of the token's claims. Callers that want the
+        default-team behavior should still go through ``get_team_id``.
+        """
+        team_ids: List[str] = list(self.get_team_ids_from_jwt(token))
+        if self.litellm_jwtauth.team_id_jwt_field is not None:
+            singular = get_nested_value(
+                data=token,
+                key_path=self.litellm_jwtauth.team_id_jwt_field,
+                default=None,
+            )
+            if isinstance(singular, list):
+                for item in singular:
+                    if item is None:
+                        continue
+                    sid = str(item)
+                    if sid and sid not in team_ids:
+                        team_ids.append(sid)
+            elif singular and str(singular) not in team_ids:
+                team_ids.append(str(singular))
+        return team_ids
+
     def get_end_user_id(
         self, token: dict, default_value: Optional[str]
     ) -> Optional[str]:

litellm/proxy/auth/route_checks.py

@@ -50,6 +50,7 @@ _PROXY_ADMIN_VIEW_ONLY_BLOCKED_ROUTES = frozenset(
         KeyManagementRoutes.KEY_BLOCK.value,
         KeyManagementRoutes.KEY_UNBLOCK.value,
         KeyManagementRoutes.KEY_BULK_UPDATE.value,
+        KeyManagementRoutes.TEAM_KEY_BULK_UPDATE.value,
     ]
 )
 
@@ -671,6 +672,7 @@ class RouteChecks:
             "/key/service-account/generate",
             "/key/block",
             "/key/unblock",
+            "/team/key/bulk_update",
         ]
     )
 

litellm/proxy/auth/user_api_key_auth.py

@@ -8,6 +8,7 @@ Returns a UserAPIKeyAuth object if the API key is valid
 """
 
 import asyncio
+import fnmatch
 import re
 import secrets
 from datetime import datetime, timezone
@@ -183,22 +184,54 @@ def _get_bearer_token_or_received_api_key(api_key: str) -> str:
 
 
 def _routing_selector_matches_claim(
-    selector_value: Optional[Any], claim_value: Optional[Any]
+    selector_value: Optional[Any],
+    claim_value: Optional[Any],
+    *,
+    split_space_delimited: bool = False,
 ) -> bool:
     if selector_value is None:
         return True
 
-    selector_list = (
+    selector_list: List[str] = (
         [str(v) for v in selector_value]
         if isinstance(selector_value, list)
         else [str(selector_value)]
     )
 
+    if claim_value is None:
+        return False
+
     if isinstance(claim_value, list):
         claim_list = [str(v) for v in claim_value]
-        return any(v in claim_list for v in selector_list)
+    elif (
+        split_space_delimited
+        and isinstance(claim_value, str)
+        and " " in claim_value.strip()
+    ):
+        # OAuth/OIDC often sends scope as a single space-delimited string. Only split
+        # for the scope selector: iss/aud/client_id must stay exact full-string match
+        # on unverified claims (see routing override security review). The elif guard
+        # (`" " in claim_value.strip()`) ensures at least two non-empty tokens survive.
+        claim_list = [v for v in claim_value.strip().split(" ") if v]
+    else:
+        claim_list = [str(claim_value)]
 
-    return str(claim_value) in selector_list if claim_value is not None else False
+    def _selector_matches_claim(selector: str, claim: str) -> bool:
+        # NOTE: wildcard matching is case-sensitive (fnmatch.fnmatchcase).
+        if "*" in selector or "?" in selector:
+            # Without scope splitting, do not let `*` span whitespace: a malformed
+            # iss like "trusted.example.com evil.com" must not match "trusted.*".
+            # Scope uses split_space_delimited so each claim token is checked separately.
+            if not split_space_delimited and any(ch.isspace() for ch in claim):
+                return False
+            return fnmatch.fnmatchcase(claim, selector)
+        return selector == claim
+
+    return any(
+        _selector_matches_claim(selector=s, claim=c)
+        for s in selector_list
+        for c in claim_list
+    )
 
 
 def _matches_routing_override(
@@ -209,6 +242,11 @@ def _matches_routing_override(
         and _routing_selector_matches_claim(
             override.client_id, token_claims.get("client_id")
         )
+        and _routing_selector_matches_claim(
+            override.scope,
+            token_claims.get("scope"),
+            split_space_delimited=True,
+        )
         and _routing_selector_matches_claim(override.aud, token_claims.get("aud"))
     )
 
@@ -1107,7 +1145,7 @@ async def _user_api_key_auth_builder(  # noqa: PLR0915
                     raise ProxyException(
                         message=f"Authentication Error - Expired Key. Key Expiry time {expiry_time} and current time {current_time}",
                         type=ProxyErrorTypes.expired_key,
-                        code=400,
+                        code=status.HTTP_401_UNAUTHORIZED,
                         param=abbreviate_api_key(api_key=api_key),
                     )
             valid_token = update_valid_token_with_end_user_params(
@@ -1432,7 +1470,7 @@ async def _user_api_key_auth_builder(  # noqa: PLR0915
                     raise ProxyException(
                         message=f"Authentication Error - Expired Key. Key Expiry time {expiry_time} and current time {current_time}",
                         type=ProxyErrorTypes.expired_key,
-                        code=400,
+                        code=status.HTTP_401_UNAUTHORIZED,
                         param=abbreviate_api_key(api_key=api_key),
                     )
 
@@ -2417,7 +2455,7 @@ async def _run_post_custom_auth_checks(
             raise ProxyException(
                 message=f"Authentication Error - Expired Key. Key Expiry time {expiry_time} and current time {current_time}",
                 type=ProxyErrorTypes.expired_key,
-                code=400,
+                code=status.HTTP_401_UNAUTHORIZED,
                 param=(
                     abbreviate_api_key(api_key=valid_token.token)
                     if valid_token.token

litellm/proxy/common_request_processing.py

@@ -1512,7 +1512,7 @@ class ProxyBaseLLMRequestProcessing:
             status_code=result.status_code,
             headers=HttpPassThroughEndpointHelpers.get_response_headers(
                 headers=result.headers,
-                custom_headers=None,
+                custom_headers=dict(fastapi_response.headers),
             ),
         )
 

litellm/proxy/common_utils/html_forms/jwt_display_template.py

@@ -14,7 +14,7 @@ jwt_display_template = """
             padding: 20px;
             display: flex;
             justify-content: center;
-            align-items: center;
+            align-items: flex-start;
             min-height: 100vh;
             color: #333;
         }
@@ -27,18 +27,18 @@ jwt_display_template = """
             width: 800px;
             max-width: 100%;
         }
-        
+
         .logo-container {
             text-align: center;
             margin-bottom: 30px;
         }
-        
+
         .logo {
             font-size: 24px;
             font-weight: 600;
             color: #1e293b;
         }
-        
+
         h2 {
             margin: 0 0 10px;
             color: #1e293b;
@@ -46,7 +46,14 @@ jwt_display_template = """
             font-weight: 600;
             text-align: center;
         }
-        
+
+        h3 {
+            margin: 0 0 12px;
+            color: #1e293b;
+            font-size: 18px;
+            font-weight: 600;
+        }
+
         .subtitle {
             color: #64748b;
             margin: 0 0 20px;
@@ -58,15 +65,15 @@ jwt_display_template = """
             background-color: #f1f5f9;
             border-radius: 6px;
             padding: 20px;
-            margin-bottom: 30px;
+            margin-bottom: 20px;
             border-left: 4px solid #2563eb;
         }
-        
+
         .success-box {
             background-color: #f0fdf4;
             border-radius: 6px;
             padding: 20px;
-            margin-bottom: 30px;
+            margin-bottom: 20px;
             border-left: 4px solid #16a34a;
         }
 
@@ -78,7 +85,7 @@ jwt_display_template = """
             font-weight: 600;
             font-size: 16px;
         }
-        
+
         .success-header {
             display: flex;
             align-items: center;
@@ -87,46 +94,53 @@ jwt_display_template = """
             font-weight: 600;
             font-size: 16px;
         }
-        
+
         .info-header svg, .success-header svg {
             margin-right: 8px;
         }
-        
+
         .data-container {
             margin-top: 20px;
         }
-        
+
         .data-row {
             display: flex;
             border-bottom: 1px solid #e2e8f0;
             padding: 12px 0;
         }
-        
+
         .data-row:last-child {
             border-bottom: none;
         }
-        
+
         .data-label {
             font-weight: 500;
             color: #334155;
-            width: 180px;
+            width: 220px;
             flex-shrink: 0;
         }
-        
+
         .data-value {
             color: #475569;
             word-break: break-all;
         }
-        
+
+        .empty-note {
+            color: #64748b;
+            font-style: italic;
+            margin: 0;
+            font-size: 14px;
+        }
+
         .jwt-container {
             background-color: #f8fafc;
             border-radius: 6px;
             padding: 15px;
-            margin-top: 20px;
+            margin-top: 12px;
             overflow-x: auto;
             border: 1px solid #e2e8f0;
         }
-        
+
         .jwt-text {
             font-family: monospace;
             white-space: pre-wrap;
@@ -134,7 +148,7 @@ jwt_display_template = """
             margin: 0;
             color: #334155;
         }
-        
+
         .back-button {
             display: inline-block;
             background-color: #6466E9;
@@ -146,18 +160,18 @@ jwt_display_template = """
             margin-top: 20px;
             text-align: center;
         }
-        
+
         .back-button:hover {
             background-color: #4138C2;
             text-decoration: none;
         }
-        
+
         .buttons {
             display: flex;
             gap: 10px;
-            margin-top: 20px;
+            margin-top: 12px;
         }
-        
+
         .copy-button {
             background-color: #e2e8f0;
             color: #334155;
@@ -169,11 +183,11 @@ jwt_display_template = """
             display: flex;
             align-items: center;
         }
-        
+
         .copy-button:hover {
             background-color: #cbd5e1;
         }
-        
+
         .copy-button svg {
             margin-right: 6px;
         }
@@ -188,7 +202,7 @@ jwt_display_template = """
         </div>
         <h2>SSO Debug Information</h2>
         <p class="subtitle">Results from the SSO authentication process.</p>
-        
+
         <div class="success-box">
             <div class="success-header">
                 <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
@@ -199,11 +213,7 @@ jwt_display_template = """
             </div>
             <p>The SSO authentication completed successfully. Below is the information returned by the provider.</p>
         </div>
-        
-        <div class="data-container" id="userData">
-            <!-- Data will be inserted here by JavaScript -->
-        </div>
-        
+
         <div class="info-box">
             <div class="info-header">
                 <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
@@ -211,22 +221,62 @@ jwt_display_template = """
                     <line x1="12" y1="16" x2="12" y2="12"></line>
                     <line x1="12" y1="8" x2="12.01" y2="8"></line>
                 </svg>
-                JSON Representation
+                Parsed by Proxy
             </div>
+            <p class="empty-note">Fields the proxy extracted into its internal user model.</p>
+            <div class="data-container" id="parsedByProxy">
+                <!-- Populated by JavaScript -->
+            </div>
+        </div>
+
+        <div class="info-box">
+            <div class="info-header">
+                <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
+                    <circle cx="12" cy="12" r="10"></circle>
+                    <line x1="12" y1="16" x2="12" y2="12"></line>
+                    <line x1="12" y1="8" x2="12.01" y2="8"></line>
+                </svg>
+                Raw Claims (userinfo)
+            </div>
+            <p class="empty-note">Complete set of claims returned by the IdP's userinfo endpoint.</p>
             <div class="jwt-container">
-                <pre class="jwt-text" id="jsonData">Loading...</pre>
+                <pre class="jwt-text" id="rawClaims">Loading...</pre>
             </div>
             <div class="buttons">
-                <button class="copy-button" onclick="copyToClipboard('jsonData')">
+                <button class="copy-button" onclick="copyToClipboard('rawClaims')">
                     <svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
                         <rect x="9" y="9" width="13" height="13" rx="2" ry="2"></rect>
                         <path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"></path>
                     </svg>
-                    Copy to Clipboard
+                    Copy
                 </button>
             </div>
         </div>
-        
+
+        <div class="info-box">
+            <div class="info-header">
+                <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
+                    <circle cx="12" cy="12" r="10"></circle>
+                    <line x1="12" y1="16" x2="12" y2="12"></line>
+                    <line x1="12" y1="8" x2="12.01" y2="8"></line>
+                </svg>
+                Access Token Claims
+            </div>
+            <p class="empty-note">Decoded payload of the access token JWT (when the IdP issues one).</p>
+            <div class="jwt-container">
+                <pre class="jwt-text" id="accessTokenClaims">Loading...</pre>
+            </div>
+            <div class="buttons">
+                <button class="copy-button" onclick="copyToClipboard('accessTokenClaims')">
+                    <svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
+                        <rect x="9" y="9" width="13" height="13" rx="2" ry="2"></rect>
+                        <path d="M5 15H4a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h9a2 2 0 0 1 2 2v1"></path>
+                    </svg>
+                    Copy
+                </button>
+            </div>
+        </div>
+
         <a href="/sso/debug/login" class="back-button">
             Try Another SSO Login
         </a>
@@ -234,39 +284,58 @@ jwt_display_template = """
 
     <script>
         // This will be populated with the actual data from the server
-        const userData = SSO_DATA;
-        
-        function renderUserData() {
-            const container = document.getElementById('userData');
-            const jsonDisplay = document.getElementById('jsonData');
-            
-            // Format JSON with indentation for display
-            jsonDisplay.textContent = JSON.stringify(userData, null, 2);
-            
-            // Clear container
+        const ssoData = SSO_DATA;
+
+        function renderParsed(container, parsed) {
             container.innerHTML = '';
-            
-            // Add each key-value pair to the UI
-            for (const [key, value] of Object.entries(userData)) {
-                if (typeof value !== 'object' || value === null) {
-                    const row = document.createElement('div');
-                    row.className = 'data-row';
-                    
-                    const label = document.createElement('div');
-                    label.className = 'data-label';
-                    label.textContent = key;
-                    
-                    const dataValue = document.createElement('div');
-                    dataValue.className = 'data-value';
-                    dataValue.textContent = value !== null ? value : 'null';
-                    
-                    row.appendChild(label);
-                    row.appendChild(dataValue);
-                    container.appendChild(row);
+            const entries = Object.entries(parsed || {});
+            if (entries.length === 0) {
+                const note = document.createElement('p');
+                note.className = 'empty-note';
+                note.textContent = 'No fields available.';
+                container.appendChild(note);
+                return;
+            }
+            for (const [key, value] of entries) {
+                const row = document.createElement('div');
+                row.className = 'data-row';
+
+                const label = document.createElement('div');
+                label.className = 'data-label';
+                label.textContent = key;
+
+                const dataValue = document.createElement('div');
+                dataValue.className = 'data-value';
+                if (value === null || value === undefined) {
+                    dataValue.textContent = 'null';
+                } else if (typeof value === 'object') {
+                    dataValue.textContent = JSON.stringify(value);
+                } else {
+                    dataValue.textContent = String(value);
                 }
+
+                row.appendChild(label);
+                row.appendChild(dataValue);
+                container.appendChild(row);
             }
         }
-        
+
+        function renderJson(elementId, value) {
+            const el = document.getElementById(elementId);
+            const obj = value || {};
+            if (Object.keys(obj).length === 0) {
+                el.textContent = '(empty — provider returned no claims for this section)';
+            } else {
+                el.textContent = JSON.stringify(obj, null, 2);
+            }
+        }
+
+        function renderUserData() {
+            renderParsed(document.getElementById('parsedByProxy'), ssoData.parsed_by_proxy);
+            renderJson('rawClaims', ssoData.raw_claims);
+            renderJson('accessTokenClaims', ssoData.access_token_claims);
+        }
+
         function copyToClipboard(elementId) {
             const text = document.getElementById(elementId).textContent;
             navigator.clipboard.writeText(text).then(() => {
@@ -275,7 +344,7 @@ jwt_display_template = """
                 console.error('Could not copy text: ', err);
             });
         }
-        
+
         // Render the data when the page loads
         document.addEventListener('DOMContentLoaded', renderUserData);
     </script>

litellm/proxy/common_utils/reset_budget_job.py

@@ -2,7 +2,7 @@ import asyncio
 import json
 import time
 from datetime import datetime, timezone
-from typing import Any, List, Literal, Optional, Union
+from typing import Any, Callable, List, Literal, Optional, Union
 
 import litellm
 from litellm._logging import verbose_proxy_logger
@@ -83,93 +83,139 @@ class ResetBudgetJob:
                 "Failed to reset spend counter %s: %s", counter_key, e
             )
 
+    @staticmethod
+    async def _invalidate_user_api_key_cache_entry(cache_key: str) -> None:
+        """Drop a stale management-cache entry so the next read fetches from DB.
+
+        Some entity types (notably tags and end-users) are not handled by
+        SpendCounterReseed.from_db, so when a spend counter expires the
+        budget check falls back to ``cached_obj.spend``. If that cached
+        object lingers in ``user_api_key_cache`` past a budget reset, the
+        stale ``.spend`` keeps the entity blocked indefinitely. Deleting
+        the cache entry forces the next auth-time fetch to reload the
+        zeroed row from Postgres.
+        """
+        try:
+            from litellm.proxy.proxy_server import user_api_key_cache
+
+            await user_api_key_cache.async_delete_cache(key=cache_key)
+        except Exception as e:
+            verbose_proxy_logger.warning(
+                "Failed to invalidate user_api_key_cache entry %s: %s",
+                cache_key,
+                e,
+            )
+
+    async def _cascade_reset_spend_for_budget_link(
+        self,
+        budgets_to_reset: List[LiteLLM_BudgetTableFull],
+        table: Any,
+        counter_key_fn: Callable[[Any], str],
+        log_subject: str,
+        extra_where: Optional[dict] = None,
+        cache_key_fn: Optional[Callable[[Any], str]] = None,
+    ):
+        """
+        Generic cascade: zero spend on rows whose budget_id is in the reset set.
+
+        ``cache_key_fn`` is optional: when provided, after the DB update each
+        matching row's entry in ``user_api_key_cache`` is also dropped. This
+        is required for entities whose spend counter is read with the cached
+        object's ``.spend`` as fallback (tags, end-users) — otherwise the
+        stale cached object pins enforcement to the pre-reset spend until
+        its TTL expires.
+        """
+        budget_ids = [b.budget_id for b in budgets_to_reset if b.budget_id is not None]
+        if not budget_ids:
+            return
+
+        where: dict = {"budget_id": {"in": budget_ids}}
+        if extra_where:
+            where.update(extra_where)
+
+        try:
+            rows = await table.find_many(where=where)
+        except Exception as e:
+            rows = []
+            verbose_proxy_logger.warning(
+                "Failed to fetch %s for counter invalidation: %s", log_subject, e
+            )
+
+        update_result = await table.update_many(where=where, data={"spend": 0})
+
+        for row in rows:
+            await self._invalidate_spend_counter(counter_key_fn(row))
+            if cache_key_fn is not None:
+                await self._invalidate_user_api_key_cache_entry(cache_key_fn(row))
+
+        return update_result
+
     async def reset_budget_for_litellm_team_members(
         self, budgets_to_reset: List[LiteLLM_BudgetTableFull]
     ):
         """
         Resets the budget for all LiteLLM Team Members if their budget has expired
         """
-        budget_ids = [
-            budget.budget_id
-            for budget in budgets_to_reset
-            if budget.budget_id is not None
-        ]
-
-        try:
-            memberships = await self.prisma_client.db.litellm_teammembership.find_many(
-                where={"budget_id": {"in": budget_ids}}
-            )
-        except Exception as e:
-            memberships = []
-            verbose_proxy_logger.warning(
-                "Failed to fetch team memberships for counter invalidation: %s", e
-            )
-
-        update_result = await self.prisma_client.db.litellm_teammembership.update_many(
-            where={"budget_id": {"in": budget_ids}},
-            data={
-                "spend": 0,
-            },
+        return await self._cascade_reset_spend_for_budget_link(
+            budgets_to_reset=budgets_to_reset,
+            table=self.prisma_client.db.litellm_teammembership,
+            counter_key_fn=lambda m: f"spend:team_member:{m.user_id}:{m.team_id}",
+            log_subject="team memberships",
         )
 
-        for m in memberships:
-            await self._invalidate_spend_counter(
-                f"spend:team_member:{m.user_id}:{m.team_id}"
-            )
-
-        return update_result
-
     async def reset_budget_for_keys_linked_to_budgets(
         self, budgets_to_reset: List[LiteLLM_BudgetTableFull]
     ):
         """
         Resets the spend for keys linked to budget tiers that are being reset.
 
-        This handles keys that have budget_id but no budget_duration set on the key
-        itself. Keys with budget_id rely on their linked budget tier's reset schedule
-        rather than having their own budget_duration.
-
-        Keys that have their own budget_duration are already handled by
-        reset_budget_for_litellm_keys() and are excluded here to avoid
-        double-resetting.
+        Excludes keys with their own budget_duration; those are reset by
+        reset_budget_for_litellm_keys() to avoid double-resetting.
         """
-        budget_ids = [
-            budget.budget_id
-            for budget in budgets_to_reset
-            if budget.budget_id is not None
-        ]
-        if not budget_ids:
-            return
-
-        where_clause: dict = {
-            "budget_id": {"in": budget_ids},
-            "budget_duration": None,  # only keys without their own reset schedule
-            "spend": {"gt": 0},  # only reset keys that have accumulated spend
-        }
-
-        try:
-            keys = await self.prisma_client.db.litellm_verificationtoken.find_many(
-                where=where_clause
-            )
-        except Exception as e:
-            keys = []
-            verbose_proxy_logger.warning(
-                "Failed to fetch keys for counter invalidation: %s", e
-            )
-
-        update_result = (
-            await self.prisma_client.db.litellm_verificationtoken.update_many(
-                where=where_clause,
-                data={
-                    "spend": 0,
-                },
-            )
+        return await self._cascade_reset_spend_for_budget_link(
+            budgets_to_reset=budgets_to_reset,
+            table=self.prisma_client.db.litellm_verificationtoken,
+            counter_key_fn=lambda k: f"spend:key:{k.token}",
+            log_subject="keys",
+            extra_where={"budget_duration": None, "spend": {"gt": 0}},
         )
 
-        for k in keys:
-            await self._invalidate_spend_counter(f"spend:key:{k.token}")
+    async def reset_budget_for_orgs_linked_to_budgets(
+        self, budgets_to_reset: List[LiteLLM_BudgetTableFull]
+    ):
+        """
+        Resets the spend for orgs linked to budget tiers that are being reset.
+        """
+        return await self._cascade_reset_spend_for_budget_link(
+            budgets_to_reset=budgets_to_reset,
+            table=self.prisma_client.db.litellm_organizationtable,
+            counter_key_fn=lambda o: f"spend:org:{o.organization_id}",
+            log_subject="orgs",
+            extra_where={"spend": {"gt": 0}},
+        )
 
-        return update_result
+    async def reset_budget_for_tags_linked_to_budgets(
+        self, budgets_to_reset: List[LiteLLM_BudgetTableFull]
+    ):
+        """
+        Resets the spend for tags linked to budget tiers that are being reset.
+
+        Also drops each tag's ``user_api_key_cache`` entry so the next
+        ``_tag_max_budget_check`` reloads the zeroed row from the DB.
+        ``SpendCounterReseed.from_db`` intentionally returns ``None`` for
+        tags, so the budget check falls back to the cached
+        ``LiteLLM_TagTable.spend`` once the spend counter expires; without
+        this invalidation, that stale ``.spend`` keeps the tag over-budget
+        indefinitely.
+        """
+        return await self._cascade_reset_spend_for_budget_link(
+            budgets_to_reset=budgets_to_reset,
+            table=self.prisma_client.db.litellm_tagtable,
+            counter_key_fn=lambda t: f"spend:tag:{t.tag_name}",
+            log_subject="tags",
+            extra_where={"spend": {"gt": 0}},
+            cache_key_fn=lambda t: f"tag:{t.tag_name}",
+        )
 
     async def reset_budget_for_litellm_budget_table(self):
         """
@@ -237,6 +283,14 @@ class ResetBudgetJob:
                     budgets_to_reset=budgets_to_reset
                 )
 
+                await self.reset_budget_for_orgs_linked_to_budgets(
+                    budgets_to_reset=budgets_to_reset
+                )
+
+                await self.reset_budget_for_tags_linked_to_budgets(
+                    budgets_to_reset=budgets_to_reset
+                )
+
             if endusers_to_reset is not None and len(endusers_to_reset) > 0:
                 for enduser in endusers_to_reset:
                     try:

litellm/proxy/db/db_spend_update_writer.py

@@ -1131,10 +1131,12 @@ class DBSpendUpdateWriter:
                         timeout=timedelta(seconds=60)
                     ) as transaction:
                         async with transaction.batch_() as batcher:
-                            for (
-                                user_id,
-                                response_cost,
-                            ) in user_list_transactions.items():
+                            # Sort by ID for consistent lock ordering across pods to prevent deadlocks.
+                            # batch_() issues statements sequentially within the tx, so iteration
+                            # order = lock acquisition order.
+                            for user_id, response_cost in sorted(
+                                user_list_transactions.items()
+                            ):
                                 batcher.litellm_usertable.update_many(
                                     where={"user_id": user_id},
                                     data={"spend": {"increment": response_cost}},
@@ -1186,10 +1188,10 @@ class DBSpendUpdateWriter:
                         timeout=timedelta(seconds=60)
                     ) as transaction:
                         async with transaction.batch_() as batcher:
-                            for (
-                                token,
-                                response_cost,
-                            ) in key_list_transactions.items():
+                            # Sort by token for consistent lock ordering across pods to prevent deadlocks.
+                            for token, response_cost in sorted(
+                                key_list_transactions.items()
+                            ):
                                 batcher.litellm_verificationtoken.update_many(  # 'update_many' prevents error from being raised if no row exists
                                     where={"token": token},
                                     data={
@@ -1230,10 +1232,10 @@ class DBSpendUpdateWriter:
                         timeout=timedelta(seconds=60)
                     ) as transaction:
                         async with transaction.batch_() as batcher:
-                            for (
-                                team_id,
-                                response_cost,
-                            ) in team_list_transactions.items():
+                            # Sort by team_id for consistent lock ordering across pods to prevent deadlocks.
+                            for team_id, response_cost in sorted(
+                                team_list_transactions.items()
+                            ):
                                 verbose_proxy_logger.debug(
                                     "Updating spend for team id={} by {}".format(
                                         team_id, response_cost
@@ -1288,10 +1290,11 @@ class DBSpendUpdateWriter:
                         timeout=timedelta(seconds=60)
                     ) as transaction:
                         async with transaction.batch_() as batcher:
-                            for (
-                                key,
-                                response_cost,
-                            ) in team_member_list_transactions.items():
+                            # Sort by composite key for consistent lock ordering across pods to prevent deadlocks.
+                            # Key format "team_id::<v>::user_id::<v>" makes the string sort equivalent to sorting by (team_id, user_id).
+                            for key, response_cost in sorted(
+                                team_member_list_transactions.items()
+                            ):
                                 # key is "team_id::<value>::user_id::<value>"
                                 team_id = key.split("::")[1]
                                 user_id = key.split("::")[3]
@@ -1348,10 +1351,10 @@ class DBSpendUpdateWriter:
                         timeout=timedelta(seconds=60)
                     ) as transaction:
                         async with transaction.batch_() as batcher:
-                            for (
-                                org_id,
-                                response_cost,
-                            ) in org_list_transactions.items():
+                            # Sort by org_id for consistent lock ordering across pods to prevent deadlocks.
+                            for org_id, response_cost in sorted(
+                                org_list_transactions.items()
+                            ):
                                 batcher.litellm_organizationtable.update_many(  # 'update_many' prevents error from being raised if no row exists
                                     where={"organization_id": org_id},
                                     data={"spend": {"increment": response_cost}},
@@ -1439,7 +1442,10 @@ class DBSpendUpdateWriter:
                         timeout=timedelta(seconds=60)
                     ) as transaction:
                         async with transaction.batch_() as batcher:
-                            for entity_id, response_cost in transactions.items():
+                            # Sort by entity_id for consistent lock ordering across pods to prevent deadlocks.
+                            for entity_id, response_cost in sorted(
+                                transactions.items()
+                            ):
                                 verbose_proxy_logger.debug(
                                     f"Updating spend for {entity_name} {where_field}={entity_id} by {response_cost}"
                                 )

litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py

@@ -5,8 +5,10 @@ from typing import Optional
 from litellm._logging import verbose_proxy_logger
 from litellm.caching import RedisCache
 from litellm.constants import (
+    SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS,
     SPEND_LOG_CLEANUP_BATCH_SIZE,
     SPEND_LOG_CLEANUP_JOB_NAME,
+    SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES,
     SPEND_LOG_RUN_LOOPS,
 )
 from litellm.litellm_core_utils.duration_parser import duration_in_seconds
@@ -74,6 +76,7 @@ class SpendLogCleanup:
         """
         total_deleted = 0
         run_count = 0
+        consecutive_failures = 0
         while True:
             if run_count > SPEND_LOG_RUN_LOOPS:
                 verbose_proxy_logger.info(
@@ -82,18 +85,50 @@ class SpendLogCleanup:
                 break
             # Step 1: Find logs and delete them in one go without fetching to application
             # Delete in batches, limited by self.batch_size
-            deleted_result = await prisma_client.db.execute_raw(
-                """
-                DELETE FROM "LiteLLM_SpendLogs"
-                WHERE "request_id" IN (
-                    SELECT "request_id" FROM "LiteLLM_SpendLogs"
-                    WHERE "startTime" < $1::timestamptz
-                    LIMIT $2
+            try:
+                deleted_result = await prisma_client.db.execute_raw(
+                    """
+                    DELETE FROM "LiteLLM_SpendLogs"
+                    WHERE "request_id" IN (
+                        SELECT "request_id" FROM "LiteLLM_SpendLogs"
+                        WHERE "startTime" < $1::timestamptz
+                        LIMIT $2
+                    )
+                    """,
+                    cutoff_date,
+                    self.batch_size,
                 )
-                """,
-                cutoff_date,
-                self.batch_size,
-            )
+            except Exception as batch_exc:
+                # A single batch failure (e.g. Prisma/DB timeout) must not abort
+                # the whole run — subsequent batches may still succeed.
+                consecutive_failures += 1
+                verbose_proxy_logger.exception(
+                    "Spend log cleanup batch failed "
+                    "(run_count=%d, consecutive_failures=%d, batch_size=%d, "
+                    "cutoff=%s, total_deleted_so_far=%d): %s: %s",
+                    run_count,
+                    consecutive_failures,
+                    self.batch_size,
+                    cutoff_date.isoformat(),
+                    total_deleted,
+                    type(batch_exc).__name__,
+                    batch_exc,
+                )
+                if (
+                    consecutive_failures
+                    >= SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES
+                ):
+                    verbose_proxy_logger.error(
+                        "Aborting spend log cleanup after %d consecutive batch "
+                        "failures; total deleted before abort: %d",
+                        consecutive_failures,
+                        total_deleted,
+                    )
+                    break
+                await asyncio.sleep(SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS)
+                continue
+
+            consecutive_failures = 0
 
             deleted_count = 0
             if isinstance(deleted_result, int):
@@ -168,7 +203,13 @@ class SpendLogCleanup:
             verbose_proxy_logger.info(f"Deleted {total_deleted} logs")
 
         except Exception as e:
-            verbose_proxy_logger.error(f"Error during cleanup: {str(e)}")
+            # .exception() captures the traceback; str(e) alone on a Prisma/DB
+            # timeout is often empty and gives operators no signal to diagnose.
+            verbose_proxy_logger.exception(
+                "Error during spend log cleanup: %s: %s",
+                type(e).__name__,
+                e,
+            )
             return  # Return after error handling
         finally:
             # Only release the lock if it was actually acquired

litellm/proxy/db/prisma_client.py

@@ -10,13 +10,64 @@ import subprocess
 import time
 import urllib
 import urllib.parse
+from dataclasses import dataclass
 from datetime import datetime, timedelta
-from typing import Any, Optional, Union
+from typing import Any, Dict, Optional, Union
 
 from litellm._logging import verbose_proxy_logger
 from litellm.secret_managers.main import str_to_bool
 
 
+@dataclass(frozen=True)
+class IAMEndpoint:
+    """Static parts of an RDS IAM-authenticated Postgres connection.
+
+    The IAM token rotates every ~15 minutes; everything else (host, port, user,
+    database name, schema) stays fixed. We capture the static fields once so
+    refresh just regenerates the token and reassembles the URL.
+    """
+
+    host: str
+    port: str
+    user: str
+    name: str
+    schema: Optional[str] = None
+
+    def build_url(self, token: str) -> str:
+        url = f"postgresql://{self.user}:{token}@{self.host}:{self.port}/{self.name}"
+        if self.schema:
+            url += f"?schema={self.schema}"
+        return url
+
+
+def parse_iam_endpoint_from_url(url: str) -> IAMEndpoint:
+    """Parse an IAMEndpoint from a Postgres URL.
+
+    Used so a reader URL can drive its own IAM refresh without requiring
+    callers to set parallel DATABASE_HOST_READ_REPLICA / etc. env vars.
+    """
+    parsed = urllib.parse.urlparse(url)
+    if not parsed.hostname or not parsed.username:
+        raise ValueError("Cannot parse IAM endpoint from URL: missing host or username")
+    name = (parsed.path or "/").lstrip("/")
+    if not name:
+        raise ValueError("Cannot parse IAM endpoint from URL: missing database name")
+    port = str(parsed.port) if parsed.port else "5432"
+    schema: Optional[str] = None
+    if parsed.query:
+        qs = urllib.parse.parse_qs(parsed.query)
+        schema_vals = qs.get("schema")
+        if schema_vals:
+            schema = schema_vals[0]
+    return IAMEndpoint(
+        host=parsed.hostname,
+        port=port,
+        user=parsed.username,
+        name=name,
+        schema=schema,
+    )
+
+
 class PrismaWrapper:
     """
     Wrapper around Prisma client that handles RDS IAM token authentication.
@@ -37,10 +88,33 @@ class PrismaWrapper:
     # Fallback refresh interval if token parsing fails (10 minutes)
     FALLBACK_REFRESH_INTERVAL_SECONDS = 600
 
-    def __init__(self, original_prisma: Any, iam_token_db_auth: bool):
+    def __init__(
+        self,
+        original_prisma: Any,
+        iam_token_db_auth: bool,
+        *,
+        db_url_env_var: str = "DATABASE_URL",
+        iam_endpoint: Optional[IAMEndpoint] = None,
+        recreate_uses_datasource: bool = False,
+        log_prefix: str = "",
+    ):
         self._original_prisma = original_prisma
         self.iam_token_db_auth = iam_token_db_auth
 
+        # Per-connection knobs so the same wrapper can be used for the writer
+        # (defaults: DATABASE_URL env, IAM endpoint from DATABASE_HOST/etc.,
+        # recreate via env reload) or for a reader (DATABASE_URL_READ_REPLICA
+        # env, IAM endpoint parsed from that URL, recreate via datasource
+        # override since Prisma only auto-reads DATABASE_URL).
+        self._db_url_env_var = db_url_env_var
+        self._iam_endpoint = iam_endpoint
+        self._recreate_uses_datasource = recreate_uses_datasource
+        # Tag every log line emitted by this wrapper instance so writer and
+        # reader can be told apart in interleaved output (e.g. "[writer] RDS
+        # IAM token refresh scheduled in 720 seconds"). Empty string (default)
+        # keeps backward-compatible logs for the single-DB case.
+        self._log_prefix = f"{log_prefix} " if log_prefix else ""
+
         # Background token refresh task management
         self._token_refresh_task: Optional[asyncio.Task] = None
         self._reconnection_lock = asyncio.Lock()
@@ -157,7 +231,7 @@ class PrismaWrapper:
             Returns 0 if token should be refreshed immediately.
             Returns FALLBACK_REFRESH_INTERVAL_SECONDS if parsing fails.
         """
-        db_url = os.getenv("DATABASE_URL")
+        db_url = os.getenv(self._db_url_env_var)
         token = self._extract_token_from_db_url(db_url)
         expiration_time = self._parse_token_expiration(token)
 
@@ -199,12 +273,30 @@ class PrismaWrapper:
         return datetime.utcnow() > expiration_time
 
     def get_rds_iam_token(self) -> Optional[str]:
-        """Generate a new RDS IAM token and update DATABASE_URL."""
-        if self.iam_token_db_auth:
-            from litellm.proxy.auth.rds_iam_token import generate_iam_auth_token
+        """Generate a new RDS IAM token and update the configured DB URL env var.
 
+        When the wrapper was constructed with an explicit `iam_endpoint`
+        (typical for a reader wrapper whose host/port/user came from a parsed
+        URL), use that. Otherwise fall back to the legacy DATABASE_HOST/PORT/
+        USER/NAME/SCHEMA env vars (writer behavior).
+        """
+        if not self.iam_token_db_auth:
+            return None
+
+        from litellm.proxy.auth.rds_iam_token import generate_iam_auth_token
+
+        if self._iam_endpoint is not None:
+            endpoint = self._iam_endpoint
+            token = generate_iam_auth_token(
+                db_host=endpoint.host, db_port=endpoint.port, db_user=endpoint.user
+            )
+            _db_url = endpoint.build_url(token)
+        else:
             db_host = os.getenv("DATABASE_HOST")
-            db_port = os.getenv("DATABASE_PORT")
+            # Default to the Postgres standard port; passing None to
+            # `generate_iam_auth_token` makes botocore embed the literal
+            # string "None" in the presigned URL, which then fails to parse.
+            db_port = os.getenv("DATABASE_PORT", "5432")
             db_user = os.getenv("DATABASE_USER")
             db_name = os.getenv("DATABASE_NAME")
             db_schema = os.getenv("DATABASE_SCHEMA")
@@ -217,9 +309,8 @@ class PrismaWrapper:
             if db_schema:
                 _db_url += f"?schema={db_schema}"
 
-            os.environ["DATABASE_URL"] = _db_url
-            return _db_url
-        return None
+        os.environ[self._db_url_env_var] = _db_url
+        return _db_url
 
     async def recreate_prisma_client(
         self, new_db_url: str, http_client: Optional[Any] = None
@@ -231,6 +322,11 @@ class PrismaWrapper:
         synchronous `subprocess.Popen.wait()` that can freeze the asyncio event
         loop for 30-120+ seconds when the engine is stuck on TCP close,
         breaking `/health/liveliness` and causing Kubernetes pod restarts.
+
+        The writer wrapper relies on Prisma re-reading `DATABASE_URL` from env;
+        the reader wrapper opts into `recreate_uses_datasource=True` so the
+        new URL is passed explicitly via `datasource={"url": ...}` (Prisma
+        does not auto-read alternate env vars like DATABASE_URL_READ_REPLICA).
         """
         from prisma import Prisma  # type: ignore
 
@@ -238,10 +334,12 @@ class PrismaWrapper:
         if old_engine_pid > 0:
             await self._kill_engine_process(old_engine_pid)
 
+        kwargs: Dict[str, Any] = {}
         if http_client is not None:
-            self._original_prisma = Prisma(http=http_client)
-        else:
-            self._original_prisma = Prisma()
+            kwargs["http"] = http_client
+        if self._recreate_uses_datasource:
+            kwargs["datasource"] = {"url": new_db_url}
+        self._original_prisma = Prisma(**kwargs)
 
         await self._original_prisma.connect()
 
@@ -265,7 +363,8 @@ class PrismaWrapper:
 
         self._token_refresh_task = asyncio.create_task(self._token_refresh_loop())
         verbose_proxy_logger.info(
-            "Started RDS IAM token proactive refresh background task"
+            "%sStarted RDS IAM token proactive refresh background task",
+            self._log_prefix,
         )
 
     async def stop_token_refresh_task(self) -> None:
@@ -283,7 +382,9 @@ class PrismaWrapper:
         except asyncio.CancelledError:
             pass
         self._token_refresh_task = None
-        verbose_proxy_logger.info("Stopped RDS IAM token refresh background task")
+        verbose_proxy_logger.info(
+            "%sStopped RDS IAM token refresh background task", self._log_prefix
+        )
 
     async def _token_refresh_loop(self) -> None:
         """
@@ -294,7 +395,7 @@ class PrismaWrapper:
         This is more efficient than polling, requiring only 1 wake-up per token cycle.
         """
         verbose_proxy_logger.info(
-            f"RDS IAM token refresh loop started. "
+            f"{self._log_prefix}RDS IAM token refresh loop started. "
             f"Tokens will be refreshed {self.TOKEN_REFRESH_BUFFER_SECONDS}s before expiration."
         )
 
@@ -305,21 +406,25 @@ class PrismaWrapper:
 
                 if sleep_seconds > 0:
                     verbose_proxy_logger.info(
-                        f"RDS IAM token refresh scheduled in {sleep_seconds:.0f} seconds "
-                        f"({sleep_seconds / 60:.1f} minutes)"
+                        f"{self._log_prefix}RDS IAM token refresh scheduled in "
+                        f"{sleep_seconds:.0f} seconds ({sleep_seconds / 60:.1f} minutes)"
                     )
                     await asyncio.sleep(sleep_seconds)
 
                 # Refresh the token
-                verbose_proxy_logger.info("Proactively refreshing RDS IAM token...")
+                verbose_proxy_logger.info(
+                    "%sProactively refreshing RDS IAM token...", self._log_prefix
+                )
                 await self._safe_refresh_token()
 
             except asyncio.CancelledError:
-                verbose_proxy_logger.info("RDS IAM token refresh loop cancelled")
+                verbose_proxy_logger.info(
+                    "%sRDS IAM token refresh loop cancelled", self._log_prefix
+                )
                 break
             except Exception as e:
                 verbose_proxy_logger.error(
-                    f"Error in RDS IAM token refresh loop: {e}. "
+                    f"{self._log_prefix}Error in RDS IAM token refresh loop: {e}. "
                     f"Retrying in {self.FALLBACK_REFRESH_INTERVAL_SECONDS}s..."
                 )
                 # On error, wait before retrying to avoid tight error loops
@@ -341,65 +446,75 @@ class PrismaWrapper:
                 await self.recreate_prisma_client(new_db_url)
                 self._last_refresh_time = datetime.utcnow()
                 verbose_proxy_logger.info(
-                    "RDS IAM token refreshed successfully. New token valid for ~15 minutes."
+                    "%sRDS IAM token refreshed successfully. New token valid for ~15 minutes.",
+                    self._log_prefix,
                 )
             else:
                 verbose_proxy_logger.error(
-                    "Failed to generate new RDS IAM token during proactive refresh"
+                    "%sFailed to generate new RDS IAM token during proactive refresh",
+                    self._log_prefix,
                 )
 
     def __getattr__(self, name: str):
         """
         Proxy attribute access to the underlying Prisma client.
 
-        If IAM token auth is enabled and the token is expired, this method
-        provides a synchronous fallback to refresh the token. However, this
-        should rarely be needed since the background task proactively refreshes
-        tokens before they expire.
+        If IAM token auth is enabled and the token is found expired here, the
+        proactive refresh task has missed its window. Behavior depends on
+        whether we're called from inside a running event loop:
 
-        FIXED: Now properly waits for reconnection to complete before returning,
-        instead of the previous fire-and-forget pattern that caused the bug.
+        - Inside the loop (typical: from a coroutine): schedule a refresh as a
+          background task and return the (stale) attribute. The caller's await
+          will likely fail with a connection error and be retried by upper
+          layers (`call_with_db_reconnect_retry`); by that time the refresh
+          has either completed or escalated to the proactive loop's error
+          path. We CANNOT block here — `run_coroutine_threadsafe(...)` +
+          `future.result()` from inside the same loop deadlocks the loop
+          (loop thread is blocked, scheduled coroutine never runs, 30s timeout).
+
+        - No running loop (sync caller, mostly tests): run the refresh in a
+          fresh loop and re-fetch the attribute.
         """
         original_attr = getattr(self._original_prisma, name)
 
         if self.iam_token_db_auth:
-            db_url = os.getenv("DATABASE_URL")
+            db_url = os.getenv(self._db_url_env_var)
 
             # Check if token is expired (should be rare if background task is running)
             if self.is_token_expired(db_url):
-                verbose_proxy_logger.warning(
-                    "RDS IAM token expired in __getattr__ - proactive refresh may have failed. "
-                    "Triggering synchronous fallback refresh..."
-                )
+                try:
+                    running_loop = asyncio.get_running_loop()
+                except RuntimeError:
+                    running_loop = None
 
-                new_db_url = self.get_rds_iam_token()
-                if new_db_url:
-                    loop = asyncio.get_event_loop()
-
-                    if loop.is_running():
-                        # FIXED: Actually wait for the reconnection to complete!
-                        # The previous code used fire-and-forget which caused the bug.
-                        future = asyncio.run_coroutine_threadsafe(
-                            self.recreate_prisma_client(new_db_url), loop
-                        )
-                        try:
-                            # Wait up to 30 seconds for reconnection
-                            future.result(timeout=30)
-                            verbose_proxy_logger.info(
-                                "Synchronous token refresh completed successfully"
-                            )
-                        except Exception as e:
-                            verbose_proxy_logger.error(
-                                f"Failed to refresh token synchronously: {e}"
-                            )
-                            raise
-                    else:
-                        asyncio.run(self.recreate_prisma_client(new_db_url))
-
-                    # Get the NEW attribute after reconnection
-                    original_attr = getattr(self._original_prisma, name)
+                if running_loop is not None:
+                    verbose_proxy_logger.warning(
+                        "%sRDS IAM token expired in __getattr__ — proactive refresh "
+                        "may have failed. Scheduling async refresh; the current "
+                        "request may fail and be retried with the fresh token.",
+                        self._log_prefix,
+                    )
+                    # Non-blocking: schedule the locked refresh on the
+                    # running loop. The reconnection lock inside
+                    # `_safe_refresh_token` coalesces concurrent triggers.
+                    running_loop.create_task(self._safe_refresh_token())
                 else:
-                    raise ValueError("Failed to get RDS IAM token")
+                    verbose_proxy_logger.warning(
+                        "%sRDS IAM token expired in __getattr__ — proactive refresh "
+                        "may have failed. Triggering synchronous fallback refresh...",
+                        self._log_prefix,
+                    )
+                    new_db_url = self.get_rds_iam_token()
+                    if new_db_url:
+                        asyncio.run(self.recreate_prisma_client(new_db_url))
+                        # Re-fetch attribute against the recreated Prisma instance.
+                        original_attr = getattr(self._original_prisma, name)
+                        verbose_proxy_logger.info(
+                            "%sSynchronous token refresh completed successfully",
+                            self._log_prefix,
+                        )
+                    else:
+                        raise ValueError("Failed to get RDS IAM token")
 
         return original_attr
 

litellm/proxy/db/routing_prisma_wrapper.py

@@ -0,0 +1,213 @@
+"""
+RoutingPrismaWrapper: routes Prisma reads to a read-replica client and writes
+to a writer client. Used when DATABASE_URL_READ_REPLICA is configured;
+otherwise PrismaClient uses the writer-only PrismaWrapper directly.
+"""
+
+import os
+from typing import Any, Callable, Optional
+
+from litellm._logging import verbose_proxy_logger
+from litellm.proxy.db.prisma_client import PrismaWrapper
+
+# Per-model action methods that read from the database. These are routed to
+# the read replica when one is configured.
+_MODEL_READ_METHODS = frozenset(
+    {
+        "find_first",
+        "find_first_or_raise",
+        "find_many",
+        "find_unique",
+        "find_unique_or_raise",
+        "count",
+        "group_by",
+        "query_first",
+        "query_raw",
+    }
+)
+
+# Top-level Prisma client methods that read from the database.
+_TOP_LEVEL_READ_METHODS = frozenset({"query_first", "query_raw"})
+
+
+class _RoutedActions:
+    """Per-model accessor that sends reads to the reader and writes to the writer.
+
+    `should_use_reader` is consulted on every read dispatch so a mid-call flip
+    of the routing wrapper's reader-availability flag (e.g. after the reader
+    fails a recreate) is observed without re-fetching the actions accessor.
+    """
+
+    __slots__ = ("_writer_actions", "_reader_actions", "_should_use_reader")
+
+    def __init__(
+        self,
+        writer_actions: Any,
+        reader_actions: Any,
+        should_use_reader: Callable[[], bool],
+    ):
+        self._writer_actions = writer_actions
+        self._reader_actions = reader_actions
+        self._should_use_reader = should_use_reader
+
+    def __getattr__(self, name: str) -> Any:
+        if name in _MODEL_READ_METHODS and self._should_use_reader():
+            return getattr(self._reader_actions, name)
+        return getattr(self._writer_actions, name)
+
+
+class RoutingPrismaWrapper:
+    """
+    Routes Prisma operations between a writer and a reader Prisma client.
+
+    Reads (find_*, count, group_by, query_raw, query_first) go to the reader;
+    everything else (writes, transactions, raw execute) goes to the writer.
+    Lifecycle methods (connect, disconnect, IAM token refresh) act on both
+    clients so callers do not need to know about the split. When
+    IAM_TOKEN_DB_AUTH is enabled, both writer and reader refresh their tokens
+    independently on their own ~12-minute cadence.
+
+    Reader degradation: a reader-side failure (failed connect, failed
+    recreate) is non-fatal — the wrapper sets `_reader_unavailable=True`, logs
+    a warning, and routes subsequent reads to the writer. The next successful
+    `connect()` or `recreate_prisma_client()` clears the flag. This keeps the
+    proxy serving traffic during transient reader outages instead of failing
+    startup or returning errors for read-heavy endpoints.
+    """
+
+    def __init__(self, writer: PrismaWrapper, reader: PrismaWrapper):
+        self._writer = writer
+        self._reader = reader
+        # When True, reads fall back to the writer. Flipped on by reader
+        # connect/recreate failures and flipped off on the next reader recovery.
+        self._reader_unavailable: bool = False
+
+    @property
+    def writer(self) -> PrismaWrapper:
+        return self._writer
+
+    @property
+    def reader(self) -> PrismaWrapper:
+        return self._reader
+
+    @property
+    def reader_unavailable(self) -> bool:
+        return self._reader_unavailable
+
+    def _should_use_reader(self) -> bool:
+        return not self._reader_unavailable
+
+    async def connect(self, *args: Any, **kwargs: Any) -> None:
+        await self._writer.connect(*args, **kwargs)
+        verbose_proxy_logger.info("[writer] DB connected")
+        try:
+            await self._reader.connect(*args, **kwargs)
+            self._reader_unavailable = False
+            verbose_proxy_logger.info("[reader] DB connected")
+        except Exception as e:
+            # Degrade gracefully: the proxy keeps serving traffic with reads
+            # routed to the writer until the reader endpoint is reachable.
+            # Aborting startup here would tie proxy availability to an
+            # opt-in, best-effort reader endpoint.
+            self._reader_unavailable = True
+            verbose_proxy_logger.warning(
+                "Failed to connect to read replica DB: %s. "
+                "Falling back to the writer for reads until the reader is reachable.",
+                e,
+            )
+
+    async def disconnect(self, *args: Any, **kwargs: Any) -> None:
+        first_error: Optional[BaseException] = None
+        for client in (self._writer, self._reader):
+            try:
+                await client.disconnect(*args, **kwargs)
+            except Exception as e:
+                if first_error is None:
+                    first_error = e
+                verbose_proxy_logger.warning("Error disconnecting Prisma client: %s", e)
+        if first_error is not None:
+            raise first_error
+
+    def is_connected(self) -> bool:
+        # Reflects writer health only. The reader is best-effort; its
+        # availability is tracked via `_reader_unavailable` and a degraded
+        # reader must NOT cause a writer reconnect (would loop indefinitely
+        # since recreate_prisma_client only fixes writer-side problems).
+        return bool(self._writer.is_connected())
+
+    async def start_token_refresh_task(self) -> None:
+        await self._writer.start_token_refresh_task()
+        await self._reader.start_token_refresh_task()
+
+    async def stop_token_refresh_task(self) -> None:
+        await self._writer.stop_token_refresh_task()
+        await self._reader.stop_token_refresh_task()
+
+    async def recreate_prisma_client(
+        self, new_db_url: str, http_client: Optional[Any] = None
+    ) -> None:
+        """Recreate both writer and reader Prisma clients.
+
+        The writer reconnect path in PrismaClient calls
+        `self.db.recreate_prisma_client(...)`. Without this method, a DB-wide
+        connectivity event would only re-create the writer; the reader engine
+        would stay broken and every routed read would fail. We always recreate
+        the writer first (its URL is the one passed in), then best-effort
+        recreate the reader. A reader failure flips `_reader_unavailable=True`
+        so reads transparently fall through to the writer.
+        """
+        await self._writer.recreate_prisma_client(new_db_url, http_client=http_client)
+        try:
+            await self._recreate_reader(http_client=http_client)
+            self._reader_unavailable = False
+        except Exception as e:
+            self._reader_unavailable = True
+            verbose_proxy_logger.warning(
+                "Failed to recreate reader Prisma client: %s. "
+                "Reads will fall back to the writer until the reader recovers.",
+                e,
+            )
+
+    async def _recreate_reader(self, http_client: Optional[Any] = None) -> None:
+        """Resolve the reader URL and recreate its Prisma client.
+
+        IAM-enabled readers regenerate their token (host/port/user came from
+        the parsed reader URL at construction time). Non-IAM readers reuse
+        the URL stored in `DATABASE_URL_READ_REPLICA`.
+        """
+        if self._reader.iam_token_db_auth:
+            new_reader_url = self._reader.get_rds_iam_token()
+            if not new_reader_url:
+                raise RuntimeError(
+                    "Failed to generate fresh IAM token for read replica"
+                )
+            await self._reader.recreate_prisma_client(
+                new_reader_url, http_client=http_client
+            )
+            return
+        reader_url = os.getenv("DATABASE_URL_READ_REPLICA", "")
+        if not reader_url:
+            raise RuntimeError(
+                "DATABASE_URL_READ_REPLICA not set; cannot recreate read replica client"
+            )
+        await self._reader.recreate_prisma_client(reader_url, http_client=http_client)
+
+    def __getattr__(self, name: str) -> Any:
+        if name in _TOP_LEVEL_READ_METHODS:
+            target = self._writer if self._reader_unavailable else self._reader
+            return getattr(target, name)
+        writer_attr = getattr(self._writer, name)
+        # Per-model action accessors are non-callable instances that expose
+        # both `find_many` and `create`. Methods like execute_raw / batch_ /
+        # tx are callables and stay on the writer untouched.
+        if (
+            not callable(writer_attr)
+            and hasattr(writer_attr, "find_many")
+            and hasattr(writer_attr, "create")
+        ):
+            try:
+                reader_attr = getattr(self._reader, name)
+            except AttributeError:
+                return writer_attr
+            return _RoutedActions(writer_attr, reader_attr, self._should_use_reader)
+        return writer_attr

litellm/proxy/guardrails/guardrail_endpoints.py

@@ -242,6 +242,10 @@ async def list_guardrails_v2(
             gid = guardrail.get("guardrail_id")
             if gid in seen_guardrail_ids:
                 continue
+            # Skip stale DB-backed entries — the DB row was deleted (likely by
+            # another pod) and reconciliation hasn't fired yet on this pod.
+            if gid is not None and IN_MEMORY_GUARDRAIL_HANDLER.get_source(gid) == "db":
+                continue
             if not is_admin:
                 g_team_id = guardrail.get("team_id")
                 if g_team_id is not None and g_team_id not in caller_team_ids:
@@ -360,7 +364,7 @@ async def create_guardrail(
 
         try:
             IN_MEMORY_GUARDRAIL_HANDLER.initialize_guardrail(
-                guardrail=cast(Guardrail, result)
+                guardrail=cast(Guardrail, result), source="db"
             )
             verbose_proxy_logger.info(
                 f"Immediate sync: Successfully initialized guardrail '{guardrail_name}' (ID: {guardrail_id})"
@@ -1017,7 +1021,7 @@ async def approve_guardrail_submission(
         }
         try:
             IN_MEMORY_GUARDRAIL_HANDLER.initialize_guardrail(
-                guardrail=cast(Guardrail, guardrail_dict)
+                guardrail=cast(Guardrail, guardrail_dict), source="db"
             )
             verbose_proxy_logger.info(
                 "Approved guardrail %s (ID: %s) and initialized in memory",
@@ -1295,10 +1299,18 @@ async def get_guardrail_info(guardrail_id: str):
             guardrail_id=guardrail_id, prisma_client=prisma_client
         )
         if result is None:
-            result = IN_MEMORY_GUARDRAIL_HANDLER.get_guardrail_by_id(
+            in_memory = IN_MEMORY_GUARDRAIL_HANDLER.get_guardrail_by_id(
                 guardrail_id=guardrail_id
             )
-            guardrail_definition_location = GUARDRAIL_DEFINITION_LOCATION.CONFIG
+            # Only return config-loaded entries here. A DB-backed entry that's
+            # missing from the DB is stale (deleted on another pod, awaiting
+            # reconciliation on this one) and must surface as 404.
+            if (
+                in_memory is not None
+                and IN_MEMORY_GUARDRAIL_HANDLER.get_source(guardrail_id) == "config"
+            ):
+                result = in_memory
+                guardrail_definition_location = GUARDRAIL_DEFINITION_LOCATION.CONFIG
 
         if result is None:
             raise HTTPException(

litellm/proxy/guardrails/guardrail_registry.py

@@ -3,7 +3,7 @@
 import importlib
 import os
 from datetime import datetime, timezone
-from typing import Any, Dict, List, Optional, Type, cast
+from typing import Any, Dict, List, Literal, Optional, Set, Type, cast
 
 import litellm
 from litellm import Router
@@ -403,11 +403,19 @@ class InMemoryGuardrailHandler:
         Guardrail id to CustomGuardrail object mapping
         """
 
+        self._sources: Dict[str, Literal["db", "config"]] = {}
+        """
+        Guardrail id to provenance marker. "db" entries are reconciled against
+        the DB on each polling tick; "config" entries are owned by proxy_config.yaml
+        and never deleted by reconciliation.
+        """
+
     def initialize_guardrail(
         self,
         guardrail: Guardrail,
         config_file_path: Optional[str] = None,
         llm_router: Optional["Router"] = None,
+        source: Literal["db", "config"] = "config",
     ) -> Optional[Guardrail]:
         """
         Initialize a guardrail from a dictionary and add it to the litellm callback manager
@@ -420,6 +428,10 @@ class InMemoryGuardrailHandler:
             verbose_proxy_logger.debug(
                 "guardrail_id already exists in IN_MEMORY_GUARDRAILS"
             )
+            # Honor the caller's source even on the early-return path so a
+            # racing polling tick or a hot-reload of config can correct an
+            # entry's provenance.
+            self._sources[guardrail_id] = source
             return self.IN_MEMORY_GUARDRAILS[guardrail_id]
 
         custom_guardrail_callback: Optional[CustomGuardrail] = None
@@ -482,6 +494,11 @@ class InMemoryGuardrailHandler:
                 "skip_system_message_in_guardrail",
                 getattr(litellm_params, "skip_system_message_in_guardrail", None),
             )
+            setattr(
+                custom_guardrail_callback,
+                "skip_tool_message_in_guardrail",
+                getattr(litellm_params, "skip_tool_message_in_guardrail", None),
+            )
 
         parsed_guardrail = Guardrail(
             guardrail_id=guardrail.get("guardrail_id"),
@@ -492,6 +509,7 @@ class InMemoryGuardrailHandler:
         # store references to the guardrail in memory
         self.IN_MEMORY_GUARDRAILS[guardrail_id] = parsed_guardrail
         self.guardrail_id_to_custom_guardrail[guardrail_id] = custom_guardrail_callback
+        self._sources[guardrail_id] = source
 
         return parsed_guardrail
 
@@ -552,7 +570,10 @@ class InMemoryGuardrailHandler:
         return _guardrail_callback
 
     def update_in_memory_guardrail(
-        self, guardrail_id: str, guardrail: Guardrail
+        self,
+        guardrail_id: str,
+        guardrail: Guardrail,
+        source: Literal["db", "config"] = "db",
     ) -> None:
         """
         Update a guardrail in memory
@@ -561,6 +582,7 @@ class InMemoryGuardrailHandler:
         - updates the guardrail params in litellm.callback_manager
         """
         self.IN_MEMORY_GUARDRAILS[guardrail_id] = guardrail
+        self._sources[guardrail_id] = source
 
         custom_guardrail_callback = self.guardrail_id_to_custom_guardrail.get(
             guardrail_id
@@ -579,6 +601,7 @@ class InMemoryGuardrailHandler:
         """
         # Remove from in-memory storage
         self.IN_MEMORY_GUARDRAILS.pop(guardrail_id, None)
+        self._sources.pop(guardrail_id, None)
 
         # Remove the callback from litellm.callbacks
         custom_guardrail_callback = self.guardrail_id_to_custom_guardrail.pop(
@@ -603,6 +626,34 @@ class InMemoryGuardrailHandler:
         """
         return self.IN_MEMORY_GUARDRAILS.get(guardrail_id)
 
+    def get_source(self, guardrail_id: str) -> Optional[Literal["db", "config"]]:
+        """
+        Return the provenance of an in-memory guardrail.
+        """
+        return self._sources.get(guardrail_id)
+
+    def reconcile_db_guardrails(self, db_guardrail_ids: Set[str]) -> List[str]:
+        """
+        Drop in-memory entries that originated from the DB but are no longer
+        present in db_guardrail_ids. Config-loaded guardrails are never touched.
+
+        Called by the periodic DB polling tick so that a guardrail deleted
+        on another pod is eventually purged from this pod's memory + callbacks.
+        """
+        stale_ids = [
+            guardrail_id
+            for guardrail_id, source in self._sources.items()
+            if source == "db" and guardrail_id not in db_guardrail_ids
+        ]
+        for guardrail_id in stale_ids:
+            verbose_proxy_logger.info(
+                "Reconcile: removing stale DB-backed guardrail '%s' from memory "
+                "(deleted in DB by another pod)",
+                guardrail_id,
+            )
+            self.delete_in_memory_guardrail(guardrail_id)
+        return stale_ids
+
     def _has_guardrail_params_changed(
         self, guardrail_id: str, new_guardrail: Guardrail
     ) -> bool:
@@ -656,7 +707,10 @@ class InMemoryGuardrailHandler:
         return len(changed_fields) > 0
 
     def reinitialize_guardrail(
-        self, guardrail: Guardrail, config_file_path: Optional[str] = None
+        self,
+        guardrail: Guardrail,
+        config_file_path: Optional[str] = None,
+        source: Literal["db", "config"] = "config",
     ) -> Optional[Guardrail]:
         """
         Force re-initialization of a guardrail even if it exists in memory.
@@ -675,7 +729,7 @@ class InMemoryGuardrailHandler:
 
         # Initialize fresh (will add new callback to litellm.callbacks)
         return self.initialize_guardrail(
-            guardrail=guardrail, config_file_path=config_file_path
+            guardrail=guardrail, config_file_path=config_file_path, source=source
         )
 
     def sync_guardrail_from_db(
@@ -696,9 +750,15 @@ class InMemoryGuardrailHandler:
                 f"Guardrail '{guardrail_name}' (ID: {guardrail_id}) params changed, re-initializing..."
             )
             return self.reinitialize_guardrail(
-                guardrail=guardrail, config_file_path=config_file_path
+                guardrail=guardrail,
+                config_file_path=config_file_path,
+                source="db",
             )
 
+        # Params unchanged but the entry is still DB-backed; make sure the
+        # source marker reflects that even if it was previously set differently
+        # (e.g. a config entry whose UUID later collided with a DB row).
+        self._sources[guardrail_id] = "db"
         return self.IN_MEMORY_GUARDRAILS.get(guardrail_id)
 
 

litellm/proxy/guardrails/init_guardrails.py

@@ -30,6 +30,7 @@ def init_guardrails_v2(
             guardrail=cast(Guardrail, guardrail),
             config_file_path=config_file_path,
             llm_router=llm_router,
+            source="config",
         )
         if initialized_guardrail:
             guardrail_list.append(initialized_guardrail)

litellm/proxy/health_check_utils/shared_health_check_manager.py

@@ -253,27 +253,63 @@ class SharedHealthCheckManager:
                 # Always release the lock
                 await self.release_health_check_lock()
         else:
-            # Lock not acquired, wait briefly and try to get cached results
+            # If Redis is not configured, skip polling — there is no cache
+            # to wait for.
+            if self.redis_cache is None:
+                return await perform_health_check(
+                    model_list=model_list,
+                    details=details,
+                    max_concurrency=max_concurrency,
+                )
+
+            # Lock not acquired — poll for cached results until the lock
+            # holder finishes or the lock expires, rather than falling back
+            # to a redundant local health check after only 2 seconds.
             verbose_proxy_logger.debug(
                 "Pod %s waiting for other pod to complete health check", self.pod_id
             )
 
-            # Wait a bit for the other pod to complete
-            await asyncio.sleep(2)
+            poll_interval = 5  # seconds between cache checks
+            max_wait = self.lock_ttl  # wait at most as long as the lock can live
+            elapsed = 0
 
-            # Try to get cached results again
-            cached_results = await self.get_cached_health_check_results()
-            if cached_results is not None:
-                return (
-                    cached_results.get("healthy_endpoints", []),
-                    cached_results.get("unhealthy_endpoints", []),
-                    {},
-                )
+            while elapsed < max_wait:
+                await asyncio.sleep(poll_interval)
+                elapsed += poll_interval
 
-            # Still no cache, fall back to local health check
+                cached_results = await self.get_cached_health_check_results()
+                if cached_results is not None:
+                    verbose_proxy_logger.info(
+                        "Pod %s using cached health check results after waiting %ds",
+                        self.pod_id,
+                        elapsed,
+                    )
+                    return (
+                        cached_results.get("healthy_endpoints", []),
+                        cached_results.get("unhealthy_endpoints", []),
+                        {},
+                    )
+
+                # Check if the lock is still held — if it was released without
+                # caching (e.g. the holder crashed), stop waiting early.
+                try:
+                    lock_key = self.get_health_check_lock_key()
+                    current_owner = await self.redis_cache.async_get_cache(lock_key)
+                    if current_owner is None:
+                        verbose_proxy_logger.debug(
+                            "Pod %s detected lock released without cache, stopping wait",
+                            self.pod_id,
+                        )
+                        break
+                except Exception:
+                    # Redis hiccup — continue polling rather than crashing out
+                    pass
+
+            # Exhausted wait — fall back to local health check
             verbose_proxy_logger.warning(
-                "Pod %s falling back to local health check (no cache available)",
+                "Pod %s falling back to local health check after waiting %ds (no cache available)",
                 self.pod_id,
+                elapsed,
             )
 
             return await perform_health_check(

litellm/proxy/health_endpoints/_health_endpoints.py

@@ -1742,29 +1742,54 @@ async def test_model_connection(
         # Look up model configuration from router if model name is provided
         # This gets the litellm_params from proxy config (with resolved env vars)
         config_litellm_params: dict = {}
-        if model_name and llm_router is not None:
+        if llm_router is not None:
+            # Prefer disambiguation by deployment id (`model_info.id`) when
+            # the caller supplies it. This is required when multiple
+            # deployments share a `model_name` (e.g. wildcard `openai/*`
+            # with multiple `api_base` values for failover): the UI's
+            # "Test Connection" button targets a specific row, and that
+            # row's id is the only thing that uniquely identifies which
+            # deployment to probe. Without this, all duplicates collapse
+            # onto `deployments[0]`.
+            request_model_info = model_info or {}
+            request_model_id = request_model_info.get("id")
             try:
-                # First try to find by proxy model_name (e.g., "gpt-4o")
-                deployments = llm_router.get_model_list(model_name=model_name)
-
-                # If not found, try to find by litellm model name (e.g., "azure/gpt-4o")
-                if not deployments or len(deployments) == 0:
-                    all_deployments = llm_router.get_model_list(model_name=None)
-                    if all_deployments:
-                        for deployment in all_deployments:
-                            if (
-                                deployment.get("litellm_params", {}).get("model")
-                                == model_name
-                            ):
-                                deployments = [deployment]
-                                break
-
-                if deployments and len(deployments) > 0:
-                    # Use the first deployment's litellm_params as base config
-                    # These already have resolved environment variables from proxy config
-                    config_litellm_params = dict(
-                        deployments[0].get("litellm_params", {})
+                deployment_by_id = None
+                if request_model_id:
+                    deployment_by_id = llm_router.get_deployment(
+                        model_id=request_model_id
                     )
+
+                if deployment_by_id is not None:
+                    config_litellm_params = deployment_by_id.litellm_params.model_dump(
+                        exclude_none=True
+                    )
+                elif model_name:
+                    # Fall back to model_name lookup for callers (e.g. the
+                    # "Add Model" wizard, or curl) that don't supply an id.
+                    # First try to find by proxy model_name (e.g., "gpt-4o")
+                    deployments = llm_router.get_model_list(model_name=model_name)
+
+                    # If not found, try to find by litellm model name
+                    # (e.g., "azure/gpt-4o")
+                    if not deployments or len(deployments) == 0:
+                        all_deployments = llm_router.get_model_list(model_name=None)
+                        if all_deployments:
+                            for deployment in all_deployments:
+                                if (
+                                    deployment.get("litellm_params", {}).get("model")
+                                    == model_name
+                                ):
+                                    deployments = [deployment]
+                                    break
+
+                    if deployments and len(deployments) > 0:
+                        # Use the first deployment's litellm_params as base
+                        # config. These already have resolved environment
+                        # variables from proxy config.
+                        config_litellm_params = dict(
+                            deployments[0].get("litellm_params", {})
+                        )
             except Exception as e:
                 verbose_proxy_logger.debug(
                     f"Could not find model {model_name} in router: {e}. "

litellm/proxy/health_endpoints/health_app_factory.py

@@ -1,8 +0,0 @@
-from fastapi import FastAPI
-from litellm.proxy.health_endpoints._health_endpoints import router as health_router
-
-
-def build_health_app():
-    health_app = FastAPI(title="LiteLLM Health Endpoints")
-    health_app.include_router(health_router)
-    return health_app

litellm/proxy/hooks/model_max_budget_limiter.py

@@ -319,6 +319,9 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting):
                     response_cost=response_cost,
                 )
 
+        if self.dual_cache.redis_cache is not None:
+            await self._push_in_memory_increments_to_redis()
+
         verbose_proxy_logger.debug(
             "current state of in memory cache %s",
             json.dumps(

litellm/proxy/litellm_pre_call_utils.py

@@ -1,5 +1,6 @@
 import asyncio
 import copy
+import json
 import re
 import time
 from collections import OrderedDict
@@ -794,8 +795,17 @@ class LiteLLMProxyRequestSetup:
                 )
             )
             for k, v in litellm_logging_metadata_headers.items():
-                if v is not None:
+                if v is None:
+                    continue
+                # httpx requires header values to be str or bytes; coerce numbers/bools
+                # to str and JSON-encode dict/list (e.g. user_api_key_spend is float,
+                # user_api_key_auth_metadata is dict). See #27458.
+                if isinstance(v, (dict, list)):
+                    returned_headers["x-litellm-{}".format(k)] = json.dumps(v)
+                elif isinstance(v, (str, bytes)):
                     returned_headers["x-litellm-{}".format(k)] = v
+                else:
+                    returned_headers["x-litellm-{}".format(k)] = str(v)
 
         return returned_headers
 
@@ -1731,6 +1741,7 @@ async def add_litellm_data_to_request(  # noqa: PLR0915
         data=data,
         user_api_key_dict=user_api_key_dict,
         pre_alias_model_name=_pre_alias_model,
+        llm_router=llm_router,
     )
 
     ## ENFORCED PARAMS CHECK
@@ -1864,6 +1875,7 @@ def _apply_credential_overrides_from_model_config(
     data: dict,
     user_api_key_dict: UserAPIKeyAuth,
     pre_alias_model_name: Optional[str] = None,
+    llm_router: Optional[Router] = None,
 ) -> None:
     """
     Walk the model_config precedence chain in team/project metadata.
@@ -1899,10 +1911,19 @@ def _apply_credential_overrides_from_model_config(
     if not project_model_config and not team_model_config:
         return
 
-    # Extract provider hint from model name (e.g. "azure/gpt-4" -> "azure")
+    # Extract provider hint from model name (e.g. "azure/gpt-4" -> "azure").
+    # When the user-facing name has no provider prefix, fall back to the
+    # deployment's litellm_params so multi-provider defaultconfig entries
+    # don't silently match the first dict key (#27516).
     provider: Optional[str] = None
     if "/" in model_name:
         provider = model_name.split("/", 1)[0]
+    elif llm_router is not None:
+        provider = _resolve_provider_from_deployment(
+            llm_router=llm_router,
+            model_name=model_name,
+            pre_alias_model_name=pre_alias_model_name,
+        )
 
     credential_name = _resolve_credential_from_model_config(
         model_name=model_name,
@@ -1938,6 +1959,48 @@ def _apply_credential_overrides_from_model_config(
     )
 
 
+def _resolve_provider_from_deployment(
+    llm_router: Router,
+    model_name: str,
+    pre_alias_model_name: Optional[str] = None,
+) -> Optional[str]:
+    """
+    Resolve a provider hint from the deployment's litellm_params when the
+    user-facing model name has no provider prefix.
+
+    Tries the post-alias name first (the resolved model group), then the
+    pre-alias name. Returns None if no deployment is found or the deployment
+    has no usable provider info.
+    """
+    candidates = [model_name]
+    if pre_alias_model_name and pre_alias_model_name != model_name:
+        candidates.append(pre_alias_model_name)
+
+    for name in candidates:
+        try:
+            deployment = llm_router.get_deployment_by_model_group_name(
+                model_group_name=name
+            )
+        except Exception:
+            deployment = None
+        if deployment is None:
+            continue
+
+        litellm_params = getattr(deployment, "litellm_params", None)
+        if litellm_params is None:
+            continue
+
+        custom_provider = getattr(litellm_params, "custom_llm_provider", None)
+        if custom_provider:
+            return custom_provider
+
+        deployment_model = getattr(litellm_params, "model", "") or ""
+        if "/" in deployment_model:
+            return deployment_model.split("/", 1)[0]
+
+    return None
+
+
 def _resolve_credential_from_model_config(
     model_name: str,
     project_model_config: Optional[dict],

litellm/proxy/management_endpoints/key_management_endpoints.py

@@ -88,8 +88,8 @@ from litellm.router import Router
 from litellm.secret_managers.main import get_secret
 from litellm.types.proxy.management_endpoints.key_management_endpoints import (
     BulkUpdateKeyRequest,
-    BulkUpdateKeyRequestItem,
     BulkUpdateKeyResponse,
+    BulkUpdateTeamKeysRequest,
     FailedKeyUpdate,
     SuccessfulKeyUpdate,
 )
@@ -1881,7 +1881,7 @@ async def _get_and_validate_existing_key(
 
 
 async def _process_single_key_update(
-    key_update_item: BulkUpdateKeyRequestItem,
+    update_key_request: UpdateKeyRequest,
     user_api_key_dict: UserAPIKeyAuth,
     litellm_changed_by: Optional[str],
     prisma_client: Optional[PrismaClient],
@@ -1889,6 +1889,7 @@ async def _process_single_key_update(
     proxy_logging_obj: Any,
     llm_router: Optional[Router],
     user_custom_key_update: Optional[Callable] = None,
+    existing_key_row: Optional[LiteLLM_VerificationToken] = None,
 ) -> Dict[str, Any]:
     """
     Process a single key update with all validations and checks.
@@ -1897,13 +1898,14 @@ async def _process_single_key_update(
     including validation, permission checks, team checks, and database updates.
 
     Args:
-        key_update_item: The key update request item
+        update_key_request: Fully-constructed UpdateKeyRequest for the target key
         user_api_key_dict: The authenticated user's API key info
         litellm_changed_by: Optional header for tracking who made the change
         prisma_client: Prisma client instance
         user_api_key_cache: User API key cache
         proxy_logging_obj: Proxy logging object
         llm_router: LLM router instance
+        existing_key_row: Optional pre-fetched key row to avoid redundant lookups
 
     Returns:
         Dict containing the updated key information
@@ -1912,13 +1914,14 @@ async def _process_single_key_update(
         HTTPException: For various validation and permission errors
     """
     # Validate max_budget
-    _validate_max_budget(key_update_item.max_budget)
+    _validate_max_budget(update_key_request.max_budget)
 
     # Get and validate existing key
-    existing_key_row = await _get_and_validate_existing_key(
-        token=key_update_item.key,
-        prisma_client=prisma_client,
-    )
+    if existing_key_row is None:
+        existing_key_row = await _get_and_validate_existing_key(
+            token=update_key_request.key,
+            prisma_client=prisma_client,
+        )
 
     # Check team member permissions
     if prisma_client is not None:
@@ -1930,15 +1933,6 @@ async def _process_single_key_update(
             user_api_key_cache=user_api_key_cache,
         )
 
-    # Create UpdateKeyRequest from BulkUpdateKeyRequestItem
-    update_key_request = UpdateKeyRequest(
-        key=key_update_item.key,
-        budget_id=key_update_item.budget_id,
-        max_budget=key_update_item.max_budget,
-        team_id=key_update_item.team_id,
-        tags=key_update_item.tags,
-    )
-
     # Custom key update hook
     if user_custom_key_update is not None:
         if inspect.iscoroutinefunction(user_custom_key_update):
@@ -2003,12 +1997,12 @@ async def _process_single_key_update(
             detail={"error": "Database not connected"},
         )
 
-    _data = {**non_default_values, "token": key_update_item.key}
-    response = await prisma_client.update_data(token=key_update_item.key, data=_data)
+    _data = {**non_default_values, "token": update_key_request.key}
+    response = await prisma_client.update_data(token=update_key_request.key, data=_data)
 
     # Delete cache
     await _delete_cache_key_object(
-        hashed_token=_hash_token_if_needed(key_update_item.key),
+        hashed_token=_hash_token_if_needed(update_key_request.key),
         user_api_key_cache=user_api_key_cache,
         proxy_logging_obj=proxy_logging_obj,
     )
@@ -2598,9 +2592,15 @@ async def bulk_update_keys(
 
     for key_update_item in data.keys:
         try:
-            # Process single key update using reusable function
+            update_key_request = UpdateKeyRequest(
+                key=key_update_item.key,
+                budget_id=key_update_item.budget_id,
+                max_budget=key_update_item.max_budget,
+                team_id=key_update_item.team_id,
+                tags=key_update_item.tags,
+            )
             updated_key_info = await _process_single_key_update(
-                key_update_item=key_update_item,
+                update_key_request=update_key_request,
                 user_api_key_dict=user_api_key_dict,
                 litellm_changed_by=litellm_changed_by,
                 prisma_client=prisma_client,
@@ -2665,6 +2665,223 @@ async def bulk_update_keys(
     )
 
 
+def _build_failed_team_key_update(
+    token: str,
+    exception: Exception,
+    existing_key_row: Optional[LiteLLM_VerificationToken],
+) -> FailedKeyUpdate:
+    """Normalize an exception from the per-key update loop into a FailedKeyUpdate."""
+    if isinstance(exception, HTTPException):
+        detail = exception.detail
+        if isinstance(detail, dict):
+            error_message = detail.get("error", str(exception))
+        else:
+            error_message = str(detail)
+    elif isinstance(exception, ProxyException):
+        error_message = exception.message
+    else:
+        error_message = str(exception)
+
+    key_info: Optional[Dict[str, Any]] = None
+    if existing_key_row is not None:
+        if hasattr(existing_key_row, "model_dump"):
+            key_info = existing_key_row.model_dump()
+        elif hasattr(existing_key_row, "dict"):
+            key_info = existing_key_row.dict()
+        if key_info:
+            key_info.pop("token", None)
+
+    return FailedKeyUpdate(key=token, key_info=key_info, failed_reason=error_message)
+
+
+@router.post(
+    "/team/key/bulk_update",
+    tags=["key management"],
+    dependencies=[Depends(user_api_key_auth)],
+    response_model=BulkUpdateKeyResponse,
+)
+@management_endpoint_wrapper
+async def bulk_update_team_keys(
+    data: BulkUpdateTeamKeysRequest,
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+    litellm_changed_by: Optional[str] = Header(
+        None,
+        description="The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability",
+    ),
+):
+    """
+    Apply one update payload to many keys inside a single team.
+
+    Pass `team_id` plus either `key_ids` or `all_keys_in_team=True`. The
+    `update_fields` payload is broadcast to every selected key. Per-key
+    failures are returned in `failed_updates` rather than aborting the batch.
+
+    Callable by proxy admins, or by team admins with `KEY_UPDATE` permission.
+    """
+    from litellm.proxy.proxy_server import (
+        llm_router,
+        prisma_client,
+        proxy_logging_obj,
+        user_api_key_cache,
+        user_custom_key_update,
+    )
+
+    if prisma_client is None:
+        raise HTTPException(
+            status_code=500,
+            detail={"error": "Database not connected"},
+        )
+
+    if not data.team_id:
+        raise HTTPException(
+            status_code=400,
+            detail={"error": "team_id is required"},
+        )
+
+    MAX_BATCH_SIZE = 500
+    if data.key_ids is not None and len(data.key_ids) > MAX_BATCH_SIZE:
+        raise HTTPException(
+            status_code=400,
+            detail={
+                "error": f"Maximum {MAX_BATCH_SIZE} keys can be updated at once. Found {len(data.key_ids)} key_ids."
+            },
+        )
+
+    if data.all_keys_in_team:
+        # "all" excludes blocked/expired — bulk refresh shouldn't revive a key an admin disabled.
+        # `blocked` is Boolean? with no default; `/key/generate` writes NULL. Prisma's `NOT`
+        # excludes NULLs, so explicitly OR `false` with `null` to include them.
+        now = datetime.now(timezone.utc)
+        existing_keys = await prisma_client.db.litellm_verificationtoken.find_many(
+            where={
+                "team_id": data.team_id,
+                "AND": [
+                    {"OR": [{"blocked": False}, {"blocked": None}]},
+                    {"OR": [{"expires": None}, {"expires": {"gt": now}}]},
+                ],
+            },
+            order={"token": "asc"},
+            take=MAX_BATCH_SIZE + 1,
+        )
+        if len(existing_keys) > MAX_BATCH_SIZE:
+            raise HTTPException(
+                status_code=400,
+                detail={
+                    "error": f"Team {data.team_id} has more than {MAX_BATCH_SIZE} keys. Use `key_ids` to update in batches of {MAX_BATCH_SIZE}."
+                },
+            )
+        requested_tokens = [row.token for row in existing_keys]
+    else:
+        if data.key_ids is None or len(data.key_ids) == 0:
+            raise HTTPException(
+                status_code=400,
+                detail={
+                    "error": "key_ids must be provided when all_keys_in_team is False"
+                },
+            )
+        # Dedupe by hashed form — duplicates collapse to one update.
+        requested_tokens = []
+        hashed_key_ids = []
+        seen_hashes = set()
+        for k in data.key_ids:
+            h = _hash_token_if_needed(k)
+            if h in seen_hashes:
+                continue
+            seen_hashes.add(h)
+            requested_tokens.append(k)
+            hashed_key_ids.append(h)
+        existing_keys = await prisma_client.db.litellm_verificationtoken.find_many(
+            where={"team_id": data.team_id, "token": {"in": hashed_key_ids}}
+        )
+
+    # Anchor membership check on data.team_id (not existing_keys[0]); empty result must still gate non-admins.
+    if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value:
+        auth_anchor = (
+            existing_keys[0]
+            if existing_keys
+            else LiteLLM_VerificationToken(
+                token="__team_scope_auth_check__",
+                team_id=data.team_id,
+                models=[],
+            )
+        )
+        await TeamMemberPermissionChecks.can_team_member_execute_key_management_endpoint(
+            user_api_key_dict=user_api_key_dict,
+            route=KeyManagementRoutes.KEY_UPDATE,
+            prisma_client=prisma_client,
+            existing_key_row=auth_anchor,
+            user_api_key_cache=user_api_key_cache,
+        )
+
+    # Block metadata.allowed_passthrough_routes for non-admins — the runtime
+    # route checker reads it from key/team metadata to grant passthrough.
+    _check_passthrough_routes_caller_permission(
+        data=data.update_fields, user_api_key_dict=user_api_key_dict
+    )
+
+    if not requested_tokens:
+        raise HTTPException(
+            status_code=404,
+            detail={"error": f"No keys found for team {data.team_id}"},
+        )
+
+    existing_by_token = {row.token: row for row in existing_keys}
+    update_field_dict = data.update_fields.model_dump(exclude_unset=True)
+
+    successful_updates: List[SuccessfulKeyUpdate] = []
+    failed_updates: List[FailedKeyUpdate] = []
+
+    for token in requested_tokens:
+        db_token = _hash_token_if_needed(token)
+        try:
+            if db_token not in existing_by_token:
+                raise HTTPException(
+                    status_code=404,
+                    detail={"error": f"Key not found in team {data.team_id}"},
+                )
+
+            # team_id from validated scope, never user payload — drives _check_team_key_limits.
+            update_key_request = UpdateKeyRequest(
+                key=token,
+                team_id=data.team_id,
+                **update_field_dict,
+            )
+            updated_key_info = await _process_single_key_update(
+                update_key_request=update_key_request,
+                user_api_key_dict=user_api_key_dict,
+                litellm_changed_by=litellm_changed_by,
+                prisma_client=prisma_client,
+                user_api_key_cache=user_api_key_cache,
+                proxy_logging_obj=proxy_logging_obj,
+                llm_router=llm_router,
+                user_custom_key_update=user_custom_key_update,
+                existing_key_row=existing_by_token[db_token],
+            )
+
+            successful_updates.append(
+                SuccessfulKeyUpdate(key=token, key_info=updated_key_info)
+            )
+
+        except Exception as e:
+            # Log the hashed prefix — `token` may be a raw sk-... and ERROR logs persist.
+            verbose_proxy_logger.exception(
+                f"Failed to update key {db_token[:12]}... in team {data.team_id}: {e}"
+            )
+            failed_updates.append(
+                _build_failed_team_key_update(
+                    token=token,
+                    exception=e,
+                    existing_key_row=existing_by_token.get(db_token),
+                )
+            )
+
+    return BulkUpdateKeyResponse(
+        total_requested=len(requested_tokens),
+        successful_updates=successful_updates,
+        failed_updates=failed_updates,
+    )
+
+
 async def validate_key_team_change(
     key: LiteLLM_VerificationToken,
     team: LiteLLM_TeamTable,

litellm/proxy/management_endpoints/scim/scim_v2.py

@@ -4,6 +4,7 @@
 This is an enterprise feature and requires a premium license.
 """
 
+import re
 from typing import Any, Dict, List, Optional, Set, Tuple
 
 from fastapi import (
@@ -843,6 +844,18 @@ async def get_service_provider_config(request: Request):
     return SCIMServiceProviderConfig(meta=meta)
 
 
+def _parse_scim_eq_filter(scim_filter: str) -> Optional[Tuple[str, str]]:
+    """Parse the SCIM equality filters Okta uses before user lifecycle changes."""
+    match = re.match(
+        r"""\s*([\w.]+)\s+eq\s+(['"]?)(.*?)\2\s*$""",
+        scim_filter,
+        flags=re.IGNORECASE,
+    )
+    if not match:
+        return None
+    return match.group(1).lower(), match.group(3)
+
+
 # User Endpoints
 @scim_router.get(
     "/Users",
@@ -867,15 +880,21 @@ async def get_users(
     try:
         prisma_client = await _get_prisma_client_or_raise_exception()
         # Parse filter if provided (basic support)
-        where_conditions = {}
+        where_conditions: Dict[str, Any] = {}
         if filter:
-            # Very basic filter support - only handling userName eq and emails.value eq
-            if "userName eq" in filter:
-                user_id = filter.split("userName eq ")[1].strip("\"'")
-                where_conditions["user_id"] = user_id
-            elif "emails.value eq" in filter:
-                email = filter.split("emails.value eq ")[1].strip("\"'")
-                where_conditions["user_email"] = email
+            # Okta locates users by userName before deprovisioning. LiteLLM
+            # exposes SCIM userName from user_email, while older SCIM-created
+            # users may still have user_id == userName, so support both.
+            parsed_filter = _parse_scim_eq_filter(filter)
+            if parsed_filter:
+                filter_attribute, filter_value = parsed_filter
+                if filter_attribute == "username":
+                    where_conditions["OR"] = [
+                        {"user_email": filter_value},
+                        {"user_id": filter_value},
+                    ]
+                elif filter_attribute == "emails.value":
+                    where_conditions["user_email"] = filter_value
 
         # Get users from database
         users: List[LiteLLM_UserTable] = (

litellm/proxy/management_endpoints/tag_management_endpoints.py

@@ -12,16 +12,13 @@ All /tag management endpoints
 
 import asyncio
 import json
-from typing import TYPE_CHECKING, Dict, List, Optional, Union
+from datetime import datetime
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
-from fastapi import APIRouter, Depends, HTTPException
+from fastapi import APIRouter, Depends, HTTPException, Query
 
 from litellm._logging import verbose_proxy_logger
-from litellm.proxy._types import (
-    LitellmUserRoles,
-    UserAPIKeyAuth,
-    user_api_key_has_admin_view,
-)
+from litellm.proxy._types import UserAPIKeyAuth, user_api_key_has_admin_view
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.management_endpoints.common_daily_activity import (
     SpendAnalyticsPaginatedResponse,
@@ -43,20 +40,12 @@ if TYPE_CHECKING:
 router = APIRouter()
 
 
-def _is_internal_user_role(user_api_key_dict: UserAPIKeyAuth) -> bool:
-    return user_api_key_dict.user_role in (
-        LitellmUserRoles.INTERNAL_USER,
-        LitellmUserRoles.INTERNAL_USER.value,
-        LitellmUserRoles.INTERNAL_USER_VIEW_ONLY,
-        LitellmUserRoles.INTERNAL_USER_VIEW_ONLY.value,
-    )
-
-
 async def _get_internal_user_api_keys(
     prisma_client,
     user_api_key_dict: UserAPIKeyAuth,
 ) -> List[str]:
-    if not _is_internal_user_role(user_api_key_dict):
+    user_role = user_api_key_dict.user_role
+    if user_role is None or not user_role.is_internal_user_role:
         return []
 
     user_api_keys = set()
@@ -84,8 +73,9 @@ async def _get_tag_list_scope(
     prisma_client,
     user_api_key_dict: UserAPIKeyAuth,
 ) -> Optional[Dict[str, dict]]:
-    if user_api_key_has_admin_view(user_api_key_dict) or not _is_internal_user_role(
-        user_api_key_dict
+    user_role = user_api_key_dict.user_role
+    if user_api_key_has_admin_view(user_api_key_dict) or (
+        user_role is None or not user_role.is_internal_user_role
     ):
         return None
 
@@ -101,8 +91,9 @@ async def _get_tag_daily_activity_api_key_filter(
     user_api_key_dict: UserAPIKeyAuth,
     requested_api_key: Optional[str],
 ) -> Optional[Union[str, List[str]]]:
-    if user_api_key_has_admin_view(user_api_key_dict) or not _is_internal_user_role(
-        user_api_key_dict
+    user_role = user_api_key_dict.user_role
+    if user_api_key_has_admin_view(user_api_key_dict) or (
+        user_role is None or not user_role.is_internal_user_role
     ):
         return requested_api_key
 
@@ -471,6 +462,32 @@ async def info_tag(
         raise HTTPException(status_code=500, detail=str(e))
 
 
+def _validate_tag_list_date_range(
+    start_date: Optional[str], end_date: Optional[str]
+) -> None:
+    """Require both dates together, and enforce YYYY-MM-DD format with start <= end."""
+    if (start_date is None) != (end_date is None):
+        raise HTTPException(
+            status_code=400,
+            detail="start_date and end_date must be provided together",
+        )
+    if start_date is None:
+        return
+    try:
+        start = datetime.strptime(start_date, "%Y-%m-%d")
+        end = datetime.strptime(end_date, "%Y-%m-%d")  # type: ignore[arg-type]
+    except ValueError as e:
+        raise HTTPException(
+            status_code=400,
+            detail=f"Invalid date format, expected YYYY-MM-DD: {e}",
+        )
+    if start > end:
+        raise HTTPException(
+            status_code=400,
+            detail="start_date must be on or before end_date",
+        )
+
+
 @router.get(
     "/tag/list",
     tags=["tag management"],
@@ -478,6 +495,18 @@ async def info_tag(
 )
 async def list_tags(
     user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+    start_date: Optional[str] = Query(
+        None,
+        description=(
+            "Optional start date (YYYY-MM-DD). When provided together with "
+            "end_date, dynamic tags are limited to those active in the window. "
+            "Stored tags are always returned."
+        ),
+    ),
+    end_date: Optional[str] = Query(
+        None,
+        description="Optional end date (YYYY-MM-DD). Must be given with start_date.",
+    ),
 ):
     """
     List all available tags with their budget information.
@@ -487,6 +516,8 @@ async def list_tags(
     if prisma_client is None:
         raise HTTPException(status_code=500, detail="Database not connected")
 
+    _validate_tag_list_date_range(start_date, end_date)
+
     try:
         tag_scope = await _get_tag_list_scope(
             prisma_client=prisma_client,
@@ -498,9 +529,11 @@ async def list_tags(
         # Prisma's distinct fetches all columns for all rows and deduplicates
         # in application code, which is extremely slow on large tables.
         # See: https://www.prisma.io/docs/orm/prisma-client/queries/aggregation-grouping-summarizing#distinct-under-the-hood
-        dynamic_tag_where = {"tag": {"not": None}}
+        dynamic_tag_where: Dict[str, Any] = {"tag": {"not": None}}
         if tag_scope:
             dynamic_tag_where = {**dynamic_tag_where, **tag_scope}
+        if start_date is not None and end_date is not None:
+            dynamic_tag_where["date"] = {"gte": start_date, "lte": end_date}
 
         dynamic_tag_rows = await prisma_client.db.litellm_dailytagspend.group_by(
             by=["tag"],

litellm/proxy/management_endpoints/ui_sso.py

@@ -740,7 +740,7 @@ def generic_response_convertor(
 
     all_teams = []
     if sso_jwt_handler is not None:
-        team_ids = sso_jwt_handler.get_team_ids_from_jwt(cast(dict, response))
+        team_ids = sso_jwt_handler.get_all_jwt_team_ids(cast(dict, response))
         all_teams.extend(team_ids)
 
     if team_mappings is not None and team_mappings.team_ids_jwt_field is not None:
@@ -755,7 +755,7 @@ def generic_response_convertor(
                 f"Loaded team_ids from DB team_mappings.team_ids_jwt_field='{team_mappings.team_ids_jwt_field}': {team_ids_from_db_mapping}"
             )
     else:
-        team_ids = jwt_handler.get_team_ids_from_jwt(cast(dict, response))
+        team_ids = jwt_handler.get_all_jwt_team_ids(cast(dict, response))
         all_teams.extend(team_ids)
 
     # Determine user role based on role_mappings if available
@@ -4078,6 +4078,8 @@ async def debug_sso_callback(request: Request):
         redirect_url += "/sso/debug/callback"
 
     result = None
+    received_response: Optional[dict] = None
+    access_token_payload: Optional[dict] = None
     if google_client_id is not None:
         result = await GoogleSSOHandler.get_google_callback_response(
             request=request,
@@ -4094,12 +4096,14 @@ async def debug_sso_callback(request: Request):
         )
 
     elif generic_client_id is not None:
-        result, _, _ = await get_generic_sso_response(
-            request=request,
-            jwt_handler=jwt_handler,
-            generic_client_id=generic_client_id,
-            redirect_url=redirect_url,
-            sso_jwt_handler=sso_jwt_handler,
+        result, received_response, access_token_payload = (
+            await get_generic_sso_response(
+                request=request,
+                jwt_handler=jwt_handler,
+                generic_client_id=generic_client_id,
+                redirect_url=redirect_url,
+                sso_jwt_handler=sso_jwt_handler,
+            )
         )
 
     # If result is None, return a basic error message
@@ -4128,10 +4132,32 @@ async def debug_sso_callback(request: Request):
                 except Exception as e:
                     filtered_result[key] = f"Complex value (not displayable): {str(e)}"
 
+    # Defense-in-depth: ensure no bearer tokens leak into the rendered HTML even if
+    # a non-conforming IdP places them in its userinfo response.
+    safe_raw_claims = {
+        k: v
+        for k, v in (received_response or {}).items()
+        if k not in _OAUTH_TOKEN_FIELDS
+    }
+    safe_access_token_claims = {
+        k: v
+        for k, v in (access_token_payload or {}).items()
+        if k not in _OAUTH_TOKEN_FIELDS
+    }
+
+    sso_payload = {
+        "parsed_by_proxy": filtered_result,
+        "raw_claims": safe_raw_claims,
+        "access_token_claims": safe_access_token_claims,
+    }
+
     # Replace the placeholder in the template with the actual data
+    sso_payload_json = json.dumps(sso_payload, indent=2, default=str).replace(
+        "</", "<\\/"
+    )
     html_content = jwt_display_template.replace(
-        "const userData = SSO_DATA;",
-        f"const userData = {json.dumps(filtered_result, indent=2)};",
+        "const ssoData = SSO_DATA;",
+        f"const ssoData = {sso_payload_json};",
     )
 
     return HTMLResponse(content=html_content)

litellm/proxy/management_helpers/audit_logs.py

@@ -46,25 +46,46 @@ def get_audit_log_changed_by(
 
 
 def _resolve_audit_log_callback(name: str) -> Optional[CustomLogger]:
-    """Resolve a string callback name to a CustomLogger instance, with caching."""
+    """Resolve a string callback name to a CustomLogger instance, with caching.
+
+    For "s3_v2" with `litellm.s3_audit_callback_params` set, constructs a
+    dedicated `S3Logger` so audit logs can target a different bucket than the
+    normal-log singleton served by `_init_custom_logger_compatible_class`.
+    """
     if name in _audit_log_callback_cache:
         return _audit_log_callback_cache[name]
 
-    from litellm.litellm_core_utils.litellm_logging import (
-        _init_custom_logger_compatible_class,
-    )
+    instance: Optional[CustomLogger]
+    if (
+        name == "s3_v2"
+        and getattr(litellm, "s3_audit_callback_params", None) is not None
+    ):
+        from litellm.integrations.s3_v2 import S3Logger as S3V2Logger
 
-    instance = _init_custom_logger_compatible_class(
-        logging_integration=name,  # type: ignore
-        internal_usage_cache=None,
-        llm_router=None,
-    )
+        instance = S3V2Logger(
+            s3_callback_params_override=litellm.s3_audit_callback_params
+        )
+    else:
+        from litellm.litellm_core_utils.litellm_logging import (
+            _init_custom_logger_compatible_class,
+        )
+
+        instance = _init_custom_logger_compatible_class(
+            logging_integration=name,  # type: ignore
+            internal_usage_cache=None,
+            llm_router=None,
+        )
 
     if instance is not None:
         _audit_log_callback_cache[name] = instance
     return instance
 
 
+def reset_audit_log_callback_cache() -> None:
+    """Clear cached audit-log callback instances. Call on config reload."""
+    _audit_log_callback_cache.clear()
+
+
 def _build_audit_log_payload(
     request_data: LiteLLM_AuditLogs,
 ) -> StandardAuditLogPayload:

litellm/proxy/middleware/prometheus_auth_middleware.py

@@ -79,7 +79,10 @@ class PrometheusAuthMiddleware:
                 # Send 401 response directly via ASGI protocol
                 error_message = getattr(e, "message", str(e))
                 body = json.dumps(
-                    f"Unauthorized access to metrics endpoint: {error_message}"
+                    f"Unauthorized access to metrics endpoint: {error_message} "
+                    f"To allow unauthenticated access, set "
+                    f"`litellm_settings.require_auth_for_metrics_endpoint: false` "
+                    f"in your proxy_config.yaml."
                 ).encode("utf-8")
                 await send(
                     {

litellm/proxy/middleware/request_size_limit_middleware.py

@@ -0,0 +1,121 @@
+import json
+from typing import Callable, Optional, Union
+
+from starlette.types import ASGIApp, Message, Receive, Scope, Send
+
+MaxRequestSizeGetter = Callable[[], Optional[Union[int, float]]]
+RequestSizeLimitEnabledGetter = Callable[[], bool]
+
+
+class RequestEntityTooLarge(Exception):
+    pass
+
+
+class RequestSizeLimitMiddleware:
+    """
+    Reject oversized requests before downstream auth/routes parse the body.
+
+    Content-Length can be rejected without reading any body bytes. Requests
+    without Content-Length are counted as the ASGI stream is consumed, limiting
+    memory exposure to the configured threshold plus the current chunk.
+    """
+
+    def __init__(
+        self,
+        app: ASGIApp,
+        get_max_request_size_mb: MaxRequestSizeGetter,
+        is_request_size_limit_enabled: RequestSizeLimitEnabledGetter,
+    ) -> None:
+        self.app = app
+        self.get_max_request_size_mb = get_max_request_size_mb
+        self.is_request_size_limit_enabled = is_request_size_limit_enabled
+
+    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+        if scope["type"] != "http":
+            await self.app(scope, receive, send)
+            return
+
+        max_request_size_mb = self.get_max_request_size_mb()
+        max_request_size_bytes = _mb_to_bytes(max_request_size_mb)
+        if max_request_size_bytes is None or not self.is_request_size_limit_enabled():
+            await self.app(scope, receive, send)
+            return
+
+        content_length = _get_content_length(scope=scope)
+        if content_length is not None and content_length > max_request_size_bytes:
+            await _send_request_too_large(
+                send=send, max_request_size_mb=max_request_size_mb
+            )
+            return
+
+        received_body_bytes = 0
+        response_started = False
+
+        async def limited_receive() -> Message:
+            nonlocal received_body_bytes
+
+            message = await receive()
+            if message["type"] != "http.request":
+                return message
+
+            received_body_bytes += len(message.get("body", b""))
+            if received_body_bytes > max_request_size_bytes:
+                raise RequestEntityTooLarge
+            return message
+
+        async def tracking_send(message: Message) -> None:
+            nonlocal response_started
+
+            if message["type"] == "http.response.start":
+                response_started = True
+            await send(message)
+
+        try:
+            await self.app(scope, limited_receive, tracking_send)
+        except RequestEntityTooLarge:
+            if response_started:
+                raise
+            await _send_request_too_large(
+                send=send, max_request_size_mb=max_request_size_mb
+            )
+
+
+def _mb_to_bytes(max_request_size_mb: Optional[Union[int, float]]) -> Optional[int]:
+    if max_request_size_mb is None:
+        return None
+    if max_request_size_mb <= 0:
+        return None
+    return int(max_request_size_mb * 1024 * 1024)
+
+
+def _get_content_length(scope: Scope) -> Optional[int]:
+    headers = dict(scope.get("headers") or [])
+    raw_content_length = headers.get(b"content-length")
+    if raw_content_length is None:
+        return None
+
+    try:
+        return int(raw_content_length)
+    except ValueError:
+        return None
+
+
+async def _send_request_too_large(
+    send: Send,
+    max_request_size_mb: Optional[Union[int, float]],
+) -> None:
+    body = json.dumps(
+        {"error": f"Request size is too large. Max size is {max_request_size_mb} MB"},
+        separators=(",", ":"),
+    ).encode("utf-8")
+    await send(
+        {
+            "type": "http.response.start",
+            "status": 413,
+            "headers": [
+                (b"content-type", b"application/json"),
+                (b"content-length", str(len(body)).encode("latin-1")),
+            ],
+        }
+    )
+    await send({"type": "http.response.body", "body": body, "more_body": False})