Merge remote-tracking branch 'origin/main' into litellm_fix-alert-type-toggle

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

# Conflicts:
#	ui/litellm-dashboard/src/components/settings.test.tsx

.github/workflows/osv-scan.yml

@@ -6,6 +6,8 @@ on:
       - main
       - "litellm_**"
       - "rc/**"
+      - "stable/**"
+      - "lts/**"
   schedule:
     - cron: "23 6 * * *"
   workflow_dispatch:

.github/workflows/test-unit.yml

@@ -5,6 +5,9 @@ on:
     branches:
       - main
       - "litellm_**"
+      - "rc/**"
+      - "stable/**"
+      - "lts/**"
   push:
     branches:
       - main
@@ -280,6 +283,7 @@ jobs:
               tests/unit/llms/e2b
               tests/unit/llms/edenai
               tests/unit/llms/elevenlabs
+              tests/unit/llms/empiriolabs
               tests/unit/llms/exa_ai
               tests/unit/llms/fal_ai
               tests/unit/llms/fastcrw
@@ -973,10 +977,10 @@ jobs:
             dist: loadscope
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        timeout-minutes: ${{ matrix.shard == 'db-and-spend' && 10 || 3 }}
+        timeout-minutes: 3
         with:
           persist-credentials: false
-          fetch-depth: ${{ matrix.shard == 'db-and-spend' && '0' || '1' }}
+          fetch-depth: 1
 
       - name: Detect relevant changes
         id: changes
@@ -1009,13 +1013,11 @@ jobs:
         with:
           artifact: ${{ needs.rust-bridge.outputs.artifact }}
 
-      - name: Install dependencies and check schema
+      - name: Install dependencies
         if: steps.changes.outputs.decision != 'skip'
-        timeout-minutes: 8
+        timeout-minutes: 5
         env:
           RUST_BRIDGE_ARTIFACT: ${{ needs.rust-bridge.outputs.artifact }}
-          UNIT_SHARD: ${{ matrix.shard }}
-          SCHEMA_BASE_REF: origin/${{ github.event.repository.default_branch }}
         run: |
           diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
           if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
@@ -1027,9 +1029,21 @@ jobs:
             uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))"
           fi
           uv run --no-sync python -c 'import os, sys; print(sys.version); assert f"{sys.version_info.major}.{sys.version_info.minor}" == os.environ["UV_PYTHON"]'
-          if [ "$UNIT_SHARD" = "db-and-spend" ]; then
-            uv run --no-sync python scripts/check_schema_compatibility.py --base-ref "$SCHEMA_BASE_REF"
-          fi
+
+      - name: Check schema.prisma against the merge base
+        if: steps.changes.outputs.decision != 'skip' && matrix.shard == 'db-and-spend' && github.event_name == 'pull_request'
+        timeout-minutes: 3
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
+          merge_base=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${base_ref}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
+          test -n "$merge_base"
+          git fetch --no-tags --depth=1 origin "$merge_base"
+          uv run --no-sync python scripts/check_schema_compatibility.py --base-commit "$merge_base"
 
       - name: Cache Prisma binaries
         if: steps.changes.outputs.decision != 'skip'

README.md

@@ -398,6 +398,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
 | [Deepseek (`deepseek`)](https://docs.litellm.ai/docs/providers/deepseek) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Eden AI (`edenai`)](https://docs.litellm.ai/docs/providers/edenai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |  |
 | [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ |  |  | ✅ | ✅ |  |  |  |
+| [EmpirioLabs AI (`empiriolabs`)](https://docs.litellm.ai/docs/providers/empiriolabs) | ✅ |  | ✅ |  |  |  |  |  |  |  |
 | [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [eu/jev Decisions API (`eu_jev`)](https://docs.litellm.ai/docs/pass_through/eu_jev) |  |  |  |  |  |  |  |  |  |  |
 | [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ |  | ✅ |  |  |  |  |  |

enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py

@@ -200,7 +200,6 @@ async def log_billed_background_response(billed: BilledBackgroundResponse) -> Bi
     unpriced: Final = _without_read_price(billed.response)
     cost: Final = logging_obj.response_cost_calculator(result=unpriced)
     end_time: Final = datetime.now()
-    logging_obj.capture_usage_before_logging(unpriced, logging_obj.start_time, end_time)
     GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
         async_coroutine=logging_obj.async_success_handler(
             result=unpriced, start_time=logging_obj.start_time, end_time=end_time

helm/litellm-helm/Chart.yaml

@@ -18,7 +18,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.1.3
+version: 1.1.4
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm/litellm-helm/README.md

@@ -56,8 +56,8 @@ If `db.useStackgresOperator` is used (not yet implemented):
 
 | `billingMetrics.enabled`    | Enable enterprise billable-request metering. Requires an enterprise license.                                                                                                                                                                  | `false`                   |
 | `billingMetrics.endpoint`   | Collector that the billable-request counter is pushed to.                                                                                                                                                                                    | `https://telemetry.litellm.ai` |
-| `billingMetrics.secretName` | Name of an existing Secret holding the mTLS client certificate, under the keys `tls.crt` and `tls.key`.                                                                                                                                       | `litellm-billing-metrics-mtls` |
-| `billingMetrics.caSecretName` | Name of an existing Secret holding a CA bundle under the key `ca.crt`. Only needed for a private or test collector whose server certificate is not on the public web PKI.                                                                    | `""`                      |
+| `billingMetrics.tokenUrl`   | Cognito OAuth2 client-credentials token endpoint.                                                                                                                                                                                               | `https://litellm-prod.auth.us-west-1.amazoncognito.com/oauth2/token` |
+| `billingMetrics.secretName` | Name of an existing Secret with `client-id` and `client-secret` keys.                                                                                                                                                                           | `litellm-billing-metrics` |
 | `billingMetrics.exportIntervalMs` | How often the counter is pushed, in milliseconds. The proxy defaults to `60000` when unset.                                                                                                                                             | `""`                      |
 
 #### Example `proxy_config` ConfigMap from values (default):
@@ -109,10 +109,10 @@ type: Opaque
 
 #### Enterprise billable-request metering
 
-Enterprise licenses meter billable requests by pushing a counter to LiteLLM's collector over mutual TLS. The chart does not create the client certificate; it mounts one you already hold, read-only, so the private key is never exposed through the environment. Create the Secret under the name the chart expects, then turn the block on:
+Enterprise licenses meter billable requests by pushing a counter to LiteLLM's collector with Cognito bearer authentication. Create a Secret with `client-id` and `client-secret` keys, then enable the chart values:
 
 ```
-kubectl create secret tls litellm-billing-metrics-mtls --cert=client.crt --key=client.key
+kubectl create secret generic litellm-billing-metrics --from-literal=client-id=... --from-literal=client-secret=...
 ```
 
 ```
@@ -120,7 +120,8 @@ billingMetrics:
   enabled: true
 ```
 
-Set `billingMetrics.caSecretName` only when the collector is a private or test one whose server certificate is not on the public web PKI; the production collector needs no CA override. The chart fails the render rather than deploying a proxy that silently never exports, so a missing `secretName` or an emptied `endpoint` surfaces at `helm install` time.
+The endpoint and token URL must use HTTPS. The chart requires both URLs and
+the Secret name when billing metrics are enabled
 
 ### Database Settings
 

helm/litellm-helm/templates/_helpers.tpl

@@ -50,53 +50,27 @@ app.kubernetes.io/name: {{ include "litellm.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
 
-{{/*
-Enterprise billable-request metering. The client certificate identifies the
-deployment to LiteLLM's collector, so it is mounted read-only from an existing
-Secret rather than passed through the environment.
-*/}}
-{{- define "litellm.billingMetrics.certDir" -}}/etc/litellm/billing-mtls{{- end -}}
-{{- define "litellm.billingMetrics.caDir" -}}/etc/litellm/billing-mtls-ca{{- end -}}
-
 {{- define "litellm.billingMetricsEnv" -}}
 - name: LITELLM_BILLING_METRICS_ENDPOINT
   value: {{ required "billingMetrics.endpoint is required when billingMetrics.enabled is true" .Values.billingMetrics.endpoint | quote }}
-- name: LITELLM_BILLING_METRICS_CLIENT_CERT
-  value: {{ printf "%s/tls.crt" (include "litellm.billingMetrics.certDir" .) | quote }}
-- name: LITELLM_BILLING_METRICS_CLIENT_KEY
-  value: {{ printf "%s/tls.key" (include "litellm.billingMetrics.certDir" .) | quote }}
-{{- if .Values.billingMetrics.caSecretName }}
-- name: LITELLM_BILLING_METRICS_CA_CERT
-  value: {{ printf "%s/ca.crt" (include "litellm.billingMetrics.caDir" .) | quote }}
-{{- end }}
+- name: LITELLM_BILLING_METRICS_TOKEN_URL
+  value: {{ required "billingMetrics.tokenUrl is required when billingMetrics.enabled is true" .Values.billingMetrics.tokenUrl | quote }}
+- name: LITELLM_BILLING_METRICS_CLIENT_ID
+  valueFrom:
+    secretKeyRef:
+      name: {{ required "billingMetrics.secretName is required when billingMetrics.enabled is true" .Values.billingMetrics.secretName }}
+      key: client-id
+- name: LITELLM_BILLING_METRICS_CLIENT_SECRET
+  valueFrom:
+    secretKeyRef:
+      name: {{ required "billingMetrics.secretName is required when billingMetrics.enabled is true" .Values.billingMetrics.secretName }}
+      key: client-secret
 {{- with .Values.billingMetrics.exportIntervalMs }}
 - name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS
   value: {{ . | quote }}
 {{- end }}
 {{- end -}}
 
-{{- define "litellm.billingMetricsVolumes" -}}
-- name: billing-metrics-mtls
-  secret:
-    secretName: {{ required "billingMetrics.secretName is required when billingMetrics.enabled is true (an existing Secret with tls.crt and tls.key)" .Values.billingMetrics.secretName }}
-{{- if .Values.billingMetrics.caSecretName }}
-- name: billing-metrics-mtls-ca
-  secret:
-    secretName: {{ .Values.billingMetrics.caSecretName }}
-{{- end }}
-{{- end -}}
-
-{{- define "litellm.billingMetricsVolumeMounts" -}}
-- name: billing-metrics-mtls
-  mountPath: {{ include "litellm.billingMetrics.certDir" . }}
-  readOnly: true
-{{- if .Values.billingMetrics.caSecretName }}
-- name: billing-metrics-mtls-ca
-  mountPath: {{ include "litellm.billingMetrics.caDir" . }}
-  readOnly: true
-{{- end }}
-{{- end -}}
-
 {{/*
 Create the name of the service account to use
 */}}

helm/litellm-helm/templates/deployment.yaml

@@ -148,9 +148,6 @@ spec:
             - name: npm
               mountPath: /.npm
           {{- end }}
-          {{- if .Values.billingMetrics.enabled }}
-            {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }}
-          {{- end }}
           {{- if include "litellm.collector.socketDir" . }}
             - name: collector-socket
               mountPath: {{ include "litellm.collector.socketDir" . }}
@@ -234,9 +231,6 @@ spec:
             items:
               - key: {{ .Values.proxyConfigMap.key | default "config.yaml" }}
                 path: "config.yaml"
-        {{- if .Values.billingMetrics.enabled }}
-        {{- include "litellm.billingMetricsVolumes" . | nindent 8 }}
-        {{- end }}
         {{- if include "litellm.collector.socketDir" . }}
         - name: collector-socket
           emptyDir:

helm/litellm-helm/tests/billing_metrics_tests.yaml

@@ -1,31 +1,21 @@
-suite: test billingMetrics wiring on the proxy deployment
+suite: test bearer-only billingMetrics wiring
 templates:
   - deployment.yaml
   - configmap-litellm.yaml
-  - migrations-job.yaml
 tests:
-  - it: is off by default, adding no env, volume, or mount
+  - it: is disabled by default
     template: deployment.yaml
     asserts:
-      - notContains:
-          path: spec.template.spec.volumes
-          content:
-            name: billing-metrics-mtls
-            secret:
-              secretName: litellm-billing-metrics-mtls
-      - notContains:
-          path: spec.template.spec.containers[0].volumeMounts
-          content:
-            name: billing-metrics-mtls
-            mountPath: /etc/litellm/billing-mtls
-            readOnly: true
       - notContains:
           path: spec.template.spec.containers[0].env
           content:
             name: LITELLM_BILLING_METRICS_ENDPOINT
-            value: https://telemetry.litellm.ai
+      - notContains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics
 
-  - it: renders the endpoint and the mounted cert paths when enabled
+  - it: renders required URLs and Cognito secret references when enabled
     template: deployment.yaml
     set:
       billingMetrics:
@@ -39,119 +29,34 @@ tests:
       - contains:
           path: spec.template.spec.containers[0].env
           content:
-            name: LITELLM_BILLING_METRICS_CLIENT_CERT
-            value: /etc/litellm/billing-mtls/tls.crt
+            name: LITELLM_BILLING_METRICS_TOKEN_URL
+            value: https://litellm-prod.auth.us-west-1.amazoncognito.com/oauth2/token
       - contains:
           path: spec.template.spec.containers[0].env
           content:
-            name: LITELLM_BILLING_METRICS_CLIENT_KEY
-            value: /etc/litellm/billing-mtls/tls.key
-
-  # The conventional Secret name is the default, so enabling the block is enough.
-  - it: mounts the default cert secret read-only alongside the config volume
-    template: deployment.yaml
-    set:
-      billingMetrics:
-        enabled: true
-    asserts:
-      - contains:
-          path: spec.template.spec.volumes
-          content:
-            name: billing-metrics-mtls
-            secret:
-              secretName: litellm-billing-metrics-mtls
-      - contains:
-          path: spec.template.spec.containers[0].volumeMounts
-          content:
-            name: billing-metrics-mtls
-            mountPath: /etc/litellm/billing-mtls
-            readOnly: true
-
-  - it: honours a secretName override
-    template: deployment.yaml
-    set:
-      billingMetrics:
-        enabled: true
-        secretName: my-billing-mtls
-    asserts:
-      - contains:
-          path: spec.template.spec.volumes
-          content:
-            name: billing-metrics-mtls
-            secret:
-              secretName: my-billing-mtls
-      - notContains:
-          path: spec.template.spec.volumes
-          content:
-            name: billing-metrics-mtls
-            secret:
-              secretName: litellm-billing-metrics-mtls
-
-  - it: honours an endpoint override
-    template: deployment.yaml
-    set:
-      billingMetrics:
-        enabled: true
-        endpoint: https://collector.internal:4318
-    asserts:
+            name: LITELLM_BILLING_METRICS_CLIENT_ID
+            valueFrom:
+              secretKeyRef:
+                name: litellm-billing-metrics
+                key: client-id
       - contains:
           path: spec.template.spec.containers[0].env
           content:
-            name: LITELLM_BILLING_METRICS_ENDPOINT
-            value: https://collector.internal:4318
-
-  # The production collector presents a public web-PKI certificate, so the CA
-  # override must stay absent unless a private collector is configured.
-  - it: omits the CA env, volume, and mount when no caSecretName is set
-    template: deployment.yaml
-    set:
-      billingMetrics:
-        enabled: true
-    asserts:
+            name: LITELLM_BILLING_METRICS_CLIENT_SECRET
+            valueFrom:
+              secretKeyRef:
+                name: litellm-billing-metrics
+                key: client-secret
       - notContains:
           path: spec.template.spec.volumes
           content:
-            name: billing-metrics-mtls-ca
-            secret:
-              secretName: billing-ca
+            name: billing-metrics
       - notContains:
           path: spec.template.spec.containers[0].volumeMounts
           content:
-            name: billing-metrics-mtls-ca
-            mountPath: /etc/litellm/billing-mtls-ca
-            readOnly: true
-      - notContains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_BILLING_METRICS_CA_CERT
-            value: /etc/litellm/billing-mtls-ca/ca.crt
+            name: billing-metrics
 
-  - it: mounts the CA secret when caSecretName is set
-    template: deployment.yaml
-    set:
-      billingMetrics:
-        enabled: true
-        caSecretName: billing-ca
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_BILLING_METRICS_CA_CERT
-            value: /etc/litellm/billing-mtls-ca/ca.crt
-      - contains:
-          path: spec.template.spec.volumes
-          content:
-            name: billing-metrics-mtls-ca
-            secret:
-              secretName: billing-ca
-      - contains:
-          path: spec.template.spec.containers[0].volumeMounts
-          content:
-            name: billing-metrics-mtls-ca
-            mountPath: /etc/litellm/billing-mtls-ca
-            readOnly: true
-
-  - it: passes the export interval through only when set
+  - it: renders the export interval only when set
     template: deployment.yaml
     set:
       billingMetrics:
@@ -164,129 +69,7 @@ tests:
             name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS
             value: "5000"
 
-  - it: omits the export interval when unset
-    template: deployment.yaml
-    set:
-      billingMetrics:
-        enabled: true
-    asserts:
-      - notContains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS
-            value: "60000"
-
-  # Kubernetes resolves duplicate env names last-wins, so the chart-owned billing
-  # entries must render after .Values.envVars or a user could silently redirect
-  # the metering export. The three billing entries are the last ones emitted here
-  # (migrationJob, which appends DISABLE_SCHEMA_UPDATE, is off for this case).
-  - it: renders the billing endpoint after envVars so it cannot be shadowed
-    template: deployment.yaml
-    set:
-      migrationJob:
-        enabled: false
-      billingMetrics:
-        enabled: true
-      envVars:
-        LITELLM_BILLING_METRICS_ENDPOINT: https://shadowed.example
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_BILLING_METRICS_ENDPOINT
-            value: https://shadowed.example
-      - equal:
-          path: spec.template.spec.containers[0].env[-3]
-          value:
-            name: LITELLM_BILLING_METRICS_ENDPOINT
-            value: https://telemetry.litellm.ai
-      - equal:
-          path: spec.template.spec.containers[0].env[-2].name
-          value: LITELLM_BILLING_METRICS_CLIENT_CERT
-      - equal:
-          path: spec.template.spec.containers[0].env[-1].name
-          value: LITELLM_BILLING_METRICS_CLIENT_KEY
-
-  - it: keeps user-supplied volumes and mounts alongside the billing secret
-    template: deployment.yaml
-    set:
-      billingMetrics:
-        enabled: true
-      volumes:
-        - name: custom-callbacks
-          configMap:
-            name: my-callbacks
-      volumeMounts:
-        - name: custom-callbacks
-          mountPath: /app/callbacks
-    asserts:
-      - contains:
-          path: spec.template.spec.volumes
-          content:
-            name: custom-callbacks
-            configMap:
-              name: my-callbacks
-      - contains:
-          path: spec.template.spec.volumes
-          content:
-            name: billing-metrics-mtls
-            secret:
-              secretName: litellm-billing-metrics-mtls
-      - contains:
-          path: spec.template.spec.containers[0].volumeMounts
-          content:
-            name: custom-callbacks
-            mountPath: /app/callbacks
-      - contains:
-          path: spec.template.spec.containers[0].volumeMounts
-          content:
-            name: billing-metrics-mtls
-            mountPath: /etc/litellm/billing-mtls
-            readOnly: true
-
-  - it: still mounts the proxy config when enabled
-    template: deployment.yaml
-    set:
-      billingMetrics:
-        enabled: true
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].volumeMounts
-          content:
-            name: litellm-config
-            mountPath: /etc/litellm/config.yaml
-            subPath: config.yaml
-
-  # Only the proxy serves billable traffic. The migrations Job must never mount
-  # the client certificate, and it renders its own env and volumes, so nothing
-  # stops a future edit from wiring the billing include into it by mistake.
-  - it: does not touch the migrations job when enabled
-    template: migrations-job.yaml
-    set:
-      billingMetrics:
-        enabled: true
-    asserts:
-      - notContains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_BILLING_METRICS_ENDPOINT
-            value: https://telemetry.litellm.ai
-      - notExists:
-          path: spec.template.spec.containers[0].volumeMounts
-      - notExists:
-          path: spec.template.spec.volumes
-
-  - it: fails loudly when enabled with an emptied secretName
-    template: deployment.yaml
-    set:
-      billingMetrics:
-        enabled: true
-        secretName: ""
-    asserts:
-      - failedTemplate:
-          errorMessage: billingMetrics.secretName is required when billingMetrics.enabled is true (an existing Secret with tls.crt and tls.key)
-
-  - it: fails loudly when enabled without an endpoint
+  - it: requires the endpoint when enabled
     template: deployment.yaml
     set:
       billingMetrics:
@@ -295,3 +78,23 @@ tests:
     asserts:
       - failedTemplate:
           errorMessage: billingMetrics.endpoint is required when billingMetrics.enabled is true
+
+  - it: requires the token URL when enabled
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        tokenUrl: ""
+    asserts:
+      - failedTemplate:
+          errorMessage: billingMetrics.tokenUrl is required when billingMetrics.enabled is true
+
+  - it: requires the secret name when enabled
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        secretName: ""
+    asserts:
+      - failedTemplate:
+          errorMessage: billingMetrics.secretName is required when billingMetrics.enabled is true

helm/litellm-helm/tests/collector_tests.yaml

@@ -191,7 +191,7 @@ tests:
       metricsServer.port: 9090
       billingMetrics.enabled: true
       billingMetrics.endpoint: https://metering.example.com
-      billingMetrics.secretName: billing-mtls
+      billingMetrics.secretName: billing-metrics
     asserts:
       - contains:
           path: spec.template.spec.containers[0].env
@@ -214,9 +214,9 @@ tests:
             name: LITELLM_BILLING_METRICS_ENDPOINT
           any: true
       - notContains:
-          path: spec.template.spec.containers[1].volumeMounts
+          path: spec.template.spec.containers[1].env
           content:
-            name: billing-metrics-mtls
+            name: LITELLM_BILLING_METRICS_CLIENT_SECRET
           any: true
 
   - it: should give the sidecar the same scratch mounts as the proxy on a read-only root

helm/litellm-helm/values.yaml

@@ -154,19 +154,13 @@ masterkeySecretName: ""
 # if set, use this secret key for the master key; otherwise, use the default key
 masterkeySecretKey: ""
 
-# Optional: enterprise billable-request metering. When enabled, the proxy counts
-# successful requests to inference, MCP, and A2A endpoints and pushes them to
-# LiteLLM's collector over mutual TLS. Requires an enterprise license.
-# The client certificate identifies the deployment, so it is mounted read-only
-# from an existing Secret and never passed through the environment.
+# Optional: enterprise billable-request metering. Requires an enterprise license.
 billingMetrics:
   enabled: false
-  endpoint: https://telemetry.litellm.ai   # collector to push the counter to
-  secretName: litellm-billing-metrics-mtls # existing Secret holding tls.crt and tls.key
-  # Only for private or test collectors whose server certificate is not on the
-  # public web PKI. The production collector needs no CA override.
-  caSecretName: ""                         # existing Secret holding ca.crt
-  exportIntervalMs: ""                     # push cadence; the proxy defaults to 60000
+  endpoint: https://telemetry.litellm.ai
+  tokenUrl: https://litellm-prod.auth.us-west-1.amazoncognito.com/oauth2/token
+  secretName: litellm-billing-metrics
+  exportIntervalMs: ""
 
 proxyConfigMap:
   # when true, creates a new configmap

helm/litellm/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: litellm
 description: LiteLLM componentized — gateway, UI backend, and UI as separate services
 type: application
-version: 0.1.0
+version: 0.1.1
 appVersion: "0.1.0"
 annotations:
   org.opencontainers.image.source: "https://github.com/BerriAI/litellm"

helm/litellm/README.md

@@ -22,6 +22,23 @@ backend:
   gatewayUrl: https://gateway.example.com
 ```
 
+## Enterprise billing metrics
+
+Billing metrics use Cognito bearer authentication. Create the Secret with
+`client-id` and `client-secret` keys, then enable billing metrics:
+
+```bash
+kubectl create secret generic litellm-billing-metrics --from-literal=client-id=... --from-literal=client-secret=...
+```
+
+```yaml
+billingMetrics:
+  enabled: true
+```
+
+The chart requires an HTTPS endpoint, an HTTPS token URL, and the Secret name
+when billing metrics are enabled
+
 ## Routes to another Service
 
 For a shared hostname, `ingress.extraPaths` can route to a Service in the same namespace using `serviceName` and `servicePort`. The port can be a number or a named Service port. Omit `service` when using these fields

helm/litellm/templates/_helpers.tpl

@@ -58,56 +58,27 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
 helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }}
 {{- end -}}
 
-{{/*
-Enterprise billable-request metering. Wired into gateway and backend, not the
-migrations job. The gateway serves nearly all billable traffic, but the backend
-keeps the named-server MCP transport (/{mcp_server_name}/mcp), which writes a
-SpendLogs row, so metering only the gateway would silently drop that traffic.
-The client certificate identifies the deployment to LiteLLM's collector, so it is
-mounted read-only from an existing Secret rather than passed through the
-environment.
-*/}}
-{{- define "litellm.billingMetrics.certDir" -}}/etc/litellm/billing-mtls{{- end -}}
-{{- define "litellm.billingMetrics.caDir" -}}/etc/litellm/billing-mtls-ca{{- end -}}
-
 {{- define "litellm.billingMetricsEnv" -}}
 - name: LITELLM_BILLING_METRICS_ENDPOINT
   value: {{ required "billingMetrics.endpoint is required when billingMetrics.enabled is true" .Values.billingMetrics.endpoint | quote }}
-- name: LITELLM_BILLING_METRICS_CLIENT_CERT
-  value: {{ printf "%s/tls.crt" (include "litellm.billingMetrics.certDir" .) | quote }}
-- name: LITELLM_BILLING_METRICS_CLIENT_KEY
-  value: {{ printf "%s/tls.key" (include "litellm.billingMetrics.certDir" .) | quote }}
-{{- if .Values.billingMetrics.caSecretName }}
-- name: LITELLM_BILLING_METRICS_CA_CERT
-  value: {{ printf "%s/ca.crt" (include "litellm.billingMetrics.caDir" .) | quote }}
-{{- end }}
+- name: LITELLM_BILLING_METRICS_TOKEN_URL
+  value: {{ required "billingMetrics.tokenUrl is required when billingMetrics.enabled is true" .Values.billingMetrics.tokenUrl | quote }}
+- name: LITELLM_BILLING_METRICS_CLIENT_ID
+  valueFrom:
+    secretKeyRef:
+      name: {{ required "billingMetrics.secretName is required when billingMetrics.enabled is true" .Values.billingMetrics.secretName }}
+      key: client-id
+- name: LITELLM_BILLING_METRICS_CLIENT_SECRET
+  valueFrom:
+    secretKeyRef:
+      name: {{ required "billingMetrics.secretName is required when billingMetrics.enabled is true" .Values.billingMetrics.secretName }}
+      key: client-secret
 {{- with .Values.billingMetrics.exportIntervalMs }}
 - name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS
   value: {{ . | quote }}
 {{- end }}
 {{- end -}}
 
-{{- define "litellm.billingMetricsVolumes" -}}
-- name: billing-metrics-mtls
-  secret:
-    secretName: {{ required "billingMetrics.secretName is required when billingMetrics.enabled is true (an existing Secret with tls.crt and tls.key)" .Values.billingMetrics.secretName }}
-{{- if .Values.billingMetrics.caSecretName }}
-- name: billing-metrics-mtls-ca
-  secret:
-    secretName: {{ .Values.billingMetrics.caSecretName }}
-{{- end }}
-{{- end -}}
-
-{{- define "litellm.billingMetricsVolumeMounts" -}}
-- name: billing-metrics-mtls
-  mountPath: {{ include "litellm.billingMetrics.certDir" . }}
-  readOnly: true
-{{- if .Values.billingMetrics.caSecretName }}
-- name: billing-metrics-mtls-ca
-  mountPath: {{ include "litellm.billingMetrics.caDir" . }}
-  readOnly: true
-{{- end }}
-{{- end -}}
 
 {{/*
 Per-component selector labels — used in both Service selectors and Deployment matchLabels.

helm/litellm/templates/backend/deployment.yaml

@@ -79,16 +79,13 @@ spec:
             {{- include "litellm.billingMetricsEnv" . | nindent 12 }}
             {{- end }}
           {{- include "litellm.envFrom" .Values.backend | nindent 10 }}
-          {{- if or .Values.gateway.config.create .Values.backend.volumeMounts .Values.billingMetrics.enabled }}
+          {{- if or .Values.gateway.config.create .Values.backend.volumeMounts }}
           volumeMounts:
             {{- if .Values.gateway.config.create }}
             - name: gateway-config
               mountPath: /app/config/config.yaml
               subPath: config.yaml
             {{- end }}
-            {{- if .Values.billingMetrics.enabled }}
-            {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }}
-            {{- end }}
             {{- with .Values.backend.volumeMounts }}
             {{- toYaml . | nindent 12 }}
             {{- end }}
@@ -114,16 +111,13 @@ spec:
       {{- with .Values.backend.extraContainers }}
         {{- tpl (toYaml .) $ | nindent 8 }}
       {{- end }}
-      {{- if or .Values.gateway.config.create .Values.backend.volumes .Values.billingMetrics.enabled }}
+      {{- if or .Values.gateway.config.create .Values.backend.volumes }}
       volumes:
         {{- if .Values.gateway.config.create }}
         - name: gateway-config
           configMap:
             name: {{ include "litellm.gateway.fullname" . }}-config
         {{- end }}
-        {{- if .Values.billingMetrics.enabled }}
-        {{- include "litellm.billingMetricsVolumes" . | nindent 8 }}
-        {{- end }}
         {{- with .Values.backend.volumes }}
         {{- toYaml . | nindent 8 }}
         {{- end }}

helm/litellm/templates/gateway/deployment.yaml

@@ -86,7 +86,7 @@ spec:
             {{- include "litellm.gateway.collectorEnv" . | nindent 12 }}
             {{- end }}
           {{- include "litellm.envFrom" .Values.gateway | nindent 10 }}
-          {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled (include "litellm.gateway.collectorSocketDir" .) }}
+          {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.gateway.metricsServer.enabled (include "litellm.gateway.collectorSocketDir" .) }}
           volumeMounts:
             {{- if .Values.gateway.config.create }}
             - name: gateway-config
@@ -101,9 +101,6 @@ spec:
             - name: collector-socket
               mountPath: {{ include "litellm.gateway.collectorSocketDir" . }}
             {{- end }}
-            {{- if .Values.billingMetrics.enabled }}
-            {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }}
-            {{- end }}
             {{- with .Values.gateway.volumeMounts }}
             {{- toYaml . | nindent 12 }}
             {{- end }}
@@ -206,7 +203,7 @@ spec:
       {{- with .Values.gateway.extraContainers }}
         {{- tpl (toYaml .) $ | nindent 8 }}
       {{- end }}
-      {{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled (include "litellm.gateway.collectorSocketDir" .) }}
+      {{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.gateway.metricsServer.enabled (include "litellm.gateway.collectorSocketDir" .) }}
       volumes:
         {{- if .Values.gateway.config.create }}
         - name: gateway-config
@@ -222,9 +219,6 @@ spec:
           emptyDir:
             sizeLimit: 1Mi
         {{- end }}
-        {{- if .Values.billingMetrics.enabled }}
-        {{- include "litellm.billingMetricsVolumes" . | nindent 8 }}
-        {{- end }}
         {{- with .Values.gateway.volumes }}
         {{- toYaml . | nindent 8 }}
         {{- end }}

helm/litellm/tests/billing_metrics_tests.yaml

@@ -1,39 +1,30 @@
-suite: test billingMetrics wiring on gateway and backend
+suite: test bearer-only billingMetrics wiring
 templates:
   - gateway/deployment.yaml
   - gateway/configmap.yaml
   - backend/deployment.yaml
+  - backend/configmap.yaml
   - migrations-job.yaml
 values:
   - ./values/required.yaml
 tests:
-  - it: is off by default, adding no env, volume, or mount
+  - it: is disabled by default
     template: gateway/deployment.yaml
     asserts:
-      - notContains:
-          path: spec.template.spec.volumes
-          content:
-            name: billing-metrics-mtls
-            secret:
-              secretName: billing-mtls
-      - equal:
-          path: spec.template.spec.containers[0].volumeMounts
-          value:
-            - name: gateway-config
-              mountPath: /app/config/config.yaml
-              subPath: config.yaml
       - notContains:
           path: spec.template.spec.containers[0].env
           content:
             name: LITELLM_BILLING_METRICS_ENDPOINT
-            value: https://telemetry.litellm.ai
+      - notContains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics
 
-  - it: renders the endpoint and the mounted cert paths when enabled
+  - it: wires gateway URLs and bearer credentials from Secret keys
     template: gateway/deployment.yaml
     set:
       billingMetrics:
         enabled: true
-        secretName: billing-mtls
     asserts:
       - contains:
           path: spec.template.spec.containers[0].env
@@ -43,87 +34,68 @@ tests:
       - contains:
           path: spec.template.spec.containers[0].env
           content:
-            name: LITELLM_BILLING_METRICS_CLIENT_CERT
-            value: /etc/litellm/billing-mtls/tls.crt
+            name: LITELLM_BILLING_METRICS_TOKEN_URL
+            value: https://litellm-prod.auth.us-west-1.amazoncognito.com/oauth2/token
       - contains:
           path: spec.template.spec.containers[0].env
           content:
-            name: LITELLM_BILLING_METRICS_CLIENT_KEY
-            value: /etc/litellm/billing-mtls/tls.key
-
-  - it: mounts the cert secret read-only alongside the config volume
-    template: gateway/deployment.yaml
-    set:
-      billingMetrics:
-        enabled: true
-        secretName: billing-mtls
-    asserts:
+            name: LITELLM_BILLING_METRICS_CLIENT_ID
+            valueFrom:
+              secretKeyRef:
+                name: litellm-billing-metrics
+                key: client-id
       - contains:
-          path: spec.template.spec.volumes
+          path: spec.template.spec.containers[0].env
           content:
-            name: billing-metrics-mtls
-            secret:
-              secretName: billing-mtls
-      - contains:
-          path: spec.template.spec.containers[0].volumeMounts
-          content:
-            name: billing-metrics-mtls
-            mountPath: /etc/litellm/billing-mtls
-            readOnly: true
-
-  # The production collector presents a public web-PKI certificate, so the CA
-  # override must stay absent unless a private collector is configured.
-  - it: omits the CA env, volume, and mount when no caSecretName is set
-    template: gateway/deployment.yaml
-    set:
-      billingMetrics:
-        enabled: true
-        secretName: billing-mtls
-    asserts:
+            name: LITELLM_BILLING_METRICS_CLIENT_SECRET
+            valueFrom:
+              secretKeyRef:
+                name: litellm-billing-metrics
+                key: client-secret
       - notContains:
           path: spec.template.spec.volumes
           content:
-            name: billing-metrics-mtls-ca
-            secret:
-              secretName: billing-ca
+            name: billing-metrics
       - notContains:
-          path: spec.template.spec.containers[0].env
+          path: spec.template.spec.containers[0].volumeMounts
           content:
-            name: LITELLM_BILLING_METRICS_CA_CERT
-            value: /etc/litellm/billing-mtls-ca/ca.crt
+            name: billing-metrics
 
-  - it: mounts the CA secret when caSecretName is set
-    template: gateway/deployment.yaml
+  - it: wires billing metrics on the backend without volumes
+    template: backend/deployment.yaml
     set:
       billingMetrics:
         enabled: true
-        secretName: billing-mtls
-        caSecretName: billing-ca
     asserts:
       - contains:
           path: spec.template.spec.containers[0].env
           content:
-            name: LITELLM_BILLING_METRICS_CA_CERT
-            value: /etc/litellm/billing-mtls-ca/ca.crt
-      - contains:
+            name: LITELLM_BILLING_METRICS_CLIENT_SECRET
+            valueFrom:
+              secretKeyRef:
+                name: litellm-billing-metrics
+                key: client-secret
+      - notContains:
           path: spec.template.spec.volumes
           content:
-            name: billing-metrics-mtls-ca
-            secret:
-              secretName: billing-ca
-      - contains:
-          path: spec.template.spec.containers[0].volumeMounts
-          content:
-            name: billing-metrics-mtls-ca
-            mountPath: /etc/litellm/billing-mtls-ca
-            readOnly: true
+            name: billing-metrics
 
-  - it: passes the export interval through only when set
+  - it: does not add billing metrics to migrations
+    template: migrations-job.yaml
+    set:
+      billingMetrics:
+        enabled: true
+    asserts:
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+
+  - it: renders an optional export interval
     template: gateway/deployment.yaml
     set:
       billingMetrics:
         enabled: true
-        secretName: billing-mtls
         exportIntervalMs: 5000
     asserts:
       - contains:
@@ -132,102 +104,27 @@ tests:
             name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS
             value: "5000"
 
-  - it: keeps user-supplied gateway volumes alongside the billing secret
+  - it: requires the endpoint when enabled
     template: gateway/deployment.yaml
     set:
       billingMetrics:
         enabled: true
-        secretName: billing-mtls
-      gateway.volumes:
-        - name: custom-callbacks
-          configMap:
-            name: my-callbacks
-      gateway.volumeMounts:
-        - name: custom-callbacks
-          mountPath: /app/callbacks
+        endpoint: ""
     asserts:
-      - contains:
-          path: spec.template.spec.volumes
-          content:
-            name: custom-callbacks
-            configMap:
-              name: my-callbacks
-      - contains:
-          path: spec.template.spec.volumes
-          content:
-            name: billing-metrics-mtls
-            secret:
-              secretName: billing-mtls
+      - failedTemplate:
+          errorMessage: billingMetrics.endpoint is required when billingMetrics.enabled is true
 
-  # The backend keeps the named-server MCP transport (/{mcp_server_name}/mcp),
-  # which writes a SpendLogs row, so it must meter too or that traffic is lost.
-  - it: meters the backend as well, since it serves the MCP transport
-    template: backend/deployment.yaml
-    set:
-      billingMetrics:
-        enabled: true
-        secretName: billing-mtls
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_BILLING_METRICS_ENDPOINT
-            value: https://telemetry.litellm.ai
-      - contains:
-          path: spec.template.spec.containers[0].volumeMounts
-          content:
-            name: billing-metrics-mtls
-            mountPath: /etc/litellm/billing-mtls
-            readOnly: true
-      - contains:
-          path: spec.template.spec.volumes
-          content:
-            name: billing-metrics-mtls
-            secret:
-              secretName: billing-mtls
-
-  - it: leaves the backend alone when metering is off
-    template: backend/deployment.yaml
-    asserts:
-      - notContains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_BILLING_METRICS_ENDPOINT
-            value: https://telemetry.litellm.ai
-
-  # The migrations job runs prisma and serves no traffic; it must never receive
-  # the client key.
-  - it: never mounts the billing cert on the migrations job
-    template: migrations-job.yaml
-    set:
-      billingMetrics:
-        enabled: true
-        secretName: billing-mtls
-    asserts:
-      - notContains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_BILLING_METRICS_ENDPOINT
-            value: https://telemetry.litellm.ai
-      - isNull:
-          path: spec.template.spec.volumes
-
-  # The conventional Secret name is the default, so enabling metering needs no
-  # secretName at all; the guard below only fires on an explicitly blanked one.
-  - it: uses the conventional secret name by default
+  - it: requires the token URL when enabled
     template: gateway/deployment.yaml
     set:
       billingMetrics:
         enabled: true
+        tokenUrl: ""
     asserts:
-      - contains:
-          path: spec.template.spec.volumes
-          content:
-            name: billing-metrics-mtls
-            secret:
-              secretName: litellm-billing-metrics-mtls
+      - failedTemplate:
+          errorMessage: billingMetrics.tokenUrl is required when billingMetrics.enabled is true
 
-  - it: fails loudly when the secretName is explicitly blanked
+  - it: requires the secret name when enabled
     template: gateway/deployment.yaml
     set:
       billingMetrics:
@@ -235,15 +132,4 @@ tests:
         secretName: ""
     asserts:
       - failedTemplate:
-          errorMessage: billingMetrics.secretName is required when billingMetrics.enabled is true (an existing Secret with tls.crt and tls.key)
-
-  - it: fails loudly when enabled without an endpoint
-    template: gateway/deployment.yaml
-    set:
-      billingMetrics:
-        enabled: true
-        endpoint: ""
-        secretName: billing-mtls
-    asserts:
-      - failedTemplate:
-          errorMessage: billingMetrics.endpoint is required when billingMetrics.enabled is true
+          errorMessage: billingMetrics.secretName is required when billingMetrics.enabled is true

helm/litellm/tests/migration_job_tests.yaml

@@ -91,7 +91,7 @@ tests:
             app.kubernetes.io/name: litellm
             app.kubernetes.io/instance: RELEASE-NAME
             app.kubernetes.io/managed-by: Helm
-            helm.sh/chart: litellm-0.1.0
+            helm.sh/chart: litellm-0.1.1
             app.kubernetes.io/component: migrations
 
   - it: renders pod-level and container-level securityContext in their own scopes

helm/litellm/values.yaml

@@ -180,23 +180,14 @@ masterKey:
   secretKey: master-key
 
 # Optional: enterprise billable-request metering. When enabled, the gateway and
-# backend count successful requests to inference, MCP, and A2A endpoints and push
-# them to LiteLLM's collector over mutual TLS. Both components serve billable
-# routes: the backend keeps the named-server MCP transport. Requires an
-# enterprise license. The client certificate identifies the deployment, so it is
-# mounted read-only from an existing Secret and never passed through the env.
+# backend count successful requests to inference, MCP, and A2A endpoints. Requires
+# an enterprise license.
 billingMetrics:
   enabled: false
-  endpoint: https://telemetry.litellm.ai              # collector to push the counter to
-  # An existing Secret holding the client certificate under tls.crt and its key
-  # under tls.key, usually created from the onboarding artifact. The default is
-  # the conventional name, so the common path is to create that Secret and set
-  # enabled: true. Override only if yours is named differently.
-  secretName: litellm-billing-metrics-mtls
-  # Only for private or test collectors whose server certificate is not on the
-  # public web PKI. The production collector needs no CA override.
-  caSecretName: ""                                    # existing Secret holding ca.crt
-  exportIntervalMs: ""                                # push cadence; the proxy defaults to 60000
+  endpoint: https://telemetry.litellm.ai
+  tokenUrl: https://litellm-prod.auth.us-west-1.amazoncognito.com/oauth2/token
+  secretName: litellm-billing-metrics
+  exportIntervalMs: ""
 
 # External Postgres connection. 
 database:

litellm-proxy-extras/litellm_proxy_extras/migrations/20261103010000_spendlogs_accounting_delivery/migration.sql

@@ -0,0 +1,19 @@
+SET LOCAL lock_timeout = '10s';
+CREATE TABLE IF NOT EXISTS "LiteLLM_SpendLogDelivery" (
+    "delivery_id" TEXT PRIMARY KEY DEFAULT gen_random_uuid()::text,
+    "request_id" TEXT NOT NULL,
+    "state" "AccountingDeliveryState" NOT NULL DEFAULT 'pending',
+    "attempts" INTEGER NOT NULL DEFAULT 0,
+    "next_attempt_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "applied_at" TIMESTAMP(3),
+    "last_error" TEXT,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "accounting_payload" JSONB NOT NULL,
+    "is_batch" BOOLEAN NOT NULL
+);
+CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogDelivery_state_next_attempt_at_idx"
+    ON "LiteLLM_SpendLogDelivery" ("state", "next_attempt_at");
+CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogDelivery_state_applied_at_idx"
+    ON "LiteLLM_SpendLogDelivery" ("state", "applied_at");
+CREATE INDEX IF NOT EXISTS "LiteLLM_SpendLogDelivery_request_id_idx"
+    ON "LiteLLM_SpendLogDelivery" ("request_id");

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -2141,6 +2141,23 @@ model LiteLLM_AccountingDelivery {
   @@index([target, state, next_attempt_at])
 }
 
+model LiteLLM_SpendLogDelivery {
+  delivery_id        String                  @id @default(dbgenerated("gen_random_uuid()::text"))
+  request_id         String
+  state              AccountingDeliveryState @default(pending)
+  accounting_payload Json
+  is_batch           Boolean
+  attempts           Int                     @default(0)
+  next_attempt_at    DateTime                @default(now())
+  applied_at         DateTime?
+  last_error         String?
+  created_at         DateTime                @default(now())
+
+  @@index([state, next_attempt_at])
+  @@index([state, applied_at])
+  @@index([request_id])
+}
+
 model LiteLLM_SubscriptionAccountTable {
   subscription_account_id String @id @default(uuid())
   custom_llm_provider String

litellm-rust/Cargo.lock

@@ -3389,15 +3389,19 @@ version = "0.1.0"
 dependencies = [
  "aws-config",
  "aws-credential-types",
+ "aws-runtime",
  "aws-sdk-sts",
  "aws-sigv4",
  "aws-smithy-runtime-api",
+ "aws-smithy-types",
  "aws-types",
  "bytes",
  "futures-util",
  "litellm-auth-types",
  "litellm-error",
  "litellm-http",
+ "litellm-secret-value",
+ "litellm-secrets",
  "moka",
  "reqwest",
  "rstest",
@@ -3406,6 +3410,7 @@ dependencies = [
  "sha2 0.10.9",
  "thiserror 2.0.19",
  "tokio",
+ "url",
  "veil",
 ]
 
@@ -3418,6 +3423,7 @@ dependencies = [
  "litellm-auth-types",
  "litellm-error",
  "litellm-secret-value",
+ "litellm-secrets",
  "moka",
  "rstest",
  "serde_json",
@@ -3431,6 +3437,7 @@ dependencies = [
 name = "litellm-auth-gcp"
 version = "0.1.0"
 dependencies = [
+ "axum",
  "base64 0.22.1",
  "gcp_auth",
  "google-cloud-auth",
@@ -3440,10 +3447,12 @@ dependencies = [
  "litellm-secret-value",
  "moka",
  "rstest",
+ "rustls 0.23.42",
  "serde",
  "serde_json",
  "sha2 0.10.9",
  "tempfile",
+ "time",
  "tokio",
  "veil",
 ]

litellm-rust/crates/auth-aws/Cargo.toml

@@ -11,14 +11,16 @@ litellm-error.workspace = true
 litellm-http.workspace = true
 reqwest.workspace = true
 
-moka = { workspace = true, features = ["sync"] }
+moka.workspace = true
+litellm-secret-value.workspace = true
 serde.workspace = true
 serde_json = { workspace = true, features = ["preserve_order"] }
 sha2.workspace = true
 thiserror.workspace = true
+url.workspace = true
 veil.workspace = true
 
-aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio"] }
+aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio", "credentials-process"] }
 aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"] }
 aws-sdk-sts = { version = "1.108.0", default-features = false, features = ["rustls", "rt-tokio"] }
 aws-sigv4 = "1.5.1"
@@ -26,6 +28,9 @@ aws-types = "1.4.0"
 aws-smithy-runtime-api = "1.13.0"
 
 [dev-dependencies]
+litellm-secrets.workspace = true
+aws-smithy-types = "1.6.4"
+aws-runtime = { version = "1.9.4", default-features = false }
 rstest.workspace = true
 litellm-http = { workspace = true, features = ["test-support"] }
 bytes.workspace = true

litellm-rust/crates/auth-aws/src/constants.rs

@@ -11,8 +11,8 @@ pub const AWS_SESSION_NAME: &str = AwsParams::SESSION_NAME.env[0];
 pub const AWS_PROFILE_NAME: &str = AwsParams::PROFILE_NAME.env[0];
 pub const AWS_ROLE_NAME: &str = AwsParams::ROLE_NAME.env[0];
 pub const AWS_WEB_IDENTITY_TOKEN: &str = AwsParams::WEB_IDENTITY_TOKEN.env[0];
-pub const AWS_ROLE_ARN: &str = "AWS_ROLE_ARN";
-pub const AWS_WEB_IDENTITY_TOKEN_FILE: &str = "AWS_WEB_IDENTITY_TOKEN_FILE";
+pub const AWS_ROLE_ARN: &str = AwsParams::ROLE_ARN_ENV;
+pub const AWS_WEB_IDENTITY_TOKEN_FILE: &str = AwsParams::WEB_IDENTITY_TOKEN_FILE_ENV;
 pub const AWS_STS_ENDPOINT: &str = AwsParams::STS_ENDPOINT.env[0];
 pub const AWS_EXTERNAL_ID: &str = AwsParams::EXTERNAL_ID.env[0];
 pub const AWS_BEARER_TOKEN_BEDROCK: &str = "AWS_BEARER_TOKEN_BEDROCK";

litellm-rust/crates/auth-aws/src/error.rs

@@ -2,6 +2,8 @@ use litellm_error::Source;
 
 #[derive(Clone, Debug, thiserror::Error, PartialEq, Eq)]
 pub enum Error {
+    #[error("AWS credential provider returned expired credentials")]
+    ExpiredCredentials,
     #[error("could not load AWS {flow} credentials")]
     Credentials {
         flow: CredentialFlow,

litellm-rust/crates/auth-aws/src/signer.rs

@@ -75,7 +75,8 @@ impl RequestSigner for SigV4Signer {
         )
         .map_err(|error| match error {
             Error::NonAsciiHeader { .. } => litellm_http::Error::unsignable_header(error),
-            Error::Credentials { .. }
+            Error::ExpiredCredentials
+            | Error::Credentials { .. }
             | Error::MissingWebIdentityCredentials
             | Error::WebIdentityExpiration(_)
             | Error::Signing { .. } => litellm_http::Error::signature(error),

litellm-rust/crates/auth-azure/Cargo.toml

@@ -8,17 +8,19 @@ repository.workspace = true
 [dependencies]
 litellm-secret-value.workspace = true
 litellm-auth-types.workspace = true
+litellm-error.workspace = true
 
 moka.workspace = true
 serde_json.workspace = true
 sha2.workspace = true
 strum.workspace = true
+tokio.workspace = true
 url.workspace = true
 
 azure_core = "1.0.0"
 azure_identity = { version = "1.0.0", features = ["tokio"] }
 
 [dev-dependencies]
-litellm-error.workspace = true
+litellm-secrets.workspace = true
 rstest.workspace = true
-tokio.workspace = true
+tokio = { workspace = true, features = ["test-util"] }

litellm-rust/crates/auth-azure/src/lib.rs

@@ -3,5 +3,5 @@ mod native;
 mod resolve;
 mod types;
 
-pub use resolve::{AzureAuthService, SECRET_NAMES};
+pub use resolve::{AzureAuthService, AzureIdentity, SECRET_NAMES};
 pub use types::{AzureAuthInputs, ConfigValue};

litellm-rust/crates/auth-azure/src/native.rs

@@ -59,6 +59,7 @@ pub(crate) enum NativeAzureRequest {
 pub(crate) struct ValidatedAzureRequest {
     request: NativeAzureRequest,
     credential_source: InputSource,
+    probe_imds: bool,
 }
 
 impl ValidatedAzureRequest {
@@ -68,9 +69,18 @@ impl ValidatedAzureRequest {
         Ok(Self {
             request,
             credential_source,
+            probe_imds: false,
         })
     }
 
+    pub(crate) fn with_imds_probe(self, enabled: bool) -> Self {
+        Self {
+            probe_imds: enabled
+                && matches!(self.request, NativeAzureRequest::ManagedIdentity { .. }),
+            ..self
+        }
+    }
+
     pub(crate) fn credential_source(&self) -> InputSource {
         self.credential_source
     }
@@ -87,9 +97,34 @@ impl ValidatedAzureRequest {
     }
 }
 
+pub(crate) enum CredentialAttempt {
+    Resolved(ResolvedCredential),
+    Unavailable(Error),
+    Failed(Error),
+}
+
+impl CredentialAttempt {
+    pub(crate) fn into_result(self) -> Result<ResolvedCredential, Error> {
+        match self {
+            Self::Resolved(credential) => Ok(credential),
+            Self::Unavailable(error) | Self::Failed(error) => Err(error),
+        }
+    }
+}
+
 pub(crate) struct NativeAzureTokenAcquirer {
     cache: AzureCredentialProviderCache,
     transport: Option<azure_core::http::Transport>,
+    imds_available: tokio::sync::Mutex<ImdsAvailability>,
+}
+
+enum ImdsAvailability {
+    Unknown,
+    Available,
+    Unavailable {
+        retry_at: tokio::time::Instant,
+        error: Error,
+    },
 }
 
 impl Default for NativeAzureTokenAcquirer {
@@ -103,6 +138,7 @@ impl NativeAzureTokenAcquirer {
         Self {
             cache: AzureCredentialProviderCache::new(cache_capacity),
             transport: None,
+            imds_available: tokio::sync::Mutex::new(ImdsAvailability::Unknown),
         }
     }
 
@@ -114,10 +150,51 @@ impl NativeAzureTokenAcquirer {
         Self {
             cache: AzureCredentialProviderCache::new(cache_capacity),
             transport: Some(transport),
+            imds_available: tokio::sync::Mutex::new(ImdsAvailability::Unknown),
         }
     }
 
-    pub(crate) async fn acquire(
+    pub(crate) async fn acquire(&self, request: ValidatedAzureRequest) -> CredentialAttempt {
+        let discover_imds = request.probe_imds;
+        if discover_imds && let Err(error) = self.discover_imds().await {
+            return CredentialAttempt::Unavailable(error);
+        }
+        match self.acquire_token(request).await {
+            Ok(credential) => CredentialAttempt::Resolved(credential),
+            Err(error) => {
+                let missing_identity = discover_imds && unassigned_identity(&error);
+                if missing_identity {
+                    CredentialAttempt::Unavailable(error)
+                } else {
+                    CredentialAttempt::Failed(error)
+                }
+            }
+        }
+    }
+
+    async fn discover_imds(&self) -> Result<(), Error> {
+        let mut availability = self.imds_available.lock().await;
+        match &*availability {
+            ImdsAvailability::Available => return Ok(()),
+            ImdsAvailability::Unavailable { retry_at, error }
+                if tokio::time::Instant::now() < *retry_at =>
+            {
+                return Err(error.clone());
+            }
+            ImdsAvailability::Unknown | ImdsAvailability::Unavailable { .. } => {}
+        }
+        let result = probe_imds(self.transport.clone().unwrap_or_default()).await;
+        *availability = match &result {
+            Ok(()) => ImdsAvailability::Available,
+            Err(error) => ImdsAvailability::Unavailable {
+                retry_at: tokio::time::Instant::now() + Duration::from_secs(30),
+                error: error.clone(),
+            },
+        };
+        result
+    }
+
+    async fn acquire_token(
         &self,
         request: ValidatedAzureRequest,
     ) -> Result<ResolvedCredential, Error> {
@@ -148,6 +225,54 @@ impl NativeAzureTokenAcquirer {
     }
 }
 
+fn unassigned_identity(error: &(dyn std::error::Error + 'static)) -> bool {
+    std::iter::successors(Some(error), |cause| cause.source()).any(|cause| {
+        cause
+            .downcast_ref::<azure_core::Error>()
+            .or_else(|| {
+                cause
+                    .downcast_ref::<litellm_error::Source>()?
+                    .downcast_ref::<azure_core::Error>()
+            })
+            .is_some_and(|error| {
+                error.http_status() == Some(azure_core::http::StatusCode::BadRequest)
+            })
+    })
+}
+
+async fn probe_imds(transport: azure_core::http::Transport) -> Result<(), Error> {
+    use azure_core::http::{Context, Method, Request, Url};
+
+    let endpoint =
+        Url::parse("http://169.254.169.254/metadata/identity/oauth2/token?api-version=2019-08-01")
+            .map_err(|error| {
+                Error::InvalidConfiguration(ErrorDetail::failed("Azure IMDS probe endpoint", error))
+            })?;
+    let mut request = Request::new(endpoint, Method::Get);
+    let response = tokio::time::timeout(
+        Duration::from_secs(1),
+        transport.send(&Context::new(), &mut request),
+    )
+    .await
+    .map_err(|error| {
+        Error::CredentialAcquisition(ErrorDetail::failed("Azure IMDS availability probe", error))
+    })?
+    .map_err(|error| {
+        Error::CredentialAcquisition(ErrorDetail::failed("Azure IMDS availability probe", error))
+    })?;
+    // Headerless IMDS discovery expects 400, checked 2026-11-01
+    // https://learn.microsoft.com/en-us/azure/virtual-machines/instance-metadata-service#errors-and-debugging
+    if response.status() != azure_core::http::StatusCode::BadRequest {
+        return Err(Error::CredentialAcquisition(
+            ErrorDetail::UnexpectedStatus {
+                operation: "Azure IMDS availability probe",
+                status: u16::from(response.status()),
+            },
+        ));
+    }
+    Ok(())
+}
+
 impl NativeAzureRequest {
     fn scope(&self) -> &str {
         match self {
@@ -493,6 +618,7 @@ impl ClientAssertion for StaticAssertion {
 #[cfg(test)]
 mod tests {
     use std::sync::{Arc, Mutex};
+    use std::time::Duration;
 
     use azure_core::http::headers::Headers;
     use azure_core::http::{AsyncRawResponse, HttpClient, Request, StatusCode, Transport};
@@ -569,6 +695,296 @@ mod tests {
         }
     }
 
+    #[derive(Debug)]
+    struct AvailableImdsClient {
+        probe_status: StatusCode,
+        probe_delay: Duration,
+        recover_after: Option<usize>,
+        response_status: Option<StatusCode>,
+        probes: std::sync::atomic::AtomicUsize,
+    }
+
+    impl Default for AvailableImdsClient {
+        fn default() -> Self {
+            Self {
+                probe_status: StatusCode::BadRequest,
+                probe_delay: Duration::ZERO,
+                recover_after: None,
+                response_status: None,
+                probes: std::sync::atomic::AtomicUsize::new(0),
+            }
+        }
+    }
+
+    impl HttpClient for AvailableImdsClient {
+        fn execute_request<'life0, 'life1, 'async_trait>(
+            &'life0 self,
+            request: &'life1 Request,
+        ) -> std::pin::Pin<
+            Box<dyn std::future::Future<Output = Result<AsyncRawResponse>> + Send + 'async_trait>,
+        >
+        where
+            'life0: 'async_trait,
+            'life1: 'async_trait,
+            Self: 'async_trait,
+        {
+            Box::pin(async move {
+                if request
+                    .headers()
+                    .get_optional_str(&azure_core::http::headers::HeaderName::from_static(
+                        "metadata",
+                    ))
+                    .is_none()
+                {
+                    let probe = self
+                        .probes
+                        .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+                    if self.recover_after.is_some_and(|failures| probe >= failures) {
+                        return Ok(AsyncRawResponse::from_bytes(
+                            StatusCode::BadRequest,
+                            Headers::new(),
+                            "metadata header required",
+                        ));
+                    }
+                    tokio::time::sleep(self.probe_delay).await;
+                    return Ok(AsyncRawResponse::from_bytes(
+                        self.probe_status,
+                        Headers::new(),
+                        "metadata header required",
+                    ));
+                }
+                if let Some(status) = self.response_status {
+                    return Ok(AsyncRawResponse::from_bytes(
+                        status,
+                        Headers::new(),
+                        "identity endpoint failure",
+                    ));
+                }
+                tokio::time::sleep(std::time::Duration::from_secs(2)).await;
+                let expires = std::time::SystemTime::now()
+                    .duration_since(std::time::UNIX_EPOCH)
+                    .unwrap()
+                    .as_secs()
+                    + 3600;
+                Ok(AsyncRawResponse::from_bytes(
+                    StatusCode::Ok,
+                    Headers::new(),
+                    serde_json::json!({
+                        "access_token": "managed-token", "expires_on": expires.to_string(),
+                        "token_type": "Bearer", "resource": "https://service.test",
+                    })
+                    .to_string(),
+                ))
+            })
+        }
+    }
+
+    #[rstest::rstest]
+    #[tokio::test(start_paused = true)]
+    async fn available_imds_retains_full_sdk_token_acquisition() {
+        let acquirer = NativeAzureTokenAcquirer::with_transport(
+            4,
+            Transport::new(Arc::new(AvailableImdsClient {
+                probe_status: StatusCode::BadRequest,
+                response_status: None,
+                ..Default::default()
+            })),
+        );
+        let request = ValidatedAzureRequest::new(NativeAzureRequest::ManagedIdentity {
+            client_id: None,
+            scope: deployment("https://service.test/.default".into()),
+            selection_source: InputSource::Deployment,
+        })
+        .unwrap()
+        .with_imds_probe(true);
+        let credential = acquirer.acquire(request).await.into_result().unwrap();
+        assert_eq!(credential.secret().expose(), "managed-token");
+    }
+
+    #[rstest::rstest]
+    #[case::another_metadata_service(StatusCode::NotFound)]
+    #[case::token_required(StatusCode::Unauthorized)]
+    #[case::proxy_rejection(StatusCode::Forbidden)]
+    #[case::unavailable(StatusCode::ServiceUnavailable)]
+    #[tokio::test(start_paused = true)]
+    async fn unrelated_metadata_responses_do_not_start_managed_identity_acquisition(
+        #[case] status: StatusCode,
+    ) {
+        let client = Arc::new(AvailableImdsClient {
+            probe_status: status,
+            ..Default::default()
+        });
+        let acquirer = NativeAzureTokenAcquirer::with_transport(4, Transport::new(client.clone()));
+        let request = ValidatedAzureRequest::new(NativeAzureRequest::ManagedIdentity {
+            client_id: None,
+            scope: deployment("https://service.test/.default".into()),
+            selection_source: InputSource::Deployment,
+        })
+        .unwrap()
+        .with_imds_probe(true);
+        assert!(matches!(
+            acquirer.acquire(request.clone()).await,
+            super::CredentialAttempt::Unavailable(_)
+        ));
+        assert!(matches!(
+            acquirer.acquire(request).await,
+            super::CredentialAttempt::Unavailable(_)
+        ));
+        assert_eq!(client.probes.load(std::sync::atomic::Ordering::SeqCst), 1);
+    }
+
+    #[rstest::rstest]
+    #[tokio::test(start_paused = true)]
+    async fn an_imds_timeout_is_not_repeated_on_the_next_request() {
+        let client = Arc::new(AvailableImdsClient {
+            probe_delay: Duration::from_secs(10),
+            ..Default::default()
+        });
+        let acquirer = NativeAzureTokenAcquirer::with_transport(4, Transport::new(client.clone()));
+        let request = ValidatedAzureRequest::new(NativeAzureRequest::ManagedIdentity {
+            client_id: None,
+            scope: deployment("https://service.test/.default".into()),
+            selection_source: InputSource::Deployment,
+        })
+        .unwrap()
+        .with_imds_probe(true);
+        assert!(matches!(
+            acquirer.acquire(request.clone()).await,
+            super::CredentialAttempt::Unavailable(_)
+        ));
+        let second = tokio::time::timeout(Duration::from_millis(10), acquirer.acquire(request))
+            .await
+            .unwrap();
+        assert!(matches!(second, super::CredentialAttempt::Unavailable(_)));
+        assert_eq!(client.probes.load(std::sync::atomic::Ordering::SeqCst), 1);
+    }
+
+    #[rstest::fixture]
+    fn imds_request() -> ValidatedAzureRequest {
+        ValidatedAzureRequest::new(NativeAzureRequest::ManagedIdentity {
+            client_id: None,
+            scope: deployment("https://service.test/.default".into()),
+            selection_source: InputSource::Deployment,
+        })
+        .unwrap()
+        .with_imds_probe(true)
+    }
+
+    #[rstest::rstest]
+    #[case::timeout(StatusCode::BadRequest, Duration::from_secs(10))]
+    #[case::service_unavailable(StatusCode::ServiceUnavailable, Duration::ZERO)]
+    #[tokio::test(start_paused = true)]
+    async fn failed_imds_discovery_recovers_without_restarting_the_acquirer(
+        imds_request: ValidatedAzureRequest,
+        #[case] status: StatusCode,
+        #[case] delay: Duration,
+    ) {
+        let client = Arc::new(AvailableImdsClient {
+            probe_status: status,
+            probe_delay: delay,
+            recover_after: Some(1),
+            ..Default::default()
+        });
+        let acquirer = NativeAzureTokenAcquirer::with_transport(4, Transport::new(client.clone()));
+        let (first, second) = tokio::join!(
+            acquirer.acquire(imds_request.clone()),
+            acquirer.acquire(imds_request.clone()),
+        );
+        assert!(matches!(first, super::CredentialAttempt::Unavailable(_)));
+        assert!(matches!(second, super::CredentialAttempt::Unavailable(_)));
+        assert_eq!(client.probes.load(std::sync::atomic::Ordering::SeqCst), 1);
+
+        tokio::time::advance(Duration::from_secs(30)).await;
+        let (first, second) = tokio::join!(
+            acquirer.acquire(imds_request.clone()),
+            acquirer.acquire(imds_request.clone()),
+        );
+        assert_eq!(
+            first.into_result().unwrap().secret().expose(),
+            "managed-token"
+        );
+        assert_eq!(
+            second.into_result().unwrap().secret().expose(),
+            "managed-token"
+        );
+        assert_eq!(client.probes.load(std::sync::atomic::Ordering::SeqCst), 2);
+
+        tokio::time::advance(Duration::from_secs(60)).await;
+        assert_eq!(
+            acquirer
+                .acquire(imds_request)
+                .await
+                .into_result()
+                .unwrap()
+                .secret()
+                .expose(),
+            "managed-token",
+        );
+        assert_eq!(client.probes.load(std::sync::atomic::Ordering::SeqCst), 2);
+    }
+
+    #[rstest::rstest]
+    #[tokio::test(start_paused = true)]
+    async fn a_persistent_imds_outage_reuses_each_failed_probe(
+        imds_request: ValidatedAzureRequest,
+    ) {
+        let client = Arc::new(AvailableImdsClient {
+            probe_delay: Duration::from_secs(10),
+            ..Default::default()
+        });
+        let acquirer = NativeAzureTokenAcquirer::with_transport(4, Transport::new(client.clone()));
+        assert!(matches!(
+            acquirer.acquire(imds_request.clone()).await,
+            super::CredentialAttempt::Unavailable(_),
+        ));
+        tokio::time::advance(Duration::from_secs(30)).await;
+        let (first, second) = tokio::join!(
+            acquirer.acquire(imds_request.clone()),
+            acquirer.acquire(imds_request),
+        );
+        assert!(matches!(first, super::CredentialAttempt::Unavailable(_)));
+        assert!(matches!(second, super::CredentialAttempt::Unavailable(_)));
+        assert_eq!(client.probes.load(std::sync::atomic::Ordering::SeqCst), 2);
+    }
+
+    #[rstest::rstest]
+    #[case::default_identity_not_assigned(true, StatusCode::BadRequest, true)]
+    #[case::explicit_identity_not_assigned(false, StatusCode::BadRequest, false)]
+    #[case::default_identity_forbidden(true, StatusCode::Forbidden, false)]
+    #[case::default_identity_service_failure(true, StatusCode::InternalServerError, false)]
+    #[tokio::test(start_paused = true)]
+    async fn only_unavailable_default_identities_allow_fallback(
+        #[case] discovery: bool,
+        #[case] status: StatusCode,
+        #[case] unavailable: bool,
+    ) {
+        let acquirer = NativeAzureTokenAcquirer::with_transport(
+            4,
+            Transport::new(Arc::new(AvailableImdsClient {
+                probe_status: StatusCode::BadRequest,
+                response_status: Some(status),
+                ..Default::default()
+            })),
+        );
+        let request = ValidatedAzureRequest::new(NativeAzureRequest::ManagedIdentity {
+            client_id: None,
+            scope: deployment("https://service.test/.default".into()),
+            selection_source: InputSource::Deployment,
+        })
+        .unwrap()
+        .with_imds_probe(discovery);
+        let outcome = acquirer.acquire(request).await;
+        let is_unavailable = matches!(outcome, super::CredentialAttempt::Unavailable(_));
+        let error = outcome.into_result().unwrap_err();
+        assert_eq!(
+            is_unavailable,
+            unavailable,
+            "{}",
+            litellm_error::Report::new(&error)
+        );
+    }
+
+    #[rstest::rstest]
     #[tokio::test]
     async fn client_secret_uses_sdk_protocol_and_reuses_cached_credential() {
         let transport = Arc::new(RecordingTokenClient::default());
@@ -582,8 +998,12 @@ mod tests {
             "https://login.test",
         );
 
-        let first = acquirer.acquire(request.clone()).await.unwrap();
-        let second = acquirer.acquire(request).await.unwrap();
+        let first = acquirer
+            .acquire(request.clone())
+            .await
+            .into_result()
+            .unwrap();
+        let second = acquirer.acquire(request).await.into_result().unwrap();
 
         assert_eq!(first.secret().expose(), "native-token");
         assert_eq!(second.secret().expose(), "native-token");
@@ -599,6 +1019,7 @@ mod tests {
         );
     }
 
+    #[rstest::rstest]
     #[tokio::test]
     async fn credential_provider_cache_isolates_every_client_secret_identity_field() {
         let transport = Arc::new(RecordingTokenClient::default());
@@ -645,10 +1066,10 @@ mod tests {
             ),
         ];
 
-        acquirer.acquire(base.clone()).await.unwrap();
-        acquirer.acquire(base).await.unwrap();
+        acquirer.acquire(base.clone()).await.into_result().unwrap();
+        acquirer.acquire(base).await.into_result().unwrap();
         for request in variants.into_iter().skip(1) {
-            acquirer.acquire(request).await.unwrap();
+            acquirer.acquire(request).await.into_result().unwrap();
         }
 
         assert_eq!(transport.requests.lock().unwrap().len(), 6);

litellm-rust/crates/auth-azure/src/resolve.rs

@@ -1,3 +1,4 @@
+use crate::native::CredentialAttempt;
 use litellm_auth_types::{
     CredentialFileRef, CredentialLookup, CredentialRef, InputSource, ResolvedCredential, Sourced,
     TokenProviderHandle,
@@ -12,25 +13,31 @@ use std::sync::Arc;
 use super::native::{NativeAzureRequest, NativeAzureTokenAcquirer, ValidatedAzureRequest};
 use super::types::{AzureAuthInputs, AzureCredentialType, ConfigValue, DEFAULT_AZURE_SCOPE};
 
-const AZURE_AD_TOKEN_ENV: &str = "AZURE_AD_TOKEN";
-const AZURE_TENANT_ID_ENV: &str = "AZURE_TENANT_ID";
-const AZURE_CLIENT_ID_ENV: &str = "AZURE_CLIENT_ID";
-const AZURE_CLIENT_SECRET_ENV: &str = "AZURE_CLIENT_SECRET";
-const AZURE_SCOPE_ENV: &str = "AZURE_SCOPE";
-const AZURE_AUTHORITY_HOST_ENV: &str = "AZURE_AUTHORITY_HOST";
-const AZURE_CREDENTIAL_ENV: &str = "AZURE_CREDENTIAL";
-const AZURE_FEDERATED_TOKEN_FILE_ENV: &str = "AZURE_FEDERATED_TOKEN_FILE";
+#[derive(strum::IntoStaticStr, strum::VariantArray, strum::VariantNames)]
+enum AzureSecretName {
+    #[strum(serialize = "AZURE_AD_TOKEN")]
+    AdToken,
+    #[strum(serialize = "AZURE_TENANT_ID")]
+    TenantId,
+    #[strum(serialize = "AZURE_CLIENT_ID")]
+    ClientId,
+    #[strum(serialize = "AZURE_CLIENT_SECRET")]
+    ClientSecret,
+    #[strum(serialize = "AZURE_SCOPE")]
+    Scope,
+    #[strum(serialize = "AZURE_AUTHORITY_HOST")]
+    AuthorityHost,
+    #[strum(serialize = "AZURE_CREDENTIAL")]
+    Credential,
+    #[strum(serialize = "AZURE_FEDERATED_TOKEN_FILE")]
+    FederatedTokenFile,
+    #[strum(serialize = "IDENTITY_ENDPOINT")]
+    IdentityEndpoint,
+    #[strum(serialize = "MSI_ENDPOINT")]
+    MsiEndpoint,
+}
 
-pub const SECRET_NAMES: &[&str] = &[
-    AZURE_AD_TOKEN_ENV,
-    AZURE_TENANT_ID_ENV,
-    AZURE_CLIENT_ID_ENV,
-    AZURE_CLIENT_SECRET_ENV,
-    AZURE_SCOPE_ENV,
-    AZURE_AUTHORITY_HOST_ENV,
-    AZURE_CREDENTIAL_ENV,
-    AZURE_FEDERATED_TOKEN_FILE_ENV,
-];
+pub const SECRET_NAMES: &[&str] = <AzureSecretName as strum::VariantNames>::VARIANTS;
 
 #[derive(Clone, Debug)]
 pub(crate) enum AzureCredentialPlan {
@@ -57,14 +64,14 @@ trait AzureTokenAcquirer: Send + Sync {
     fn acquire(
         &self,
         request: ValidatedAzureRequest,
-    ) -> Pin<Box<dyn Future<Output = Result<ResolvedCredential, Error>> + Send + '_>>;
+    ) -> Pin<Box<dyn Future<Output = CredentialAttempt> + Send + '_>>;
 }
 
 impl AzureTokenAcquirer for NativeAzureTokenAcquirer {
     fn acquire(
         &self,
         request: ValidatedAzureRequest,
-    ) -> Pin<Box<dyn Future<Output = Result<ResolvedCredential, Error>> + Send + '_>> {
+    ) -> Pin<Box<dyn Future<Output = CredentialAttempt> + Send + '_>> {
         Box::pin(NativeAzureTokenAcquirer::acquire(self, request))
     }
 }
@@ -83,12 +90,36 @@ impl AzureAuthService {
         Self { native }
     }
 
+    pub async fn identity_token(
+        &self,
+        identity: &AzureIdentity,
+        scope: &str,
+        env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
+    ) -> Result<Option<ResolvedCredential>, Error> {
+        let plan = identity_plan(identity, scope, env_lookup)?;
+        let inputs = AzureAuthInputs::default_credential_for_scope(scope);
+        Ok(self
+            .run_plan(plan, &inputs, env_lookup)
+            .await?
+            .map(Sourced::into_value))
+    }
+
     pub async fn get_azure_ad_token(
         &self,
         inputs: &AzureAuthInputs,
         env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
     ) -> Result<Option<Sourced<ResolvedCredential>>, Error> {
-        match select_auth_plan(inputs, env_lookup)? {
+        self.run_plan(select_auth_plan(inputs, env_lookup)?, inputs, env_lookup)
+            .await
+    }
+
+    async fn run_plan(
+        &self,
+        plan: AzureCredentialPlan,
+        inputs: &AzureAuthInputs,
+        env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
+    ) -> Result<Option<Sourced<ResolvedCredential>>, Error> {
+        match plan {
             AzureCredentialPlan::Supplied(credential) => Ok(Some(credential)),
             AzureCredentialPlan::Caller(caller) => {
                 let credential = caller.acquire().await?;
@@ -123,6 +154,7 @@ impl AzureAuthService {
                 self.native
                     .acquire(request)
                     .await
+                    .into_result()
                     .map(|credential| Sourced::new(credential, source))
                     .map(Some)
             }
@@ -131,6 +163,7 @@ impl AzureAuthService {
                 self.native
                     .acquire(request)
                     .await
+                    .into_result()
                     .map(|credential| Some(Sourced::new(credential, source)))
             }
             AzureCredentialPlan::Chain(requests) => {
@@ -138,8 +171,16 @@ impl AzureAuthService {
                 for request in requests {
                     let source = request.credential_source();
                     match self.native.acquire(request).await {
-                        Ok(credential) => return Ok(Some(Sourced::new(credential, source))),
-                        Err(error) => failures.push(error),
+                        CredentialAttempt::Resolved(credential) => {
+                            return Ok(Some(Sourced::new(credential, source)));
+                        }
+                        CredentialAttempt::Unavailable(error) => failures.push(error),
+                        CredentialAttempt::Failed(error) => {
+                            return Err(Error::CredentialChain {
+                                last: Box::new(error),
+                                earlier: failures,
+                            });
+                        }
                     }
                 }
                 match failures.pop() {
@@ -155,39 +196,161 @@ impl AzureAuthService {
     }
 }
 
+/// The identity azure-identity builds for one explicit credential class, without LiteLLM's provider settings
+/// (`AZURE_AD_TOKEN`, `AZURE_CREDENTIAL`, `AZURE_SCOPE`) that `get_azure_ad_token` also reads.
+#[derive(Clone, Debug)]
+pub enum AzureIdentity {
+    ClientSecret {
+        tenant_id: String,
+        client_id: String,
+        client_secret: SecretValue,
+    },
+    ManagedIdentity {
+        client_id: String,
+    },
+    Default,
+}
+
+fn identity_plan(
+    identity: &AzureIdentity,
+    scope: &str,
+    env_lookup: &dyn Fn(&str) -> Option<String>,
+) -> Result<AzureCredentialPlan, Error> {
+    let deployment = |value: &str| Sourced::new(value.to_owned(), InputSource::Deployment);
+    let environment = |name: &str| {
+        env_lookup(name)
+            .filter(|value| !value.trim().is_empty())
+            .map(|value| Sourced::new(value, InputSource::Environment))
+    };
+    let scope = deployment(scope);
+    let authority = environment(AzureSecretName::AuthorityHost.into());
+    match identity {
+        AzureIdentity::ClientSecret {
+            tenant_id,
+            client_id,
+            client_secret,
+        } => Ok(AzureCredentialPlan::Native(ValidatedAzureRequest::new(
+            NativeAzureRequest::ClientSecret {
+                tenant_id: deployment(tenant_id),
+                client_id: deployment(client_id),
+                client_secret: Sourced::new(client_secret.clone(), InputSource::Deployment),
+                scope,
+                authority,
+            },
+        )?)),
+        AzureIdentity::ManagedIdentity { client_id } => Ok(AzureCredentialPlan::Native(
+            ValidatedAzureRequest::new(NativeAzureRequest::ManagedIdentity {
+                client_id: Some(deployment(client_id)),
+                scope,
+                selection_source: InputSource::Deployment,
+            })?,
+        )),
+        AzureIdentity::Default => {
+            let tenant_id = environment(AzureSecretName::TenantId.into());
+            let client_id = environment(AzureSecretName::ClientId.into());
+            let from_environment = match (
+                tenant_id.clone(),
+                client_id.clone(),
+                environment(AzureSecretName::ClientSecret.into()),
+            ) {
+                (Some(tenant_id), Some(client_id), Some(client_secret)) => {
+                    Some(NativeAzureRequest::ClientSecret {
+                        tenant_id,
+                        client_id,
+                        client_secret: client_secret.map(SecretValue::new),
+                        scope: scope.clone(),
+                        authority: authority.clone(),
+                    })
+                }
+                _ => None,
+            };
+            let probe_imds = env_lookup(AzureSecretName::IdentityEndpoint.into()).is_none()
+                && env_lookup(AzureSecretName::MsiEndpoint.into()).is_none();
+            let AzureCredentialPlan::Chain(chain) = select_native_plan(
+                Some(Sourced::new(
+                    AzureCredentialType::DefaultAzureCredential,
+                    InputSource::Deployment,
+                )),
+                tenant_id,
+                client_id,
+                environment(AzureSecretName::FederatedTokenFile.into()),
+                scope,
+                authority,
+                InputSource::Environment,
+            )?
+            else {
+                unreachable!("DefaultAzureCredential always plans a chain")
+            };
+            let environment_request = from_environment
+                .map(ValidatedAzureRequest::new)
+                .transpose()?;
+            Ok(AzureCredentialPlan::Chain(
+                environment_request
+                    .into_iter()
+                    .chain(chain)
+                    .map(|request| request.with_imds_probe(probe_imds))
+                    .collect(),
+            ))
+        }
+    }
+}
+
 pub(crate) fn select_auth_plan(
     inputs: &AzureAuthInputs,
     env_lookup: &dyn Fn(&str) -> Option<String>,
 ) -> Result<AzureCredentialPlan, Error> {
-    let token = configured_secret(&inputs.azure_ad_token, AZURE_AD_TOKEN_ENV, env_lookup);
-    let tenant_id = configured_string(&inputs.tenant_id, AZURE_TENANT_ID_ENV, env_lookup);
-    let client_id = configured_string(&inputs.client_id, AZURE_CLIENT_ID_ENV, env_lookup);
-    let client_secret =
-        configured_secret(&inputs.client_secret, AZURE_CLIENT_SECRET_ENV, env_lookup);
-    let scope = configured_string(&inputs.azure_scope, AZURE_SCOPE_ENV, env_lookup)
-        .unwrap_or_else(|| Sourced::new(DEFAULT_AZURE_SCOPE.to_string(), InputSource::Environment));
-    let authority = configured_string(
-        &inputs.azure_authority_host,
-        AZURE_AUTHORITY_HOST_ENV,
+    let token = configured_secret(
+        &inputs.azure_ad_token,
+        AzureSecretName::AdToken.into(),
         env_lookup,
     );
-    let selector = configured_string(&inputs.azure_credential, AZURE_CREDENTIAL_ENV, env_lookup)
-        .map(|value| {
-            value
-                .value()
-                .parse::<AzureCredentialType>()
-                .map(|selector| Sourced::new(selector, value.source()))
-        })
-        .transpose()
-        .map_err(|error| {
-            Error::InvalidConfiguration(ErrorDetail::failed(
-                "Azure credential selector parsing",
-                error,
-            ))
-        })?;
+    let tenant_id = configured_string(
+        &inputs.tenant_id,
+        AzureSecretName::TenantId.into(),
+        env_lookup,
+    );
+    let client_id = configured_string(
+        &inputs.client_id,
+        AzureSecretName::ClientId.into(),
+        env_lookup,
+    );
+    let client_secret = configured_secret(
+        &inputs.client_secret,
+        AzureSecretName::ClientSecret.into(),
+        env_lookup,
+    );
+    let scope = configured_string(
+        &inputs.azure_scope,
+        AzureSecretName::Scope.into(),
+        env_lookup,
+    )
+    .unwrap_or_else(|| Sourced::new(DEFAULT_AZURE_SCOPE.to_string(), InputSource::Environment));
+    let authority = configured_string(
+        &inputs.azure_authority_host,
+        AzureSecretName::AuthorityHost.into(),
+        env_lookup,
+    );
+    let selector = configured_string(
+        &inputs.azure_credential,
+        AzureSecretName::Credential.into(),
+        env_lookup,
+    )
+    .map(|value| {
+        value
+            .value()
+            .parse::<AzureCredentialType>()
+            .map(|selector| Sourced::new(selector, value.source()))
+    })
+    .transpose()
+    .map_err(|error| {
+        Error::InvalidConfiguration(ErrorDetail::failed(
+            "Azure credential selector parsing",
+            error,
+        ))
+    })?;
     let federated_token_file = configured_string(
         &inputs.federated_token_file,
-        AZURE_FEDERATED_TOKEN_FILE_ENV,
+        AzureSecretName::FederatedTokenFile.into(),
         env_lookup,
     );
 
@@ -237,7 +400,12 @@ pub(crate) fn select_auth_plan(
         return Ok(AzureCredentialPlan::Missing);
     }
 
-    select_native_plan(
+    let probe_imds = selector
+        .as_ref()
+        .is_none_or(|selector| *selector.value() == AzureCredentialType::DefaultAzureCredential)
+        && env_lookup(AzureSecretName::IdentityEndpoint.into()).is_none()
+        && env_lookup(AzureSecretName::MsiEndpoint.into()).is_none();
+    match select_native_plan(
         selector,
         tenant_id,
         client_id,
@@ -245,7 +413,15 @@ pub(crate) fn select_auth_plan(
         scope,
         authority,
         inputs.enable_azure_ad_token_refresh.source(),
-    )
+    )? {
+        AzureCredentialPlan::Chain(requests) => Ok(AzureCredentialPlan::Chain(
+            requests
+                .into_iter()
+                .map(|request| request.with_imds_probe(probe_imds))
+                .collect(),
+        )),
+        plan => Ok(plan),
+    }
 }
 
 fn select_native_plan(
@@ -509,7 +685,10 @@ mod tests {
     #[derive(Debug)]
     struct FileResolver;
 
+    use crate::native::CredentialAttempt;
+
     struct ChainAcquirer {
+        terminal_on: Option<&'static str>,
         requests: Mutex<Vec<&'static str>>,
         succeed_on: Option<&'static str>,
     }
@@ -518,20 +697,25 @@ mod tests {
         fn acquire(
             &self,
             request: ValidatedAzureRequest,
-        ) -> std::pin::Pin<Box<dyn Future<Output = Result<ResolvedCredential, Error>> + Send + '_>>
-        {
+        ) -> std::pin::Pin<Box<dyn Future<Output = CredentialAttempt> + Send + '_>> {
             let kind = request.kind();
             self.requests.lock().unwrap().push(kind);
             Box::pin(async move {
                 if self.succeed_on == Some(kind) {
-                    Ok(ResolvedCredential::AccessToken {
+                    CredentialAttempt::Resolved(ResolvedCredential::AccessToken {
                         token: SecretValue::new("chain-token"),
                         expires_on: None,
                     })
                 } else {
-                    Err(Error::CredentialAcquisition(
-                        litellm_auth_types::ErrorDetail::Unresolved { subject: kind },
-                    ))
+                    let error =
+                        Error::CredentialAcquisition(litellm_auth_types::ErrorDetail::Unresolved {
+                            subject: kind,
+                        });
+                    if self.terminal_on == Some(kind) {
+                        CredentialAttempt::Failed(error)
+                    } else {
+                        CredentialAttempt::Unavailable(error)
+                    }
                 }
             })
         }
@@ -690,9 +874,210 @@ mod tests {
         assert_eq!(resolved, Some(SecretValue::new("rotated-assertion")));
     }
 
+    #[derive(Debug)]
+    struct UnavailableImdsClient;
+
+    impl azure_core::http::HttpClient for UnavailableImdsClient {
+        fn execute_request<'life0, 'life1, 'async_trait>(
+            &'life0 self,
+            _request: &'life1 azure_core::http::Request,
+        ) -> std::pin::Pin<
+            Box<
+                dyn Future<Output = azure_core::Result<azure_core::http::AsyncRawResponse>>
+                    + Send
+                    + 'async_trait,
+            >,
+        >
+        where
+            'life0: 'async_trait,
+            'life1: 'async_trait,
+            Self: 'async_trait,
+        {
+            Box::pin(std::future::pending())
+        }
+    }
+
+    struct OfflineChainAcquirer(super::NativeAzureTokenAcquirer);
+
+    impl AzureTokenAcquirer for OfflineChainAcquirer {
+        fn acquire(
+            &self,
+            request: ValidatedAzureRequest,
+        ) -> std::pin::Pin<Box<dyn Future<Output = CredentialAttempt> + Send + '_>> {
+            Box::pin(async move {
+                if request.kind() == "developer-tools" {
+                    return CredentialAttempt::Resolved(ResolvedCredential::AccessToken {
+                        token: SecretValue::new("cli-token"),
+                        expires_on: None,
+                    });
+                }
+                self.0.acquire(request).await
+            })
+        }
+    }
+
+    #[rstest::rstest]
+    #[case::default_off_azure("DefaultAzureCredential", None, true)]
+    #[case::explicit_managed_identity("ManagedIdentityCredential", None, false)]
+    #[case::app_service("DefaultAzureCredential", Some("http://app-service.test"), false)]
+    #[tokio::test(start_paused = true)]
+    async fn only_default_imds_discovery_is_bounded(
+        #[case] selector: &str,
+        #[case] identity_endpoint: Option<&str>,
+        #[case] expect_cli: bool,
+    ) {
+        let native = super::NativeAzureTokenAcquirer::with_transport(
+            4,
+            azure_core::http::Transport::new(Arc::new(UnavailableImdsClient)),
+        );
+        let service = AzureAuthService::with_acquirer(Arc::new(OfflineChainAcquirer(native)));
+        let inputs = AzureAuthInputs {
+            azure_credential: crate::types::ConfigValue::Value(Sourced::new(
+                selector.into(),
+                InputSource::Deployment,
+            )),
+            enable_azure_ad_token_refresh: Sourced::new(true, InputSource::Deployment),
+            ..Default::default()
+        };
+        let environment = |name: &str| {
+            (name == "IDENTITY_ENDPOINT")
+                .then_some(identity_endpoint)
+                .flatten()
+                .map(str::to_string)
+        };
+        let outcome = tokio::time::timeout(
+            std::time::Duration::from_secs(2),
+            service.get_azure_ad_token(&inputs, &environment),
+        )
+        .await;
+
+        assert_eq!(outcome.is_ok(), expect_cli);
+        if expect_cli {
+            assert_eq!(
+                outcome.unwrap().unwrap().unwrap().value().secret().expose(),
+                "cli-token"
+            );
+        }
+    }
+
+    #[rstest::rstest]
+    #[case::app_service(Some("IDENTITY_ENDPOINT"), false)]
+    #[case::legacy_app_service(Some("MSI_ENDPOINT"), false)]
+    #[case::off_azure(None, true)]
+    #[tokio::test(start_paused = true)]
+    async fn default_chain_uses_endpoints_from_the_secret_snapshot(
+        #[case] endpoint: Option<&str>,
+        #[case] expect_cli: bool,
+        #[values(false, true)] consumer_identity: bool,
+    ) {
+        use litellm_secrets::source::SecretSource;
+
+        struct EndpointSecrets<'a>(Option<&'a str>);
+
+        impl SecretSource for EndpointSecrets<'_> {
+            fn get_secret_str<'a>(
+                &'a self,
+                name: &'a str,
+            ) -> std::pin::Pin<
+                Box<
+                    dyn Future<Output = Result<Option<SecretValue>, litellm_secrets::Error>>
+                        + Send
+                        + 'a,
+                >,
+            > {
+                Box::pin(async move {
+                    Ok((self.0 == Some(name)).then(|| SecretValue::new("http://app-service.test")))
+                })
+            }
+        }
+
+        let snapshot = EndpointSecrets(endpoint)
+            .resolve(SECRET_NAMES)
+            .await
+            .unwrap();
+        let native = super::NativeAzureTokenAcquirer::with_transport(
+            4,
+            azure_core::http::Transport::new(Arc::new(UnavailableImdsClient)),
+        );
+        let service = AzureAuthService::with_acquirer(Arc::new(OfflineChainAcquirer(native)));
+        let lookup = |name: &str| snapshot.get(name);
+        let scope = "https://service.test/.default";
+        let outcome = tokio::time::timeout(std::time::Duration::from_secs(2), async {
+            if consumer_identity {
+                service
+                    .identity_token(&super::AzureIdentity::Default, scope, &lookup)
+                    .await
+            } else {
+                service
+                    .get_azure_ad_token(
+                        &AzureAuthInputs::default_credential_for_scope(scope),
+                        &lookup,
+                    )
+                    .await
+                    .map(|token| token.map(Sourced::into_value))
+            }
+        })
+        .await;
+
+        assert_eq!(outcome.is_ok(), expect_cli);
+        if expect_cli {
+            assert_eq!(
+                outcome.unwrap().unwrap().unwrap().secret().expose(),
+                "cli-token"
+            );
+        }
+    }
+
+    #[rstest::rstest]
+    #[case::litellm_provider_settings_are_ignored(
+        &[("AZURE_AD_TOKEN", "provider-token"), ("AZURE_CREDENTIAL", "ClientSecretCredential")],
+        &["managed-identity", "developer-tools"]
+    )]
+    #[case::environment_client_secret_comes_first(
+        &[("AZURE_TENANT_ID", "tenant"), ("AZURE_CLIENT_ID", "client"), ("AZURE_CLIENT_SECRET", "secret")],
+        &["client-secret", "managed-identity", "developer-tools"]
+    )]
+    #[case::aks_workload_identity(
+        &[("AZURE_TENANT_ID", "tenant"), ("AZURE_CLIENT_ID", "client"), ("AZURE_FEDERATED_TOKEN_FILE", "/var/run/token")],
+        &["workload-identity", "managed-identity", "developer-tools"]
+    )]
+    #[tokio::test]
+    async fn the_default_identity_follows_azure_identity_defaults(
+        #[case] environment: &[(&str, &str)],
+        #[case] expected: &[&str],
+    ) {
+        let acquirer = Arc::new(ChainAcquirer {
+            terminal_on: None,
+            requests: Mutex::new(Vec::new()),
+            succeed_on: Some("developer-tools"),
+        });
+        let service = AzureAuthService::with_acquirer(acquirer.clone());
+        let lookup = |name: &str| {
+            environment
+                .iter()
+                .find(|(key, _)| *key == name)
+                .map(|(_, value)| (*value).to_owned())
+        };
+
+        let credential = service
+            .identity_token(
+                &super::AzureIdentity::Default,
+                "https://redis.azure.com/.default",
+                &lookup,
+            )
+            .await
+            .unwrap()
+            .unwrap();
+
+        assert_eq!(credential.secret().expose(), "chain-token");
+        assert_eq!(*acquirer.requests.lock().unwrap(), expected);
+    }
+
+    #[rstest::rstest]
     #[tokio::test]
     async fn default_chain_uses_declared_order_and_stops_after_success() {
         let acquirer = Arc::new(ChainAcquirer {
+            terminal_on: None,
             requests: Mutex::new(Vec::new()),
             succeed_on: Some("developer-tools"),
         });
@@ -715,9 +1100,13 @@ mod tests {
         );
     }
 
+    #[rstest::rstest]
+    #[case::all_unavailable(false)]
+    #[case::terminal_failure_after_unavailable(true)]
     #[tokio::test]
-    async fn chain_reports_each_acquisition_failure() {
+    async fn chain_reports_each_acquisition_failure(#[case] terminal: bool) {
         let acquirer = Arc::new(ChainAcquirer {
+            terminal_on: terminal.then_some("developer-tools"),
             requests: Mutex::new(Vec::new()),
             succeed_on: None,
         });
@@ -744,6 +1133,27 @@ mod tests {
         );
     }
 
+    #[rstest::rstest]
+    #[tokio::test]
+    async fn a_configured_identity_failure_does_not_fall_back_to_a_developer_identity() {
+        let acquirer = Arc::new(ChainAcquirer {
+            terminal_on: Some("managed-identity"),
+            requests: Mutex::new(Vec::new()),
+            succeed_on: Some("developer-tools"),
+        });
+        let service = AzureAuthService::with_acquirer(acquirer.clone());
+        let inputs = AzureAuthInputs {
+            enable_azure_ad_token_refresh: Sourced::new(true, InputSource::Deployment),
+            ..Default::default()
+        };
+        let error = service
+            .get_azure_ad_token(&inputs, &|_| None)
+            .await
+            .unwrap_err();
+        assert_eq!(*acquirer.requests.lock().unwrap(), ["managed-identity"]);
+        assert!(matches!(error, Error::CredentialChain { earlier, .. } if earlier.is_empty()));
+    }
+
     #[derive(Debug)]
     struct CallerToken(&'static str);
 

litellm-rust/crates/auth-gcp/Cargo.toml

@@ -6,7 +6,7 @@ license.workspace = true
 repository.workspace = true
 
 [features]
-google-sdk = ["dep:google-cloud-auth", "dep:http"]
+google-sdk = ["dep:http"]
 
 [dependencies]
 litellm-secret-value.workspace = true
@@ -21,10 +21,14 @@ tokio.workspace = true
 veil.workspace = true
 
 gcp_auth = "0.12.7"
-google-cloud-auth = { workspace = true, optional = true }
+google-cloud-auth.workspace = true
 http = { workspace = true, optional = true }
 
 [dev-dependencies]
+time = { workspace = true, features = ["formatting"] }
+axum = { workspace = true, features = ["json"] }
 litellm-error.workspace = true
 rstest.workspace = true
+rustls.workspace = true
 tempfile.workspace = true
+tokio = { workspace = true, features = ["test-util"] }

litellm-rust/crates/auth-gcp/src/auth.rs

@@ -1,20 +1,11 @@
-use std::{future::Future, pin::Pin, sync::Arc};
+use std::sync::Arc;
 
-use gcp_auth::{CustomServiceAccount, TokenProvider};
-use litellm_auth_types::{CredentialPlacement, Error, ErrorDetail, http::apply_credential};
-use moka::future::Cache;
-use serde_json::Value;
-use sha2::{Digest, Sha256};
+use litellm_auth_types::{CredentialPlacement, Error, http::apply_credential};
 
 use crate::{
-    config::{
-        CredentialSource, VertexConfig, credential_source, get_vertex_ai_project,
-        is_base64_encoded_json, names_a_credentials_file, non_empty_env,
-    },
-    constants::{
-        CLOUD_PLATFORM_SCOPE, GOOGLE_OAUTH_TOKEN_ENDPOINT, VERTEX_AI_API_KEY_ENV,
-        VERTEXAI_API_KEY_ENV,
-    },
+    config::{VertexConfig, credential_source, get_vertex_ai_project, non_empty_env},
+    constants::{VERTEX_AI_API_KEY_ENV, VERTEXAI_API_KEY_ENV},
+    identity::{GcpAuth, GcpProviderLoader, GcpTokenSource},
 };
 
 pub struct VertexEnvironment {
@@ -22,28 +13,15 @@ pub struct VertexEnvironment {
     pub project_id: String,
 }
 
-struct VertexAccessToken {
-    token: String,
-    project_id: String,
-}
-
-#[derive(Clone)]
+#[derive(Clone, Default)]
 pub struct VertexAuth {
-    providers: Cache<CredentialCacheKey, Arc<dyn VertexTokenSource>>,
-    loader: Arc<dyn VertexProviderLoader>,
-}
-
-impl Default for VertexAuth {
-    fn default() -> Self {
-        Self::new(Arc::new(GcpProviderLoader))
-    }
+    identity: GcpAuth,
 }
 
 impl VertexAuth {
-    pub fn new(loader: Arc<dyn VertexProviderLoader>) -> Self {
+    pub fn new(loader: Arc<dyn GcpProviderLoader>) -> Self {
         Self {
-            providers: Cache::builder().max_capacity(64).build(),
-            loader,
+            identity: GcpAuth::new(loader),
         }
     }
 
@@ -52,7 +30,10 @@ impl VertexAuth {
         config: &VertexConfig,
         env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
     ) -> Result<String, Error> {
-        self.load_provider(config, env_lookup).await?.token().await
+        self.identity
+            .access_token(credential_source(config, env_lookup))
+            .await
+            .map(|token| token.expose().to_owned())
     }
 
     pub async fn validate_environment(
@@ -77,10 +58,18 @@ impl VertexAuth {
             (true, _) => None,
             (false, Some(token)) => Some(token),
             (false, None) => {
-                let access = self.get_access_token(config, env_lookup).await?;
+                let provider = self.load_provider(config, env_lookup).await?;
+                let (token, project_id) = match project_id {
+                    Some(project_id) => (provider.token().await?, project_id),
+                    None => tokio::try_join!(provider.token(), provider.project_id())?,
+                };
                 return Ok(VertexEnvironment {
-                    headers: apply_credential(headers, &access.token, CredentialPlacement::Bearer)?,
-                    project_id: project_id.unwrap_or(access.project_id),
+                    headers: apply_credential(
+                        headers,
+                        token.expose(),
+                        CredentialPlacement::Bearer,
+                    )?,
+                    project_id,
                 });
             }
         };
@@ -104,248 +93,171 @@ impl VertexAuth {
         })
     }
 
-    async fn get_access_token(
-        &self,
-        config: &VertexConfig,
-        env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
-    ) -> Result<VertexAccessToken, Error> {
-        let provider = self.load_provider(config, env_lookup).await?;
-        let (token, project_id) = tokio::try_join!(provider.token(), provider.project_id())?;
-        Ok(VertexAccessToken { token, project_id })
-    }
-
     async fn load_provider(
         &self,
         config: &VertexConfig,
         env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
-    ) -> Result<Arc<dyn VertexTokenSource>, Error> {
-        let source = credential_source(config, env_lookup);
-        let key = source.cache_key();
-        self.providers
-            .try_get_with(key, self.loader.load(source))
+    ) -> Result<Arc<dyn GcpTokenSource>, Error> {
+        self.identity
+            .load_provider(credential_source(config, env_lookup))
             .await
-            .map_err(|error| (*error).clone())
     }
 }
 
-pub trait VertexTokenSource: Send + Sync {
-    fn project_id(&self) -> VertexAuthFuture<'_, String>;
-    fn token(&self) -> VertexAuthFuture<'_, String>;
-}
-
-pub trait VertexProviderLoader: Send + Sync {
-    fn load(&self, source: CredentialSource) -> VertexAuthFuture<'_, Arc<dyn VertexTokenSource>>;
-}
-
-pub type VertexAuthFuture<'a, T> = Pin<Box<dyn Future<Output = Result<T, Error>> + Send + 'a>>;
-
-struct GcpTokenSource(Arc<dyn TokenProvider>);
-
-impl VertexTokenSource for GcpTokenSource {
-    fn project_id(&self) -> VertexAuthFuture<'_, String> {
-        Box::pin(async move {
-            self.0
-                .project_id()
-                .await
-                .map(|project| project.to_string())
-                .map_err(auth_acquisition_error)
-        })
-    }
-
-    fn token(&self) -> VertexAuthFuture<'_, String> {
-        Box::pin(async move {
-            self.0
-                .token(&[CLOUD_PLATFORM_SCOPE])
-                .await
-                .map(|token| token.as_str().to_string())
-                .map_err(auth_acquisition_error)
-        })
-    }
-}
-
-struct GcpProviderLoader;
-
-impl VertexProviderLoader for GcpProviderLoader {
-    fn load(&self, source: CredentialSource) -> VertexAuthFuture<'_, Arc<dyn VertexTokenSource>> {
-        Box::pin(async move {
-            let provider: Arc<dyn TokenProvider> = match source {
-                CredentialSource::Inline(configured) => Arc::new(
-                    CustomServiceAccount::from_json(validate_request_credentials(
-                        configured.expose(),
-                    )?)
-                    .map_err(auth_acquisition_error)?,
-                ),
-                CredentialSource::Trusted(configured) => {
-                    Arc::new(load_trusted_service_account(configured.expose())?)
-                }
-                CredentialSource::ApplicationCredentials(path) => {
-                    Arc::new(CustomServiceAccount::from_file(path).map_err(auth_acquisition_error)?)
-                }
-                CredentialSource::Adc => {
-                    gcp_auth::provider().await.map_err(auth_acquisition_error)?
-                }
-            };
-            Ok(Arc::new(GcpTokenSource(provider)) as Arc<dyn VertexTokenSource>)
-        })
-    }
-}
-
-fn validate_request_credentials(configured: &str) -> Result<&str, Error> {
-    let token_uri = serde_json::from_str::<Value>(configured)
-        .ok()
-        .and_then(|credentials| {
-            credentials
-                .get("token_uri")
-                .and_then(Value::as_str)
-                .map(str::to_string)
-        });
-    if token_uri.as_deref() != Some(GOOGLE_OAUTH_TOKEN_ENDPOINT) {
-        return Err(Error::InvalidConfiguration(ErrorDetail::InvalidType {
-            field: "request-controlled vertex_credentials token_uri".into(),
-            expected: "the canonical Google OAuth token endpoint",
-        }));
-    }
-    Ok(configured)
-}
-
-impl CredentialSource {
-    fn cache_key(&self) -> CredentialCacheKey {
-        match self {
-            Self::Inline(configured) => {
-                CredentialCacheKey::Inline(Sha256::digest(configured.expose()).into())
-            }
-            Self::Trusted(configured) => {
-                CredentialCacheKey::Trusted(Sha256::digest(configured.expose()).into())
-            }
-            Self::ApplicationCredentials(path) => {
-                CredentialCacheKey::ApplicationCredentials(path.clone())
-            }
-            Self::Adc => CredentialCacheKey::Adc,
-        }
-    }
-}
-
-#[derive(Clone, Debug, Hash, PartialEq, Eq)]
-enum CredentialCacheKey {
-    Inline([u8; 32]),
-    Trusted([u8; 32]),
-    ApplicationCredentials(String),
-    Adc,
-}
-
-fn load_trusted_service_account(configured: &str) -> Result<CustomServiceAccount, Error> {
-    if is_base64_encoded_json(configured) {
-        return Err(Error::InvalidConfiguration(
-            ErrorDetail::Base64EncodedJson {
-                field: "vertex_credentials",
-            },
-        ));
-    }
-    if names_a_credentials_file(configured) {
-        CustomServiceAccount::from_file(configured)
-    } else {
-        CustomServiceAccount::from_json(configured)
-    }
-    .map_err(auth_acquisition_error)
-}
-
-fn auth_acquisition_error(error: gcp_auth::Error) -> Error {
-    Error::CredentialAcquisition(litellm_auth_types::ErrorDetail::failed(
-        "Vertex AI credentials",
-        error,
-    ))
-}
-
 #[cfg(test)]
 mod tests {
+    use std::{
+        collections::{BTreeMap, BTreeSet},
+        sync::{
+            Arc,
+            atomic::{AtomicUsize, Ordering},
+        },
+    };
+
+    use crate::{
+        CredentialSource, GcpAuthFuture, GcpProviderLoader, GcpTokenSource, VertexAuth,
+        VertexConfig, get_vertex_ai_location, secret_names,
+    };
     use litellm_secret_value::SecretValue;
+    use rstest::rstest;
+    use serde_json::{Value, json};
 
-    use super::*;
-
-    #[rstest::rstest]
-    #[case::canonical_endpoint(r#"{"token_uri":"https://oauth2.googleapis.com/token"}"#, true)]
-    #[case::noncanonical_endpoint(r#"{"token_uri":"http://127.0.0.1/token"}"#, false)]
-    #[case::missing_endpoint("{}", false)]
-    fn request_credentials_require_canonical_token_endpoint(
-        #[case] credentials: &str,
-        #[case] accepted: bool,
-    ) {
-        assert_eq!(validate_request_credentials(credentials).is_ok(), accepted);
+    struct FakeProvider {
+        project: Option<&'static str>,
+        calls: Arc<AtomicUsize>,
     }
 
-    #[rstest::rstest]
-    #[case::missing_file(
-        "/nonexistent/vertex-key.json",
-        "failed to open application credentials file"
-    )]
-    #[case::malformed_json(
-        " {\"type\": \"service_account\",",
-        "failed to deserialize ApplicationCredentials"
-    )]
-    #[case::quoted_json(
-        "'{\"type\": \"service_account\"}'",
-        "failed to deserialize ApplicationCredentials"
-    )]
-    #[tokio::test]
-    async fn trusted_credentials_are_read_as_a_file_unless_they_are_json(
-        #[case] configured: &str,
-        #[case] expected: &str,
-    ) {
-        let Err(error) = GcpProviderLoader
-            .load(CredentialSource::Trusted(SecretValue::new(configured)))
-            .await
-        else {
-            panic!("{configured:?} should not load");
-        };
-        let report = litellm_error::Report::new(&error).to_string();
-        assert!(report.contains(expected), "{report}");
-    }
+    impl GcpTokenSource for FakeProvider {
+        fn project_id(&self) -> GcpAuthFuture<'_, String> {
+            self.calls.fetch_add(1, Ordering::SeqCst);
+            Box::pin(async move {
+                self.project.map(str::to_owned).ok_or(
+                    litellm_auth_types::Error::CredentialAcquisition(
+                        litellm_auth_types::ErrorDetail::Unresolved {
+                            subject: "credential project",
+                        },
+                    ),
+                )
+            })
+        }
 
-    #[rstest::rstest]
-    #[case::padded("eyJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCJ9")]
-    #[case::quoted_prefixed_unpadded("\"base64:eyJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCJ9\"")]
-    #[case::url_safe_wrapped("eyJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCIsICJr\nZXkiOiAiPz8_In0=")]
-    #[tokio::test]
-    async fn base64_encoded_credentials_are_named_as_base64(#[case] configured: &str) {
-        assert!(
-            trusted_load_error(configured)
-                .await
-                .contains("is base64-encoded JSON, not the JSON itself")
-        );
-    }
-
-    async fn trusted_load_error(configured: &str) -> String {
-        match GcpProviderLoader
-            .load(CredentialSource::Trusted(SecretValue::new(configured)))
-            .await
-        {
-            Ok(_) => panic!("{configured:?} should not load"),
-            Err(error) => litellm_error::Report::new(&error).to_string(),
+        fn token(&self) -> GcpAuthFuture<'_, SecretValue> {
+            self.calls.fetch_add(1, Ordering::SeqCst);
+            Box::pin(async { Ok(SecretValue::new("adc-token")) })
         }
     }
 
-    #[rstest::rstest]
-    #[tokio::test]
-    async fn a_relative_key_file_named_like_json_is_read_as_a_file() {
-        let key = tempfile::Builder::new()
-            .prefix("{vertex")
-            .suffix(".json")
-            .tempfile_in(".")
-            .unwrap();
-        std::fs::write(key.path(), r#"{"project_id":"file-project"}"#).unwrap();
-        let relative_name = key.path().file_name().unwrap().to_str().unwrap();
-        let absolute_path = std::fs::canonicalize(key.path()).unwrap();
-        assert_eq!(
-            trusted_load_error(relative_name).await,
-            trusted_load_error(absolute_path.to_str().unwrap()).await
-        );
+    struct FakeLoader {
+        loads: Arc<AtomicUsize>,
+        provider: Arc<dyn GcpTokenSource>,
     }
 
-    #[rstest::rstest]
-    fn inline_and_trusted_credentials_never_share_a_cache_entry() {
-        assert_ne!(
-            CredentialSource::Inline(SecretValue::new("same-value")).cache_key(),
-            CredentialSource::Trusted(SecretValue::new("same-value")).cache_key()
-        );
+    impl GcpProviderLoader for FakeLoader {
+        fn load(&self, _source: CredentialSource) -> GcpAuthFuture<'_, Arc<dyn GcpTokenSource>> {
+            let loads = self.loads.clone();
+            let provider = self.provider.clone();
+            Box::pin(async move {
+                loads.fetch_add(1, Ordering::SeqCst);
+                Ok(provider)
+            })
+        }
+    }
+
+    fn config(value: Value) -> VertexConfig {
+        VertexConfig::from_sourced_optional_params(value.as_object().unwrap(), &BTreeMap::new())
+            .unwrap()
+    }
+
+    fn auth(calls: Arc<AtomicUsize>, loads: Arc<AtomicUsize>) -> VertexAuth {
+        let provider: Arc<dyn GcpTokenSource> = Arc::new(FakeProvider {
+            calls,
+            project: Some("adc-project"),
+        });
+        VertexAuth::new(Arc::new(FakeLoader { loads, provider }))
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn secret_names_cover_environment_reads() {
+        let seen = Arc::new(std::sync::Mutex::new(BTreeSet::<String>::new()));
+        let recorded = seen.clone();
+        let env = |name: &str| {
+            recorded.lock().unwrap().insert(name.to_string());
+            None
+        };
+        let auth = auth(Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0)));
+        auth.validate_environment(Vec::new(), None, &VertexConfig::default(), &env)
+            .await
+            .unwrap();
+        get_vertex_ai_location(&VertexConfig::default(), &env);
+        let allowed = secret_names()
+            .iter()
+            .map(|name| (*name).to_owned())
+            .collect::<BTreeSet<_>>();
+        assert!(seen.lock().unwrap().is_subset(&allowed));
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn explicit_token_and_header_do_not_acquire_adc() {
+        let loads = Arc::new(AtomicUsize::new(0));
+        let auth = auth(Arc::new(AtomicUsize::new(0)), loads.clone());
+        let configured = config(json!({"vertex_project":"project-1"}));
+        let explicit = auth
+            .validate_environment(Vec::new(), Some("access-token"), &configured, &|_| None)
+            .await
+            .unwrap();
+        assert_eq!(explicit.headers[0].1, "Bearer access-token");
+        let existing = auth
+            .validate_environment(
+                vec![("authorization".into(), "Bearer existing".into())],
+                None,
+                &configured,
+                &|_| None,
+            )
+            .await
+            .unwrap();
+        assert_eq!(existing.headers[0].1, "Bearer existing");
+        assert_eq!(loads.load(Ordering::SeqCst), 0);
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn provider_is_reused_across_authentication_calls() {
+        let calls = Arc::new(AtomicUsize::new(0));
+        let loads = Arc::new(AtomicUsize::new(0));
+        let auth = auth(calls.clone(), loads.clone());
+        let config = VertexConfig::default();
+        let environment = |_: &str| None;
+        let (first, second) = tokio::try_join!(
+            auth.validate_environment(Vec::new(), None, &config, &environment),
+            auth.validate_environment(Vec::new(), None, &config, &environment),
+        )
+        .unwrap();
+        assert_eq!(first.project_id, "adc-project");
+        assert_eq!(first.headers[0].1, "Bearer adc-token");
+        assert_eq!(first.project_id, second.project_id);
+        assert_eq!(first.headers, second.headers);
+        assert_eq!(loads.load(Ordering::SeqCst), 1);
+        assert_eq!(calls.load(Ordering::SeqCst), 4);
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn an_explicit_project_does_not_require_project_metadata_in_the_credentials() {
+        let auth = VertexAuth::new(Arc::new(FakeLoader {
+            loads: Arc::new(AtomicUsize::new(0)),
+            provider: Arc::new(FakeProvider {
+                calls: Arc::new(AtomicUsize::new(0)),
+                project: None,
+            }),
+        }));
+        let configured = VertexConfig::new(None, Some("configured-project".into()), None);
+        let environment = auth
+            .validate_environment(Vec::new(), None, &configured, &|_| None)
+            .await
+            .unwrap();
+        assert_eq!(environment.project_id, "configured-project");
+        assert_eq!(environment.headers[0].1, "Bearer adc-token");
     }
 }

litellm-rust/crates/auth-gcp/src/config.rs

@@ -146,6 +146,20 @@ pub enum CredentialSource {
     Adc,
 }
 
+impl CredentialSource {
+    pub fn from_vertex_environment(env_lookup: &dyn Fn(&str) -> Option<String>) -> Self {
+        VertexParams::CREDENTIALS
+            .resolve(&|_| None, env_lookup)
+            .map(|configured| Self::Trusted(SecretValue::new(configured)))
+            .unwrap_or_else(|| Self::application_default(env_lookup))
+    }
+
+    pub fn application_default(env_lookup: &dyn Fn(&str) -> Option<String>) -> Self {
+        non_empty_env(env_lookup, GOOGLE_APPLICATION_CREDENTIALS_ENV)
+            .map_or(Self::Adc, Self::ApplicationCredentials)
+    }
+}
+
 pub(crate) fn credential_source(
     config: &VertexConfig,
     env_lookup: &dyn Fn(&str) -> Option<String>,
@@ -158,12 +172,7 @@ pub(crate) fn credential_source(
             }
         };
     }
-    if let Some(configured) = VertexParams::CREDENTIALS.resolve(&|_| None, env_lookup) {
-        return CredentialSource::Trusted(SecretValue::new(configured));
-    }
-    non_empty_env(env_lookup, GOOGLE_APPLICATION_CREDENTIALS_ENV)
-        .map(CredentialSource::ApplicationCredentials)
-        .unwrap_or(CredentialSource::Adc)
+    CredentialSource::from_vertex_environment(env_lookup)
 }
 
 fn optional_credentials(
@@ -319,6 +328,24 @@ mod tests {
         ));
     }
 
+    #[rstest::rstest]
+    fn application_default_ignores_vertex_credentials() {
+        let env = |name: &str| match name {
+            VERTEXAI_CREDENTIALS_ENV => Some("environment-json".to_string()),
+            GOOGLE_APPLICATION_CREDENTIALS_ENV => Some("adc.json".to_string()),
+            _ => None,
+        };
+        assert!(
+            matches!(CredentialSource::application_default(&env), CredentialSource::ApplicationCredentials(path) if path == "adc.json")
+        );
+        assert!(matches!(
+            CredentialSource::application_default(
+                &|name| (name == VERTEXAI_CREDENTIALS_ENV).then(|| "environment-json".into())
+            ),
+            CredentialSource::Adc
+        ));
+    }
+
     #[rstest::rstest]
     fn params_become_a_deployment_sourced_config() {
         let params = VertexParams {

litellm-rust/crates/auth-gcp/src/identity.rs

@@ -0,0 +1,445 @@
+use crate::{
+    config::{CredentialSource, is_base64_encoded_json, names_a_credentials_file},
+    constants::{CLOUD_PLATFORM_SCOPE, GOOGLE_OAUTH_TOKEN_ENDPOINT},
+};
+use gcp_auth::{CustomServiceAccount, TokenProvider};
+use litellm_auth_types::{Error, ErrorDetail};
+use litellm_secret_value::SecretValue;
+use moka::future::Cache;
+use serde_json::Value;
+use sha2::{Digest, Sha256};
+use std::{future::Future, pin::Pin, sync::Arc};
+
+#[derive(Clone)]
+pub struct GcpAuth {
+    providers: Cache<CredentialCacheKey, Arc<dyn GcpTokenSource>>,
+    loader: Arc<dyn GcpProviderLoader>,
+}
+
+impl Default for GcpAuth {
+    fn default() -> Self {
+        Self::new(Arc::new(NativeProviderLoader))
+    }
+}
+
+impl GcpAuth {
+    pub fn new(loader: Arc<dyn GcpProviderLoader>) -> Self {
+        Self {
+            providers: Cache::builder().max_capacity(64).build(),
+            loader,
+        }
+    }
+
+    pub async fn access_token(&self, source: CredentialSource) -> Result<SecretValue, Error> {
+        self.load_provider(source).await?.token().await
+    }
+
+    pub(crate) async fn load_provider(
+        &self,
+        source: CredentialSource,
+    ) -> Result<Arc<dyn GcpTokenSource>, Error> {
+        self.providers
+            .try_get_with(source.cache_key(), self.loader.load(source))
+            .await
+            .map_err(|error| (*error).clone())
+    }
+}
+
+pub trait GcpTokenSource: Send + Sync {
+    fn project_id(&self) -> GcpAuthFuture<'_, String>;
+    fn token(&self) -> GcpAuthFuture<'_, SecretValue>;
+}
+
+pub trait GcpProviderLoader: Send + Sync {
+    fn load(&self, source: CredentialSource) -> GcpAuthFuture<'_, Arc<dyn GcpTokenSource>>;
+}
+
+pub type GcpAuthFuture<'a, T> = Pin<Box<dyn Future<Output = Result<T, Error>> + Send + 'a>>;
+
+struct ServiceAccountTokenSource(Arc<dyn TokenProvider>);
+
+impl GcpTokenSource for ServiceAccountTokenSource {
+    fn project_id(&self) -> GcpAuthFuture<'_, String> {
+        Box::pin(async move {
+            self.0
+                .project_id()
+                .await
+                .map(|project| project.to_string())
+                .map_err(auth_acquisition_error)
+        })
+    }
+
+    fn token(&self) -> GcpAuthFuture<'_, SecretValue> {
+        Box::pin(async move {
+            self.0
+                .token(&[CLOUD_PLATFORM_SCOPE])
+                .await
+                .map(|token| SecretValue::new(token.as_str()))
+                .map_err(auth_acquisition_error)
+        })
+    }
+}
+
+struct SdkTokenSource {
+    credentials: google_cloud_auth::credentials::AccessTokenCredentials,
+    project_id: Option<String>,
+}
+
+impl GcpTokenSource for SdkTokenSource {
+    fn project_id(&self) -> GcpAuthFuture<'_, String> {
+        Box::pin(async move {
+            self.project_id
+                .clone()
+                .ok_or(Error::CredentialAcquisition(ErrorDetail::Unresolved {
+                    subject: "GCP resource project",
+                }))
+        })
+    }
+
+    fn token(&self) -> GcpAuthFuture<'_, SecretValue> {
+        Box::pin(async move {
+            self.credentials
+                .access_token()
+                .await
+                .map(|token| SecretValue::new(token.token))
+                .map_err(auth_acquisition_error)
+        })
+    }
+}
+
+async fn load_application_credentials(path: &str) -> Result<Arc<dyn GcpTokenSource>, Error> {
+    let contents = tokio::fs::read_to_string(path)
+        .await
+        .map_err(auth_acquisition_error)?;
+    load_credentials_json(&contents)
+}
+
+fn load_credentials_json(contents: &str) -> Result<Arc<dyn GcpTokenSource>, Error> {
+    let configured: Value = serde_json::from_str(contents).map_err(auth_acquisition_error)?;
+    let kind = configured.get("type").and_then(Value::as_str);
+    let project_id = configured
+        .get("project_id")
+        .and_then(Value::as_str)
+        .map(str::to_string);
+    let credentials = match kind {
+        Some("authorized_user") => {
+            google_cloud_auth::credentials::user_account::Builder::new(configured)
+                .with_scopes([CLOUD_PLATFORM_SCOPE])
+                .build_access_token_credentials()
+        }
+        Some("impersonated_service_account") => {
+            google_cloud_auth::credentials::impersonated::Builder::new(configured)
+                .with_scopes([CLOUD_PLATFORM_SCOPE])
+                .build_access_token_credentials()
+        }
+        Some("external_account") => {
+            google_cloud_auth::credentials::external_account::Builder::new(configured)
+                .with_scopes([CLOUD_PLATFORM_SCOPE])
+                .build_access_token_credentials()
+        }
+        Some("service_account") | None => {
+            let provider =
+                CustomServiceAccount::from_json(contents).map_err(auth_acquisition_error)?;
+            return Ok(Arc::new(ServiceAccountTokenSource(Arc::new(provider))));
+        }
+        Some(_) => {
+            return Err(Error::InvalidConfiguration(ErrorDetail::InvalidType {
+                field: "Google application credentials type".into(),
+                expected: "a supported Google credential type",
+            }));
+        }
+    }
+    .map_err(auth_acquisition_error)?;
+    Ok(Arc::new(SdkTokenSource {
+        credentials,
+        project_id,
+    }))
+}
+
+struct NativeProviderLoader;
+
+impl GcpProviderLoader for NativeProviderLoader {
+    fn load(&self, source: CredentialSource) -> GcpAuthFuture<'_, Arc<dyn GcpTokenSource>> {
+        Box::pin(async move {
+            let provider: Arc<dyn TokenProvider> = match source {
+                CredentialSource::Inline(configured) => Arc::new(
+                    CustomServiceAccount::from_json(validate_request_credentials(
+                        configured.expose(),
+                    )?)
+                    .map_err(auth_acquisition_error)?,
+                ),
+                CredentialSource::Trusted(configured) => {
+                    return load_trusted_credentials(configured.expose()).await;
+                }
+                CredentialSource::ApplicationCredentials(path) => {
+                    return load_application_credentials(&path).await;
+                }
+                CredentialSource::Adc => {
+                    gcp_auth::provider().await.map_err(auth_acquisition_error)?
+                }
+            };
+            Ok(Arc::new(ServiceAccountTokenSource(provider)) as Arc<dyn GcpTokenSource>)
+        })
+    }
+}
+
+fn validate_request_credentials(configured: &str) -> Result<&str, Error> {
+    let token_uri = serde_json::from_str::<Value>(configured)
+        .ok()
+        .and_then(|credentials| {
+            credentials
+                .get("token_uri")
+                .and_then(Value::as_str)
+                .map(str::to_string)
+        });
+    if token_uri.as_deref() != Some(GOOGLE_OAUTH_TOKEN_ENDPOINT) {
+        return Err(Error::InvalidConfiguration(ErrorDetail::InvalidType {
+            field: "request-controlled vertex_credentials token_uri".into(),
+            expected: "the canonical Google OAuth token endpoint",
+        }));
+    }
+    Ok(configured)
+}
+
+impl CredentialSource {
+    fn cache_key(&self) -> CredentialCacheKey {
+        match self {
+            Self::Inline(configured) => {
+                CredentialCacheKey::Inline(Sha256::digest(configured.expose()).into())
+            }
+            Self::Trusted(configured) => {
+                CredentialCacheKey::Trusted(Sha256::digest(configured.expose()).into())
+            }
+            Self::ApplicationCredentials(path) => {
+                CredentialCacheKey::ApplicationCredentials(path.clone())
+            }
+            Self::Adc => CredentialCacheKey::Adc,
+        }
+    }
+}
+
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
+enum CredentialCacheKey {
+    Inline([u8; 32]),
+    Trusted([u8; 32]),
+    ApplicationCredentials(String),
+    Adc,
+}
+
+async fn load_trusted_credentials(configured: &str) -> Result<Arc<dyn GcpTokenSource>, Error> {
+    if is_base64_encoded_json(configured) {
+        return Err(Error::InvalidConfiguration(
+            ErrorDetail::Base64EncodedJson {
+                field: "vertex_credentials",
+            },
+        ));
+    }
+    if names_a_credentials_file(configured) {
+        load_application_credentials(configured).await
+    } else {
+        load_credentials_json(configured)
+    }
+}
+
+fn auth_acquisition_error(error: impl std::error::Error + Send + Sync + 'static) -> Error {
+    Error::CredentialAcquisition(litellm_auth_types::ErrorDetail::failed(
+        "Google credentials",
+        error,
+    ))
+}
+
+#[cfg(test)]
+mod tests {
+    use litellm_secret_value::SecretValue;
+
+    use super::*;
+
+    #[rstest::rstest]
+    #[case::authorized_user("authorized_user", 2)]
+    #[case::external_account("external_account", 2)]
+    #[case::impersonated_service_account("impersonated_service_account", 4)]
+    #[tokio::test]
+    async fn explicit_application_credentials_refresh_expired_tokens(
+        #[case] kind: &str,
+        #[case] expected_requests: usize,
+        #[values(false, true)] inline: bool,
+    ) {
+        use axum::{Json, Router, routing::post};
+        use std::sync::atomic::{AtomicUsize, Ordering};
+
+        let _ = rustls::crypto::ring::default_provider().install_default();
+        let requests = Arc::new(AtomicUsize::new(0));
+        let recorded = requests.clone();
+        let impersonations = requests.clone();
+        let app = Router::new().route("/token", post(move || {
+            let recorded = recorded.clone();
+            async move {
+                Json(serde_json::json!({
+                    "access_token": format!("token-{}", recorded.fetch_add(1, Ordering::SeqCst)),
+                    "expires_in": 3600,
+                    "token_type": "Bearer",
+                    "issued_token_type": "urn:ietf:params:oauth:token-type:access_token",
+                }))
+            }
+        }));
+        let app = app.route("/v1/projects/-/serviceAccounts/test-principal:generateAccessToken", post(move || {
+            let recorded = impersonations.clone();
+            async move {
+                let expiration = (time::OffsetDateTime::now_utc() + time::Duration::hours(1)).format(&time::format_description::well_known::Rfc3339).unwrap();
+                Json(serde_json::json!({
+                    "accessToken": format!("impersonated-{}", recorded.fetch_add(1, Ordering::SeqCst)),
+                    "expireTime": expiration,
+                }))
+            }
+        }));
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let address = listener.local_addr().unwrap();
+        let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
+        let file = tempfile::NamedTempFile::new().unwrap();
+        let subject_token = tempfile::NamedTempFile::new().unwrap();
+        std::fs::write(subject_token.path(), "subject-token").unwrap();
+        let configured = if kind == "external_account" {
+            serde_json::json!({
+                "type": "external_account", "audience": "//iam.googleapis.com/projects/test/locations/global/workloadIdentityPools/test/providers/test",
+                "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", "token_url": format!("http://{address}/token"),
+                "credential_source": { "file": subject_token.path().to_str().unwrap() }, "quota_project_id": "quota-project"
+            })
+        } else {
+            let source = serde_json::json!({
+                "type": "authorized_user", "client_id": "user-client", "client_secret": "user-secret",
+                "refresh_token": "user-refresh", "quota_project_id": "quota-project", "token_uri": format!("http://{address}/token")
+            });
+            if kind == "impersonated_service_account" {
+                serde_json::json!({
+                    "type": kind, "source_credentials": source, "quota_project_id": "quota-project",
+                    "service_account_impersonation_url": format!("http://{address}/v1/projects/-/serviceAccounts/test-principal:generateAccessToken")
+                })
+            } else {
+                source
+            }
+        };
+        std::fs::write(file.path(), configured.to_string()).unwrap();
+        let source = if inline {
+            CredentialSource::Trusted(SecretValue::new(configured.to_string()))
+        } else {
+            CredentialSource::ApplicationCredentials(file.path().to_str().unwrap().into())
+        };
+        let provider = NativeProviderLoader.load(source).await.unwrap();
+
+        assert!(matches!(
+            provider.project_id().await,
+            Err(Error::CredentialAcquisition(ErrorDetail::Unresolved {
+                subject: "GCP resource project"
+            }))
+        ));
+        let original = provider.token().await.unwrap();
+        tokio::time::pause();
+        tokio::time::advance(std::time::Duration::from_secs(3601)).await;
+        tokio::time::resume();
+        let refreshed = provider.token().await.unwrap();
+        assert_ne!(original, refreshed);
+        assert_eq!(requests.load(Ordering::SeqCst), expected_requests);
+        server.abort();
+    }
+
+    #[rstest::rstest]
+    #[case::canonical_endpoint(r#"{"token_uri":"https://oauth2.googleapis.com/token"}"#, true)]
+    #[case::noncanonical_endpoint(r#"{"token_uri":"http://127.0.0.1/token"}"#, false)]
+    #[case::missing_endpoint("{}", false)]
+    fn request_credentials_require_canonical_token_endpoint(
+        #[case] credentials: &str,
+        #[case] accepted: bool,
+    ) {
+        assert_eq!(validate_request_credentials(credentials).is_ok(), accepted);
+    }
+
+    #[rstest::rstest]
+    #[case::missing_file("/nonexistent/vertex-key.json", "Google credentials")]
+    #[case::malformed_json(" {\"type\": \"service_account\",", "Google credentials")]
+    #[case::quoted_json("'{\"type\": \"service_account\"}'", "Google credentials")]
+    #[tokio::test]
+    async fn trusted_credentials_are_read_as_a_file_unless_they_are_json(
+        #[case] configured: &str,
+        #[case] expected: &str,
+    ) {
+        let Err(error) = NativeProviderLoader
+            .load(CredentialSource::Trusted(SecretValue::new(configured)))
+            .await
+        else {
+            panic!("{configured:?} should not load");
+        };
+        let report = litellm_error::Report::new(&error).to_string();
+        assert!(report.contains(expected), "{report}");
+    }
+
+    #[rstest::rstest]
+    #[case::padded("eyJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCJ9")]
+    #[case::quoted_prefixed_unpadded("\"base64:eyJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCJ9\"")]
+    #[case::url_safe_wrapped("eyJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCIsICJr\nZXkiOiAiPz8_In0=")]
+    #[tokio::test]
+    async fn base64_encoded_credentials_are_named_as_base64(#[case] configured: &str) {
+        assert!(
+            trusted_load_error(configured)
+                .await
+                .contains("is base64-encoded JSON, not the JSON itself")
+        );
+    }
+
+    async fn trusted_load_error(configured: &str) -> String {
+        match NativeProviderLoader
+            .load(CredentialSource::Trusted(SecretValue::new(configured)))
+            .await
+        {
+            Ok(_) => panic!("{configured:?} should not load"),
+            Err(error) => litellm_error::Report::new(&error).to_string(),
+        }
+    }
+
+    #[rstest::rstest]
+    #[tokio::test]
+    async fn a_relative_key_file_named_like_json_is_read_as_a_file() {
+        let key = tempfile::Builder::new()
+            .prefix("{vertex")
+            .suffix(".json")
+            .tempfile_in(".")
+            .unwrap();
+        std::fs::write(key.path(), r#"{"project_id":"file-project"}"#).unwrap();
+        let relative_name = key.path().file_name().unwrap().to_str().unwrap();
+        let absolute_path = std::fs::canonicalize(key.path()).unwrap();
+        assert_eq!(
+            trusted_load_error(relative_name).await,
+            trusted_load_error(absolute_path.to_str().unwrap()).await
+        );
+    }
+
+    #[rstest::rstest]
+    #[tokio::test]
+    async fn unknown_application_credential_types_are_rejected_without_service_account_fallback() {
+        let file = tempfile::NamedTempFile::new().unwrap();
+        std::fs::write(
+            file.path(),
+            r#"{"type":"unsupported","private_key":"credential-value"}"#,
+        )
+        .unwrap();
+        let result = GcpAuth::default()
+            .access_token(CredentialSource::ApplicationCredentials(
+                file.path().to_str().unwrap().into(),
+            ))
+            .await;
+        let error = result.unwrap_err();
+        assert!(matches!(error, Error::InvalidConfiguration(_)));
+        assert!(
+            !litellm_error::Report::new(&error)
+                .to_string()
+                .contains("credential-value")
+        );
+    }
+
+    #[rstest::rstest]
+    fn inline_and_trusted_credentials_never_share_a_cache_entry() {
+        assert_ne!(
+            CredentialSource::Inline(SecretValue::new("same-value")).cache_key(),
+            CredentialSource::Trusted(SecretValue::new("same-value")).cache_key()
+        );
+    }
+}

litellm-rust/crates/auth-gcp/src/lib.rs

@@ -1,16 +1,16 @@
 mod auth;
 mod config;
 pub mod constants;
+mod identity;
 #[cfg(feature = "google-sdk")]
 mod sdk;
 
-pub use auth::{
-    VertexAuth, VertexAuthFuture, VertexEnvironment, VertexProviderLoader, VertexTokenSource,
-};
+pub use auth::{VertexAuth, VertexEnvironment};
 pub use config::{
     CredentialSource, VertexConfig, get_vertex_ai_location, get_vertex_ai_project,
     get_vertex_ai_project_from_credentials,
 };
 pub use constants::secret_names;
+pub use identity::{GcpAuth, GcpAuthFuture, GcpProviderLoader, GcpTokenSource};
 #[cfg(feature = "google-sdk")]
 pub use sdk::GoogleCredentials;

litellm-rust/crates/auth-gcp/tests/vertex_auth.rs

@@ -1,120 +0,0 @@
-use std::{
-    collections::{BTreeMap, BTreeSet},
-    sync::{
-        Arc,
-        atomic::{AtomicUsize, Ordering},
-    },
-};
-
-use litellm_auth_gcp::{
-    CredentialSource, VertexAuth, VertexAuthFuture, VertexConfig, VertexProviderLoader,
-    VertexTokenSource, get_vertex_ai_location, secret_names,
-};
-use rstest::rstest;
-use serde_json::{Value, json};
-
-struct FakeProvider {
-    calls: Arc<AtomicUsize>,
-}
-
-impl VertexTokenSource for FakeProvider {
-    fn project_id(&self) -> VertexAuthFuture<'_, String> {
-        self.calls.fetch_add(1, Ordering::SeqCst);
-        Box::pin(async { Ok("adc-project".into()) })
-    }
-
-    fn token(&self) -> VertexAuthFuture<'_, String> {
-        self.calls.fetch_add(1, Ordering::SeqCst);
-        Box::pin(async { Ok("adc-token".into()) })
-    }
-}
-
-struct FakeLoader {
-    loads: Arc<AtomicUsize>,
-    provider: Arc<dyn VertexTokenSource>,
-}
-
-impl VertexProviderLoader for FakeLoader {
-    fn load(&self, _source: CredentialSource) -> VertexAuthFuture<'_, Arc<dyn VertexTokenSource>> {
-        let loads = self.loads.clone();
-        let provider = self.provider.clone();
-        Box::pin(async move {
-            loads.fetch_add(1, Ordering::SeqCst);
-            Ok(provider)
-        })
-    }
-}
-
-fn config(value: Value) -> VertexConfig {
-    VertexConfig::from_sourced_optional_params(value.as_object().unwrap(), &BTreeMap::new())
-        .unwrap()
-}
-
-fn auth(calls: Arc<AtomicUsize>, loads: Arc<AtomicUsize>) -> VertexAuth {
-    let provider: Arc<dyn VertexTokenSource> = Arc::new(FakeProvider { calls });
-    VertexAuth::new(Arc::new(FakeLoader { loads, provider }))
-}
-
-#[rstest]
-#[tokio::test]
-async fn secret_names_cover_environment_reads() {
-    let seen = Arc::new(std::sync::Mutex::new(BTreeSet::<String>::new()));
-    let recorded = seen.clone();
-    let env = |name: &str| {
-        recorded.lock().unwrap().insert(name.to_string());
-        None
-    };
-    let auth = auth(Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0)));
-    auth.validate_environment(Vec::new(), None, &VertexConfig::default(), &env)
-        .await
-        .unwrap();
-    get_vertex_ai_location(&VertexConfig::default(), &env);
-    assert!(
-        seen.lock()
-            .unwrap()
-            .iter()
-            .all(|name| secret_names().contains(&name.as_str()))
-    );
-}
-
-#[rstest]
-#[tokio::test]
-async fn explicit_token_and_header_do_not_acquire_adc() {
-    let loads = Arc::new(AtomicUsize::new(0));
-    let auth = auth(Arc::new(AtomicUsize::new(0)), loads.clone());
-    let configured = config(json!({"vertex_project":"project-1"}));
-    let explicit = auth
-        .validate_environment(Vec::new(), Some("access-token"), &configured, &|_| None)
-        .await
-        .unwrap();
-    assert_eq!(explicit.headers[0].1, "Bearer access-token");
-    let existing = auth
-        .validate_environment(
-            vec![("authorization".into(), "Bearer existing".into())],
-            None,
-            &configured,
-            &|_| None,
-        )
-        .await
-        .unwrap();
-    assert_eq!(existing.headers[0].1, "Bearer existing");
-    assert_eq!(loads.load(Ordering::SeqCst), 0);
-}
-
-#[rstest]
-#[tokio::test]
-async fn provider_is_reused_across_authentication_calls() {
-    let calls = Arc::new(AtomicUsize::new(0));
-    let loads = Arc::new(AtomicUsize::new(0));
-    let auth = auth(calls.clone(), loads.clone());
-    for _ in 0..2 {
-        let environment = auth
-            .validate_environment(Vec::new(), None, &VertexConfig::default(), &|_| None)
-            .await
-            .unwrap();
-        assert_eq!(environment.project_id, "adc-project");
-        assert_eq!(environment.headers[0].1, "Bearer adc-token");
-    }
-    assert_eq!(loads.load(Ordering::SeqCst), 1);
-    assert_eq!(calls.load(Ordering::SeqCst), 4);
-}

litellm-rust/crates/auth-types/src/error.rs

@@ -92,6 +92,11 @@ impl Error {
 
 #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
 pub enum ErrorDetail {
+    #[error("{operation} returned HTTP {status}")]
+    UnexpectedStatus {
+        operation: &'static str,
+        status: u16,
+    },
     #[error("{field} must be {expected}")]
     InvalidType {
         field: String,

litellm-rust/crates/auth-types/src/params/aws.rs

@@ -53,6 +53,9 @@ pub struct AwsParams {
 }
 
 impl AwsParams {
+    pub const ROLE_ARN_ENV: &str = "AWS_ROLE_ARN";
+    pub const WEB_IDENTITY_TOKEN_FILE_ENV: &str = "AWS_WEB_IDENTITY_TOKEN_FILE";
+
     pub const ACCESS_KEY_ID: ParamSpec = spec(
         "access key id",
         ParamKind::Credential,
@@ -149,6 +152,10 @@ impl AwsParams {
             AwsParams::SPECS
                 .iter()
                 .flat_map(|spec| spec.env.iter().copied())
+                .chain([
+                    AwsParams::ROLE_ARN_ENV,
+                    AwsParams::WEB_IDENTITY_TOKEN_FILE_ENV,
+                ])
                 .fold(Vec::new(), |mut names, name| {
                     if !names.contains(&name) {
                         names.push(name);

litellm-rust/crates/auth-types/tests/aws_params.rs

@@ -3,21 +3,13 @@ use rstest::rstest;
 use serde_json::{Map, Value};
 
 #[rstest]
-fn fields_name_every_param_once_and_in_declaration_order() {
+fn every_named_param_roundtrips_without_loss() {
     let filled: Map<String, Value> = AwsParams::fields()
         .map(|name| (name.to_string(), Value::from(format!("value-of-{name}"))))
         .collect();
     let typed = AwsParams::from_optional_params(&filled);
     let serialized = serde_json::to_value(&typed).unwrap();
-    assert_eq!(
-        serialized
-            .as_object()
-            .unwrap()
-            .keys()
-            .map(String::as_str)
-            .collect::<Vec<_>>(),
-        AwsParams::fields().collect::<Vec<_>>()
-    );
+    assert_eq!(filled.len(), AwsParams::fields().count());
     assert_eq!(serialized, Value::Object(filled.clone()));
     assert_eq!(
         serde_json::from_value::<AwsParams>(Value::Object(filled)).unwrap(),

litellm-rust/crates/cache-gcs/src/cache.rs

@@ -8,7 +8,7 @@ use litellm_cache::{
 use litellm_http::Client;
 use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_encode};
 
-use crate::{GcpTokenSource, TokenSource};
+use crate::{GcsTokenSource, TokenSource};
 
 pub const DEFAULT_ENDPOINT: &str = "https://storage.googleapis.com";
 
@@ -62,7 +62,7 @@ pub struct GcsCache<S: CacheCodec> {
 
 impl<S: CacheCodec> GcsCache<S> {
     pub fn new(config: GcsConfig, client: Client, codec: S) -> Self {
-        let token = Arc::new(GcpTokenSource::new(config.path_service_account.clone()));
+        let token = Arc::new(GcsTokenSource::new(config.path_service_account.clone()));
         Self::with_token_source(config, client, codec, token)
     }
 

litellm-rust/crates/cache-gcs/src/lib.rs

@@ -2,4 +2,4 @@ mod cache;
 mod token;
 
 pub use cache::{DEFAULT_ENDPOINT, GcsCache, GcsConfig, key_prefix};
-pub use token::{GcpTokenSource, StaticTokenSource, TokenSource};
+pub use token::{GcsTokenSource, StaticTokenSource, TokenSource};

litellm-rust/crates/cache-gcs/src/token.rs

@@ -1,7 +1,6 @@
 use std::{future::Future, pin::Pin};
 
-use litellm_auth_gcp::{VertexAuth, VertexConfig};
-use litellm_auth_types::{InputSource, Sourced};
+use litellm_auth_gcp::{CredentialSource, GcpAuth};
 use litellm_secret_value::SecretValue;
 
 use litellm_cache::Error;
@@ -10,28 +9,30 @@ pub trait TokenSource: Send + Sync + 'static {
     fn bearer_token(&self) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + '_>>;
 }
 
-pub struct GcpTokenSource {
-    auth: VertexAuth,
-    config: VertexConfig,
+pub struct GcsTokenSource {
+    auth: GcpAuth,
+    credentials: Option<String>,
 }
 
-impl GcpTokenSource {
+impl GcsTokenSource {
     pub fn new(path_service_account: Option<String>) -> Self {
-        let credentials = path_service_account
-            .map(|path| Sourced::new(SecretValue::new(path), InputSource::Deployment));
         Self {
-            auth: VertexAuth::default(),
-            config: VertexConfig::new(credentials, None, None),
+            auth: GcpAuth::default(),
+            credentials: path_service_account.filter(|path| !path.trim().is_empty()),
         }
     }
 }
 
-impl TokenSource for GcpTokenSource {
+impl TokenSource for GcsTokenSource {
     fn bearer_token(&self) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + '_>> {
         Box::pin(async move {
             self.auth
-                .access_token(&self.config, &|name| std::env::var(name).ok())
+                .access_token(match &self.credentials {
+                    Some(path) => CredentialSource::Trusted(SecretValue::new(path.clone())),
+                    None => CredentialSource::application_default(&|name| std::env::var(name).ok()),
+                })
                 .await
+                .map(|token| token.expose().to_owned())
                 .map_err(Error::unavailable)
         })
     }

litellm-rust/crates/cache-s3/src/auth.rs

@@ -36,9 +36,12 @@ impl ProvideCredentials for S3Credentials {
                 self.config.secret_access_key.clone(),
             ) {
                 return Ok(AwsCredentials::new(
-                    access_key_id,
-                    secret_access_key,
-                    self.config.session_token.clone(),
+                    access_key_id.expose(),
+                    secret_access_key.expose(),
+                    self.config
+                        .session_token
+                        .as_ref()
+                        .map(|value| value.expose().to_owned()),
                     None,
                     "litellm-s3-cache",
                 ));

litellm-rust/crates/cache-s3/tests/auth.rs

@@ -5,9 +5,9 @@ use rstest::rstest;
 
 fn explicit(session_token: Option<&str>) -> AwsAuthConfig {
     AwsAuthConfig {
-        access_key_id: Some("key".to_string()),
-        secret_access_key: Some("secret".to_string()),
-        session_token: session_token.map(str::to_string),
+        access_key_id: Some("key".into()),
+        secret_access_key: Some("secret".into()),
+        session_token: session_token.map(Into::into),
         region_name: Some("us-east-1".to_string()),
         ..Default::default()
     }

litellm-rust/crates/cache-s3/tests/support/mod.rs

@@ -23,8 +23,8 @@ pub fn config(endpoint: &str) -> S3CacheConfig {
             url: endpoint.to_string(),
         }),
         auth: AwsAuthConfig {
-            access_key_id: Some("key".to_string()),
-            secret_access_key: Some("secret".to_string()),
+            access_key_id: Some("key".into()),
+            secret_access_key: Some("secret".into()),
             region_name: Some("us-east-1".to_string()),
             ..Default::default()
         },

litellm-rust/crates/inference-ocr/tests/resources.rs

@@ -7,7 +7,7 @@ use std::sync::{
 
 use litellm_auth::AuthServices;
 use litellm_auth_gcp::{
-    CredentialSource, VertexAuth, VertexAuthFuture, VertexProviderLoader, VertexTokenSource,
+    CredentialSource, GcpAuthFuture, GcpProviderLoader, GcpTokenSource, VertexAuth,
 };
 use litellm_http::{HttpSettings, Resolution};
 use litellm_inference::resources::CoreResources;
@@ -20,28 +20,28 @@ use support::{ReceivedRequest, json_response, upstream};
 
 struct TokenSource(String);
 
-impl VertexTokenSource for TokenSource {
-    fn project_id(&self) -> VertexAuthFuture<'_, String> {
+impl GcpTokenSource for TokenSource {
+    fn project_id(&self) -> GcpAuthFuture<'_, String> {
         Box::pin(async { Ok(self.0.clone()) })
     }
 
-    fn token(&self) -> VertexAuthFuture<'_, String> {
-        Box::pin(async { Ok(self.0.clone()) })
+    fn token(&self) -> GcpAuthFuture<'_, litellm_secret_value::SecretValue> {
+        Box::pin(async { Ok(litellm_secret_value::SecretValue::new(self.0.clone())) })
     }
 }
 
 #[derive(Default)]
 struct Loader(AtomicUsize);
 
-impl VertexProviderLoader for Loader {
-    fn load(&self, source: CredentialSource) -> VertexAuthFuture<'_, Arc<dyn VertexTokenSource>> {
+impl GcpProviderLoader for Loader {
+    fn load(&self, source: CredentialSource) -> GcpAuthFuture<'_, Arc<dyn GcpTokenSource>> {
         Box::pin(async move {
             self.0.fetch_add(1, Ordering::SeqCst);
             let identity = match source {
                 CredentialSource::Trusted(secret) => secret.expose().to_string(),
                 other => panic!("unexpected credential source: {other:?}"),
             };
-            Ok(Arc::new(TokenSource(identity)) as Arc<dyn VertexTokenSource>)
+            Ok(Arc::new(TokenSource(identity)) as Arc<dyn GcpTokenSource>)
         })
     }
 }

litellm-rust/crates/secret-value/src/secret_value.rs

@@ -16,6 +16,18 @@ impl SecretValue {
     }
 }
 
+impl From<String> for SecretValue {
+    fn from(value: String) -> Self {
+        Self::new(value)
+    }
+}
+
+impl From<&str> for SecretValue {
+    fn from(value: &str) -> Self {
+        Self::new(value)
+    }
+}
+
 impl PartialEq for SecretValue {
     fn eq(&self, other: &Self) -> bool {
         subtle::ConstantTimeEq::ct_eq(self.0.as_bytes(), other.0.as_bytes()).into()

litellm-rust/crates/secrets-aws/src/auth.rs

@@ -39,30 +39,15 @@ impl Credentials {
         Self {
             auth,
             config: AwsAuthConfig {
-                access_key_id: context
-                    .access_key_id
-                    .as_ref()
-                    .map(|value| value.expose().to_owned()),
-                secret_access_key: context
-                    .secret_access_key
-                    .as_ref()
-                    .map(|value| value.expose().to_owned()),
-                session_token: context
-                    .session_token
-                    .as_ref()
-                    .map(|value| value.expose().to_owned()),
+                access_key_id: context.access_key_id.as_ref().cloned(),
+                secret_access_key: context.secret_access_key.as_ref().cloned(),
+                session_token: context.session_token.as_ref().cloned(),
                 region_name: region(settings, environment.as_ref()).ok(),
                 role_name: settings.aws_role_name.clone(),
                 session_name: settings.aws_session_name.clone(),
-                external_id: settings
-                    .aws_external_id
-                    .as_ref()
-                    .map(|v| v.expose().to_owned()),
+                external_id: settings.aws_external_id.as_ref().cloned(),
                 profile_name: settings.aws_profile_name.clone(),
-                web_identity_token: settings
-                    .aws_web_identity_token
-                    .as_ref()
-                    .map(|v| v.expose().to_owned()),
+                web_identity_token: settings.aws_web_identity_token.as_ref().cloned(),
                 sts_endpoint: settings.aws_sts_endpoint.clone(),
             },
             environment,

litellm/__init__.py

@@ -1796,6 +1796,9 @@ if TYPE_CHECKING:
     from .llms.bespoke.decisions.transformation import (
         BespokeDecisionsConfig as BespokeDecisionsConfig,
     )
+    from .llms.empiriolabs.decisions.transformation import (
+        EmpirioLabsDecisionsConfig as EmpirioLabsDecisionsConfig,
+    )
     from .llms.eu_jev.decisions.transformation import (
         EuJevDecisionsConfig as EuJevDecisionsConfig,
     )

litellm/_lazy_imports_registry.py

@@ -176,6 +176,7 @@ LLM_CONFIG_NAMES: Final = (
     "OllamaDecisionsConfig",
     "LayaDecisionsConfig",
     "BespokeDecisionsConfig",
+    "EmpirioLabsDecisionsConfig",
     "EuJevDecisionsConfig",
     "OpenAIDecisionsConfig",
     "AzureDecisionsConfig",
@@ -759,6 +760,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
     "OllamaDecisionsConfig": (".llms.ollama.decisions.transformation", "OllamaDecisionsConfig"),
     "LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"),
     "BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"),
+    "EmpirioLabsDecisionsConfig": (".llms.empiriolabs.decisions.transformation", "EmpirioLabsDecisionsConfig"),
     "EuJevDecisionsConfig": (".llms.eu_jev.decisions.transformation", "EuJevDecisionsConfig"),
     "OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"),
     "AzureDecisionsConfig": (".llms.azure.decisions.transformation", "AzureDecisionsConfig"),

litellm/accounting/__init__.py

@@ -1,62 +0,0 @@
-"""Public accounting contracts. Importing these values does not enable accounting."""
-
-from litellm.accounting.measurements import measurements_for
-from litellm.accounting.protocols import BudgetEnforcer, CostAggregator, CostCompiler, UsageMeter
-from litellm.accounting.types import (
-    AdmissionDecision,
-    AggregateQuery,
-    AggregateRow,
-    CostEvent,
-    DeliveryState,
-    DeliveryTarget,
-    Dimension,
-    DimensionFilter,
-    DimensionName,
-    EventKind,
-    Granularity,
-    InboundUsageEvent,
-    Measurement,
-    Metric,
-    MetricName,
-    Multiplier,
-    NamedCost,
-    OutboundUsageEvent,
-    Outcome,
-    PricedQuantity,
-    PricingSnapshot,
-    Rate,
-    Unpriced,
-    validate_aggregate_rows,
-)
-
-__all__ = [
-    "AdmissionDecision",
-    "AggregateQuery",
-    "AggregateRow",
-    "BudgetEnforcer",
-    "CostAggregator",
-    "CostCompiler",
-    "CostEvent",
-    "DeliveryState",
-    "DeliveryTarget",
-    "Dimension",
-    "DimensionFilter",
-    "DimensionName",
-    "EventKind",
-    "Granularity",
-    "InboundUsageEvent",
-    "Measurement",
-    "Metric",
-    "MetricName",
-    "Multiplier",
-    "NamedCost",
-    "OutboundUsageEvent",
-    "Outcome",
-    "PricedQuantity",
-    "PricingSnapshot",
-    "Rate",
-    "Unpriced",
-    "UsageMeter",
-    "measurements_for",
-    "validate_aggregate_rows",
-]

litellm/accounting/application.py

@@ -0,0 +1,71 @@
+from typing import TYPE_CHECKING, Final
+
+from litellm.accounting.delivery_projection import entity_transactions, project_deliveries
+from litellm.accounting.projection import Contribution
+from litellm.proxy.db.budget_window_spend_writer import commit_window_spend_updates
+from litellm.proxy.db.daily_spend_bulk_upsert import DAILY_SPEND_TABLES, build_bulk_upsert, merge_by_conflict_key
+from litellm.proxy.db.db_span import db_span
+from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import WindowSpendTransaction
+from litellm.proxy.db.model_usage_rollup import queue_model_usage_upsert
+from litellm.repositories.accounting_delivery_repository import DeliveryProjection, LedgerTransaction
+
+if TYPE_CHECKING:
+    from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter
+    from litellm.proxy.utils import PrismaClient, ProxyLogging
+
+
+async def apply_deliveries(
+    transaction: LedgerTransaction,
+    deliveries: tuple[DeliveryProjection, ...],
+    *,
+    prisma_client: "PrismaClient",
+    writer: "DBSpendUpdateWriter",
+    proxy_logging_obj: "ProxyLogging",
+) -> tuple[Contribution, ...]:
+    from litellm.proxy.guardrails.usage_tracking import UsageUnitIncrement, UsageUnitKey, upsert_usage_unit_row
+
+    contributions: Final = project_deliveries(deliveries)
+    await writer.apply_accounting_entities(
+        prisma_client, proxy_logging_obj, transaction, entity_transactions(contributions)
+    )
+    for table in sorted(DAILY_SPEND_TABLES.values(), key=lambda item: item.name):
+        rows = tuple(
+            {**dict(item.key), **dict(item.values), **dict(item.increments)}
+            for item in contributions
+            if item.table == table.name
+        )
+        merged = merge_by_conflict_key(table, rows)
+        for offset in range(0, len(merged), 100):
+            sql, params = build_bulk_upsert(table, merged[offset : offset + 100])
+            async with db_span("upsert_daily_spend", table.name):
+                await transaction.execute_raw(sql, *params)
+    async with transaction.batch_() as batcher:
+        for item in contributions:
+            if item.table == "LiteLLM_DailyModelUsage":
+                queue_model_usage_upsert(batcher, dict(item.key), dict(item.increments))
+    for item in contributions:
+        if item.table == "LiteLLM_DailyGuardrailUsageUnits":
+            key = dict(item.key)
+            counters = dict(item.increments)
+            await upsert_usage_unit_row(
+                prisma_client,
+                UsageUnitKey(key["guardrail_id"], key["date"], key["team_id"], key["api_key"], key["usage_unit"]),
+                UsageUnitIncrement(int(counters["units"]), float(counters["cost"]), int(counters["untracked_units"])),
+                outer_transaction=transaction,
+            )
+    windows: Final = tuple(_window(item) for item in contributions if item.table == "LiteLLM_BudgetWindowSpend")
+    await commit_window_spend_updates(prisma_client, windows, outer_transaction=transaction)
+    return contributions
+
+
+def _window(item: Contribution) -> WindowSpendTransaction:
+    key: Final = dict(item.key)
+    values: Final = dict(item.values)
+    return WindowSpendTransaction(
+        entity_type=key["entity_type"],
+        entity_id=key["entity_id"],
+        window_duration=key["window_duration"],
+        window_start=values["window_start"] or "",
+        spend=float(dict(item.increments)["spend"]),
+        started_at=values["started_at"],
+    )

litellm/accounting/delivery_projection.py

@@ -0,0 +1,78 @@
+from collections.abc import Iterable
+from itertools import chain, groupby
+from typing import TYPE_CHECKING, Final
+from urllib.parse import quote
+
+from litellm.accounting.projection import Contribution, project
+from litellm.repositories.accounting_delivery_repository import DeliveryProjection
+
+
+def _destination(contribution: Contribution) -> tuple[str, tuple[tuple[str, str], ...], str]:
+    return (
+        contribution.table,
+        contribution.key,
+        (dict(contribution.values).get("window_start") or "")
+        if contribution.table == "LiteLLM_BudgetWindowSpend"
+        else "",
+    )
+
+
+def _sum(contributions: tuple[Contribution, ...]) -> Contribution:
+    first: Final = contributions[0]
+    increments: Final = tuple(
+        (column, sum(dict(item.increments).get(column, 0) for item in contributions))
+        for column in dict.fromkeys(
+            column for column, _ in chain.from_iterable(item.increments for item in contributions)
+        )
+    )
+    starts: Final = tuple(
+        value
+        for column, value in chain.from_iterable(item.values for item in contributions)
+        if column == "started_at" and value
+    )
+    values: Final = tuple(
+        (column, min(starts) if column == "started_at" and starts else value) for column, value in first.values
+    )
+    return Contribution(first.table, first.key, increments, values)
+
+
+def project_deliveries(deliveries: Iterable[DeliveryProjection]) -> tuple[Contribution, ...]:
+    ordered: Final = sorted(chain.from_iterable(project(delivery.row) for delivery in deliveries), key=_destination)
+    return tuple(_sum(tuple(group)) for _, group in groupby(ordered, key=_destination))
+
+
+def entity_transactions(contributions: tuple[Contribution, ...]) -> "DBSpendUpdateTransactions":
+    return {
+        "user_list_transactions": dict(_entity_totals(contributions, "LiteLLM_UserTable")),
+        "end_user_list_transactions": dict(_entity_totals(contributions, "LiteLLM_EndUserTable")),
+        "key_list_transactions": dict(_entity_totals(contributions, "LiteLLM_VerificationToken")),
+        "team_list_transactions": dict(_entity_totals(contributions, "LiteLLM_TeamTable")),
+        "team_member_list_transactions": dict(_entity_totals(contributions, "LiteLLM_TeamMembership")),
+        "org_list_transactions": dict(_entity_totals(contributions, "LiteLLM_OrganizationTable")),
+        "org_member_list_transactions": dict(_entity_totals(contributions, "LiteLLM_OrganizationMembership")),
+        "project_list_transactions": dict(_entity_totals(contributions, "LiteLLM_ProjectTable")),
+        "tag_list_transactions": dict(_entity_totals(contributions, "LiteLLM_TagTable")),
+        "agent_list_transactions": dict(_entity_totals(contributions, "LiteLLM_AgentsTable")),
+        "model_access_group_list_transactions": dict(
+            _entity_totals(contributions, "LiteLLM_ModelAccessGroupBudgetTable")
+        ),
+    }
+
+
+def _entity_totals(contributions: tuple[Contribution, ...], table: str) -> tuple[tuple[str, float], ...]:
+    return tuple(
+        (_entity_key(item), float(dict(item.increments)["spend"])) for item in contributions if item.table == table
+    )
+
+
+def _entity_key(item: Contribution) -> str:
+    fields: Final = dict(item.key)
+    if item.table == "LiteLLM_TeamMembership":
+        return f"team_id::{fields['team_id']}::user_id::{fields['user_id']}"
+    if item.table == "LiteLLM_OrganizationMembership":
+        return f"org_id::{quote(fields['organization_id'], safe='')}::user_id::{quote(fields['user_id'], safe='')}"
+    return item.key[0][1]
+
+
+if TYPE_CHECKING:
+    from litellm.proxy._types import DBSpendUpdateTransactions

litellm/accounting/measurements.py

@@ -1,13 +0,0 @@
-from typing import Final
-
-from litellm.accounting.types import CostEvent, InboundUsageEvent, Measurement, OutboundUsageEvent
-
-
-def measurements_for(event: InboundUsageEvent | OutboundUsageEvent | CostEvent) -> tuple[Measurement, ...]:
-    """Return the complete contribution set for one event, ready for one apply call."""
-    metrics: Final = event.charges if isinstance(event, CostEvent) else event.metrics
-    occurred_at: Final = event.occurred_at if isinstance(event, CostEvent) else event.started_at
-    return tuple(
-        Measurement(source_event_id=event.event_id, occurred_at=occurred_at, dimensions=event.dimensions, metric=metric)
-        for metric in metrics or ()
-    )

litellm/accounting/metrics.py

@@ -0,0 +1,30 @@
+from typing import Final
+
+from prometheus_client import Gauge, Histogram
+
+from litellm.repositories.accounting_delivery_repository import DeliveryMetrics
+
+_PENDING: Final = Gauge("litellm_accounting_pending", "Pending SpendLogs deliveries", multiprocess_mode="livemax")
+_PARKED: Final = Gauge("litellm_accounting_parked", "Parked SpendLogs deliveries", multiprocess_mode="livemax")
+_OLDEST: Final = Gauge(
+    "litellm_accounting_oldest_pending_seconds",
+    "Age of the oldest pending SpendLogs delivery",
+    multiprocess_mode="livemax",
+)
+_BATCH_TIME: Final = Histogram("litellm_accounting_batch_seconds", "SpendLogs delivery worker batch time")
+_BATCH_COUNT: Final = Gauge(
+    "litellm_accounting_batch_deliveries", "Retained batch deliveries", multiprocess_mode="livemax"
+)
+_TABLE_BYTES: Final = Gauge(
+    "litellm_accounting_delivery_table_bytes", "Delivery table size in bytes", multiprocess_mode="livemax"
+)
+
+
+def record_deliveries(metrics: DeliveryMetrics, seconds: float) -> None:
+    _PENDING.set(metrics.pending_count)
+    _PARKED.set(metrics.parked_count)
+    _OLDEST.set(max(0.0, metrics.oldest_pending_seconds))
+    _BATCH_TIME.observe(seconds)
+
+    _BATCH_COUNT.set(metrics.batch_count)
+    _TABLE_BYTES.set(metrics.table_bytes)

litellm/accounting/projection.py

@@ -0,0 +1,326 @@
+from collections.abc import Iterator, Mapping
+from dataclasses import dataclass, replace
+from datetime import datetime, timezone
+from itertools import chain
+from types import MappingProxyType
+from typing import Final
+
+from pydantic import TypeAdapter, ValidationError
+
+from litellm.constants import (
+    BILLED_RETRY_ATTEMPT_METADATA_KEY,
+    INTERNAL_CALL_ORIGIN_METADATA_KEY,
+    MODEL_INSIGHTS_DEFAULT_TASK,
+    MODEL_INSIGHTS_TASK_TAG_PREFIX,
+)
+from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import GuardrailCostByUnitEntry
+from litellm.proxy.db.daily_spend_bulk_upsert import DAILY_SPEND_TABLES, daily_spend_entity_ids
+from litellm.proxy.db.model_insights_tasks import load_model_insight_tasks
+from litellm.proxy.db.model_usage_rollup import date_from_start_time
+from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING
+from litellm.proxy.spend_tracking.compression_savings import extract_compression_saved_tokens
+from litellm.proxy.spend_tracking.savings import extract_cache_creation_tokens, extract_cache_read_tokens
+from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
+
+_FIELDS: Final = TypeAdapter(dict[str, object])
+_STRINGS: Final = TypeAdapter(tuple[str, ...])
+_VALUES: Final = TypeAdapter(tuple[object, ...])
+_ROWS: Final = TypeAdapter(tuple[dict[str, object], ...])
+_TASK_TYPES: Final = frozenset(load_model_insight_tasks())
+_MAX_INDEXED_TEXT_BYTES: Final = 2000
+_MAX_COUNTER: Final = 2**40
+
+
+@dataclass(frozen=True, slots=True)
+class Contribution:
+    table: str
+    key: tuple[tuple[str, str], ...]
+    increments: tuple[tuple[str, int | float], ...]
+    values: tuple[tuple[str, str | None], ...] = ()
+
+
+def _fields(value: object) -> Mapping[str, object]:
+    if isinstance(value, str):
+        return _FIELDS.validate_json(value)
+    return _FIELDS.validate_python(value) if isinstance(value, Mapping) else MappingProxyType({})
+
+
+def _text(value: object) -> str:
+    return "" if value is None else str(value)
+
+
+def _number(value: object) -> int | float:
+    return value if isinstance(value, (int, float)) else 0
+
+
+def _tags(row: Mapping[str, object]) -> tuple[str, ...]:
+    value: Final = row.get("request_tags")
+    if value is None:
+        return ()
+    parsed: Final = _VALUES.validate_json(value) if isinstance(value, str) else _VALUES.validate_python(value)
+    return tuple(tag for tag in parsed if isinstance(tag, str))
+
+
+def _date(row: Mapping[str, object]) -> str | None:
+    start: Final = row.get("startTime")
+    return date_from_start_time(start) if isinstance(start, (datetime, str)) else None
+
+
+def _spend(table: str, key: tuple[tuple[str, str], ...], spend: float) -> Contribution:
+    return Contribution(table, key, (("spend", spend),))
+
+
+def _entity_contributions(row: Mapping[str, object], metadata: Mapping[str, object]) -> tuple[Contribution, ...]:
+    spend: Final = _number(row.get("spend"))
+    entities: Final = _fields(metadata.get("accounting_entities"))
+    identities: Final = (
+        ("LiteLLM_VerificationToken", "token", entities.get("api_key")),
+        ("LiteLLM_UserTable", "user_id", entities.get("user")),
+        ("LiteLLM_UserTable", "user_id", entities.get("proxy_budget_name")),
+        ("LiteLLM_TeamTable", "team_id", entities.get("team_id")),
+        ("LiteLLM_OrganizationTable", "organization_id", entities.get("organization_id")),
+        ("LiteLLM_EndUserTable", "user_id", entities.get("end_user")),
+        ("LiteLLM_AgentsTable", "agent_id", entities.get("agent_id")),
+        ("LiteLLM_ProjectTable", "project_id", entities.get("project_id")),
+    )
+    groups: Final = _STRINGS.validate_python(metadata.get("accounting_model_access_groups", ()))
+    return (
+        *(
+            _spend(table, ((column, identity),), spend)
+            for table, column, identity in identities
+            if isinstance(identity, str)
+        ),
+        *(_spend("LiteLLM_TagTable", (("tag_name", tag),), spend) for tag in _tags(row) if tag),
+        *(_spend("LiteLLM_ModelAccessGroupBudgetTable", (("access_group_name", group),), spend) for group in groups),
+        *_membership_contributions(entities, spend),
+    )
+
+
+def _membership_contributions(row: Mapping[str, object], spend: float) -> tuple[Contribution, ...]:
+    user: Final = row.get("user")
+    if not isinstance(user, str):
+        return ()
+    owners: Final = (
+        ("LiteLLM_TeamMembership", "team_id", row.get("team_id")),
+        ("LiteLLM_OrganizationMembership", "organization_id", row.get("organization_id")),
+    )
+    return tuple(
+        _spend(table, ((column, identity), ("user_id", user)), spend)
+        for table, column, identity in owners
+        if isinstance(identity, str)
+    )
+
+
+def _daily_counters(row: Mapping[str, object], metadata: Mapping[str, object]) -> tuple[tuple[str, int | float], ...]:
+    usage: Final = _fields(metadata.get("usage_object"))
+    savings: Final = _fields(metadata.get("accounting_savings"))
+    internal: Final = bool(metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY))
+    caller: Final = not internal and metadata.get(BILLED_RETRY_ATTEMPT_METADATA_KEY) is not True
+    success: Final = metadata.get("status") != "failure"
+    duration: Final = row.get("request_duration_ms")
+    timed: Final = not internal and success and isinstance(duration, int) and duration >= 0
+    return (
+        ("spend", _number(row.get("spend"))),
+        ("prompt_tokens", _number(row.get("prompt_tokens"))),
+        ("completion_tokens", _number(row.get("completion_tokens"))),
+        ("api_requests", int(caller)),
+        ("successful_requests", int(caller and success)),
+        ("failed_requests", int(caller and not success)),
+        ("cache_read_input_tokens", extract_cache_read_tokens(usage)),
+        ("cache_creation_input_tokens", extract_cache_creation_tokens(usage)),
+        ("compression_saved_tokens", extract_compression_saved_tokens(metadata)),
+        ("compression_savings_spend", _number(savings.get("compression"))),
+        ("prompt_caching_savings_spend", _number(savings.get("prompt_caching"))),
+        ("gateway_injected_caching_savings_spend", _number(savings.get("gateway_injected_caching"))),
+        ("autorouter_savings_spend", 0.0 if internal else _number(savings.get("autorouter"))),
+        ("total_response_time_ms", _number(duration) if timed else 0),
+        ("timed_requests", int(timed)),
+    )
+
+
+def _daily_contributions(row: Mapping[str, object], metadata: Mapping[str, object]) -> tuple[Contribution, ...]:
+    date: Final = _date(row)
+    if date is None or "api_key" not in row:
+        return ()
+    if "mcp_namespaced_tool_name" not in row and (
+        "model" not in row or "custom_llm_provider" not in row or "model_group" not in row
+    ):
+        return ()
+    counters: Final = _daily_counters(row, metadata)
+    endpoint: Final = ROUTE_ENDPOINT_MAPPING.get(_text(row.get("call_type")))
+    shared_key: Final = (
+        ("date", date),
+        ("api_key", _text(row.get("api_key"))),
+        ("model", _text(row.get("model"))),
+        ("custom_llm_provider", _text(row.get("custom_llm_provider"))),
+        ("mcp_namespaced_tool_name", _text(row.get("mcp_namespaced_tool_name"))),
+        ("endpoint", _text(endpoint)),
+    )
+    # The legacy daily organization writer uses the charged org argument, not
+    # SpendLogs' normalized empty-string identity for a request with no org.
+    daily_row: Final = {**row, "organization_id": _fields(metadata.get("accounting_entities")).get("organization_id")}
+    return tuple(_daily_rows(daily_row, shared_key, counters))
+
+
+def _daily_rows(
+    row: Mapping[str, object],
+    shared_key: tuple[tuple[str, str], ...],
+    counters: tuple[tuple[str, int | float], ...],
+) -> Iterator[Contribution]:
+    for entity, table in DAILY_SPEND_TABLES.items():
+        for identity in daily_spend_entity_ids(row, entity):
+            yield Contribution(
+                table.name,
+                ((table.entity_id_column, _text(identity)), *shared_key),
+                counters,
+                (
+                    ("model_group", None if row.get("model_group") is None else _text(row.get("model_group"))),
+                    *((("request_id", _text(row.get("request_id"))),) if table.carries_request_id else ()),
+                ),
+            )
+
+
+def _window_contributions(row: Mapping[str, object], metadata: Mapping[str, object]) -> tuple[Contribution, ...]:
+    windows: Final = _ROWS.validate_python(metadata.get("accounting_windows", ()))
+    return tuple(
+        Contribution(
+            "LiteLLM_BudgetWindowSpend",
+            tuple((column, _text(window[column])) for column in ("entity_type", "entity_id", "window_duration")),
+            (("spend", _number(window.get("spend", row.get("spend")))),),
+            (("window_start", _text(window["window_start"])), ("started_at", _text(row.get("startTime")))),
+        )
+        for window in windows
+    )
+
+
+def project(row: Mapping[str, object]) -> tuple[Contribution, ...]:
+    metadata: Final = _fields(row.get("metadata"))
+    contributions: Final = (
+        *_entity_contributions(row, metadata),
+        *_daily_contributions(row, metadata),
+        *_window_contributions(row, metadata),
+        *_model_contributions(row, metadata),
+        *_guardrail_contributions(row, metadata),
+    )
+    return tuple(_capped(contribution) for contribution in contributions if _indexable(contribution))
+
+
+def _indexable(contribution: Contribution) -> bool:
+    texts: Final = (value for _, value in (*contribution.key, *contribution.values) if value is not None)
+    return all(len(text.encode("utf-8", "surrogatepass")) <= _MAX_INDEXED_TEXT_BYTES for text in texts)
+
+
+def _capped(contribution: Contribution) -> Contribution:
+    """Counters are summed into bigint columns in the charge's transaction, so one capped far below that range
+    can never overflow a day's total."""
+    return replace(
+        contribution,
+        increments=tuple(
+            (name, max(-_MAX_COUNTER, min(_MAX_COUNTER, value)) if isinstance(value, int) else value)
+            for name, value in contribution.increments
+        ),
+    )
+
+
+def _model_contributions(row: Mapping[str, object], metadata: Mapping[str, object]) -> tuple[Contribution, ...]:
+    date: Final = _date(row)
+    origin: Final = metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)
+    if date is None or (origin and origin != BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN):
+        return ()
+    model: Final = _text(row.get("model")) or "unknown"
+    caller: Final = not origin
+    success: Final = row.get("status") == "success"
+    return (
+        Contribution(
+            "LiteLLM_DailyModelUsage",
+            (
+                ("date", date),
+                ("model_group", _text(row.get("model_group")) or model),
+                ("model", model),
+                ("custom_llm_provider", _text(row.get("custom_llm_provider")) or "unknown"),
+                ("task_type", _model_task(row)),
+            ),
+            (
+                ("spend", _number(row.get("spend"))),
+                ("prompt_tokens", _number(row.get("prompt_tokens"))),
+                ("completion_tokens", _number(row.get("completion_tokens"))),
+                ("request_count", int(caller)),
+                ("successful_requests", int(caller and success)),
+                ("failed_requests", int(caller and not success)),
+            ),
+        ),
+    )
+
+
+def _utc_date(row: Mapping[str, object]) -> str | None:
+    start: Final = row.get("startTime")
+    try:
+        parsed: Final = datetime.fromisoformat(start.replace("Z", "+00:00")) if isinstance(start, str) else start
+    except ValueError:
+        return None
+    if not isinstance(parsed, datetime):
+        return None
+    return (
+        (parsed.replace(tzinfo=timezone.utc) if parsed.tzinfo is None else parsed)
+        .astimezone(timezone.utc)
+        .date()
+        .isoformat()
+    )
+
+
+def _guardrail_contributions(row: Mapping[str, object], metadata: Mapping[str, object]) -> tuple[Contribution, ...]:
+    date: Final = _utc_date(row)
+    info: Final = metadata.get("guardrail_information") or metadata.get("standard_logging_guardrail_information")
+    if not row.get("request_id") or date is None or not isinstance(info, list):
+        return ()
+    return tuple(chain.from_iterable(_guardrail_entry(row, date, entry) for entry in _ROWS.validate_python(info)))
+
+
+def _guardrail_entry(row: Mapping[str, object], date: str, entry: Mapping[str, object]) -> tuple[Contribution, ...]:
+    identity: Final = _text(entry.get("guardrail_id") or entry.get("guardrail_name"))
+    usage: Final = entry.get("guardrail_usage")
+    if not identity or not isinstance(usage, Mapping):
+        return ()
+    costs: Final = _guardrail_unit_costs(entry)
+    return tuple(
+        _guardrail_unit(row, (identity, date, unit), units, None if costs is None else costs.get(unit))
+        for unit, units in _fields(usage).items()
+        if isinstance(units, int) and not isinstance(units, bool) and units > 0
+    )
+
+
+def _guardrail_unit(
+    row: Mapping[str, object], identity: tuple[str, str, str], units: int, cost: float | None
+) -> Contribution:
+    guardrail_id, date, unit = identity
+    return Contribution(
+        "LiteLLM_DailyGuardrailUsageUnits",
+        (
+            ("guardrail_id", guardrail_id),
+            ("date", date),
+            ("team_id", _text(row.get("team_id"))),
+            ("api_key", _text(row.get("api_key"))),
+            ("usage_unit", unit),
+        ),
+        (("units", units), ("cost", cost or 0.0), ("untracked_units", units if cost is None else 0)),
+    )
+
+
+def _model_task(row: Mapping[str, object]) -> str:
+    return next(
+        (
+            task
+            for tag in _tags(row)
+            if tag.startswith(MODEL_INSIGHTS_TASK_TAG_PREFIX)
+            and (task := tag.removeprefix(MODEL_INSIGHTS_TASK_TAG_PREFIX)) in _TASK_TYPES
+        ),
+        MODEL_INSIGHTS_DEFAULT_TASK,
+    )
+
+
+def _guardrail_unit_costs(entry: Mapping[str, object]) -> Mapping[str, float | None] | None:
+    try:
+        cost: Final = GuardrailCostByUnitEntry.model_validate(entry)
+    except ValidationError:
+        return None
+    return None if cost.guardrail_cost_in_spend is False else cost.guardrail_cost_by_unit

litellm/accounting/protocols.py

@@ -1,85 +0,0 @@
-"""Contracts only. Implementations own persistence, pricing, projections and live budgets.
-
-The event row's ``payload`` is a private envelope versioned by the row's ``schema_version``.
-Usage envelopes retain the public event and request-scoped pricing context: quantities with units,
-model/provider, requested and billed tier/region when known, and per-request/deployment overrides.
-These inputs, including the effective cost-map rates, must survive request teardown.
-Cost envelopes retain the public CostEvent, including its required finalized pricing snapshot.
-T5/T6 own envelope serialization; this public API does not introduce a second persisted schema.
-"""
-
-from typing import Protocol
-
-from litellm.accounting.types import (
-    AdmissionDecision,
-    AggregateQuery,
-    AggregateRow,
-    CostEvent,
-    Dimension,
-    InboundUsageEvent,
-    Measurement,
-    Metric,
-    OutboundUsageEvent,
-    Unpriced,
-)
-
-
-class UsageMeter(Protocol):
-    async def record(self, event: InboundUsageEvent | OutboundUsageEvent) -> None:
-        """Acknowledge commit, no-op identical IDs and reject conflicting content.
-
-        Persist the event and its request-scoped pricing context before acknowledging.
-        HTTP acknowledgement is a separate policy. Unknown usage remains None.
-        """
-        ...
-
-
-class CostCompiler(Protocol):
-    async def compile(self, event: OutboundUsageEvent) -> CostEvent | Unpriced:
-        """Reuse the committed charge or finalize one charge and its delivery intent.
-
-        Resolve captured request-scoped context and rates by event identity.
-        Persist the resolved inputs in the required snapshot. Replay
-        reuses the committed result and never consumes free allowance again.
-        Missing pricing is Unpriced, never an inferred zero.
-        """
-        ...
-
-
-class CostAggregator(Protocol):
-    async def apply(self, measurements: tuple[Measurement, ...]) -> None:
-        """Apply one source event's complete contribution set per target/version.
-
-        Each target commits all contributions and its receipt atomically. A receipt
-        for the first metric must not suppress the rest. Usage need not wait for cost.
-        """
-        ...
-
-    async def query(self, query: AggregateQuery) -> tuple[AggregateRow, ...]:
-        """Return disjoint [start, end) rows per group using existing daily projections.
-
-        Filters combine with AND, values within a filter with IN. DAY emits daily
-        buckets; None emits whole-interval totals. Reject unsupported metric/scope
-        combinations. Different groups may overlap, including non-additive tags.
-        """
-        ...
-
-
-class BudgetEnforcer(Protocol):
-    async def reserve(
-        self, reservation_id: str, dimensions: tuple[Dimension, ...], estimated: tuple[Metric, ...]
-    ) -> AdmissionDecision:
-        """Apply existing admission policies and estimates; duplicate IDs add no hold."""
-        ...
-
-    async def settle(self, reservation_id: str | None, cost_event: CostEvent) -> None:
-        """Settle the committed charge with atomic deduplication and hold mutation.
-
-        Preserve scope/window attribution and retry unfinished settlement after
-        restart. SQL projection receipts do not provide live-counter atomicity.
-        """
-        ...
-
-    async def release(self, reservation_id: str) -> None:
-        """Idempotently release unused exposure at the request boundary, preserving charges."""
-        ...

litellm/accounting/snapshot.py

@@ -0,0 +1,161 @@
+from collections.abc import Callable, Mapping, Sequence
+from datetime import datetime
+from typing import TYPE_CHECKING, Final
+
+from pydantic import TypeAdapter, ValidationError
+
+from litellm._logging import verbose_proxy_logger
+from litellm.litellm_core_utils.safe_json_dumps import safe_json_structure
+from litellm.proxy._types import SpendLogsPayload
+from litellm.proxy.spend_tracking.compression_savings import extract_compression_saved_tokens
+from litellm.proxy.spend_tracking.savings import SavingsSpend, compute_savings_spend, marks_gateway_injection
+
+if TYPE_CHECKING:
+    from litellm.router import Router
+
+_FIELDS: Final = TypeAdapter(dict[str, object])
+
+
+def _mapping(value: object) -> Mapping[str, object] | None:
+    return _FIELDS.validate_python(value) if isinstance(value, Mapping) else None
+
+
+_METADATA_FIELDS: Final = frozenset(
+    {
+        "usage_object",
+        "status",
+        "guardrail_information",
+        "standard_logging_guardrail_information",
+        "compression_savings",
+        "internal_call_origin",
+        "billed_retry_attempt",
+    }
+)
+
+
+_ACCOUNTING_FIELDS: Final = frozenset(
+    {
+        "request_id",
+        "call_type",
+        "api_key",
+        "spend",
+        "prompt_tokens",
+        "completion_tokens",
+        "startTime",
+        "model",
+        "model_group",
+        "custom_llm_provider",
+        "user",
+        "team_id",
+        "organization_id",
+        "end_user",
+        "agent_id",
+        "request_tags",
+        "status",
+        "request_duration_ms",
+        "mcp_namespaced_tool_name",
+    }
+)
+
+
+def build_accounting_payload(
+    payload: SpendLogsPayload,
+    *,
+    entities: Mapping[str, str | None],
+    model_access_groups: Sequence[str],
+    windows: Sequence[Mapping[str, object]],
+    router: "Callable[[], Router | None]",
+) -> Mapping[str, object]:
+    metadata: Final = _FIELDS.validate_json(payload["metadata"])
+    savings: Final = _savings_or_zero(payload, metadata, router)
+    accounting_metadata: Final = {
+        **{key: _accounting_metadata_field(key, value) for key, value in metadata.items() if key in _METADATA_FIELDS},
+        "accounting_entities": entities,
+        "accounting_savings": savings._asdict(),
+        "accounting_model_access_groups": model_access_groups,
+        "accounting_windows": windows,
+    }
+    return _FIELDS.validate_python(
+        safe_json_structure(
+            {
+                **{
+                    key: value.isoformat() if isinstance(value, datetime) else value
+                    for key, value in payload.items()
+                    if key in _ACCOUNTING_FIELDS
+                },
+                "metadata": accounting_metadata,
+            },
+            non_finite_as_null=True,
+        )
+    )
+
+
+def _savings_or_zero(
+    payload: SpendLogsPayload, metadata: Mapping[str, object], router: "Callable[[], Router | None]"
+) -> SavingsSpend:
+    """Savings are analytics computed from caller-supplied counts, so one that cannot be computed is stored as zero
+    rather than losing the charge it rides with."""
+    try:
+        return _savings(payload, metadata, router)
+    except (ArithmeticError, ValueError, TypeError) as error:
+        verbose_proxy_logger.warning(
+            "Recording zero savings for request_id=%s, savings could not be computed: %s", payload["request_id"], error
+        )
+        return SavingsSpend(compression=0.0, prompt_caching=0.0)
+
+
+def _savings(
+    payload: SpendLogsPayload, metadata: Mapping[str, object], router: "Callable[[], Router | None]"
+) -> SavingsSpend:
+    return compute_savings_spend(
+        model=payload["model"],
+        custom_llm_provider=payload["custom_llm_provider"],
+        compression_saved_tokens=extract_compression_saved_tokens(metadata),
+        gateway_injected_cache=marks_gateway_injection(metadata, payload["model_id"]),
+        routing_decision=_mapping(metadata.get("routing_decision")),
+        model_id=payload["model_id"],
+        llm_router=router,
+        usage_object=_mapping(metadata.get("usage_object")),
+        cost_breakdown=_mapping(metadata.get("cost_breakdown")),
+        recorded_autorouter_savings=metadata.get("autorouter_savings"),
+        recorded_autorouter_savings_estimate=_mapping(metadata.get("autorouter_savings_estimate")),
+        billed_at=payload["endTime"],
+    )
+
+
+_GUARDRAIL_ENTRIES: Final = TypeAdapter(tuple[dict[str, object], ...])
+_GUARDRAIL_METADATA_FIELDS: Final = frozenset({"guardrail_information", "standard_logging_guardrail_information"})
+_GUARDRAIL_ACCOUNTING_FIELDS: Final = frozenset(
+    {
+        "guardrail_id",
+        "guardrail_name",
+        "guardrail_provider",
+        "guardrail_usage",
+        "guardrail_cost_by_unit",
+        "guardrail_cost_in_spend",
+    }
+)
+_GUARDRAIL_RESPONSE: Final = TypeAdapter(dict[str, object])
+
+
+def _accounting_guardrail_entries(value: object) -> object:
+    try:
+        entries: Final = _GUARDRAIL_ENTRIES.validate_python((value,) if isinstance(value, dict) else value)
+    except ValidationError:
+        return None
+    return [_accounting_guardrail_entry(entry) for entry in entries]
+
+
+def _accounting_guardrail_entry(entry: Mapping[str, object]) -> Mapping[str, object]:
+    kept: Final = {key: item for key, item in entry.items() if key in _GUARDRAIL_ACCOUNTING_FIELDS}
+    response: Final = entry.get("guardrail_response")
+    if not isinstance(response, dict) or "tokens_saved" not in response:
+        return kept
+    return {
+        **kept,
+        "guardrail_response": {"tokens_saved": _GUARDRAIL_RESPONSE.validate_python(response)["tokens_saved"]},
+    }
+
+
+def _accounting_metadata_field(key: str, value: object) -> object:
+    return _accounting_guardrail_entries(value) if key in _GUARDRAIL_METADATA_FIELDS else value

litellm/accounting/types.py

@@ -1,428 +0,0 @@
-"""Accounting values use UTC instants, decimal strings on the wire and USD charges."""
-
-from collections.abc import Mapping
-from datetime import datetime, timezone
-from decimal import Decimal
-from enum import Enum
-from typing import Annotated, Final, Literal, TypeAlias
-
-from pydantic import AfterValidator, AwareDatetime, BaseModel, ConfigDict, Field, model_validator
-from typing_extensions import Self
-
-from litellm.types.utils import CallTypes
-
-_CONFIG: Final = ConfigDict(frozen=True, extra="forbid", validate_default=True)
-Identifier: TypeAlias = Annotated[str, Field(strict=True, min_length=1, pattern=r"\S")]
-Quantity: TypeAlias = Annotated[Decimal, Field(strict=True, ge=0, allow_inf_nan=False)]
-
-
-def _utc(value: datetime) -> datetime:
-    return value.astimezone(timezone.utc)
-
-
-Instant: TypeAlias = Annotated[AwareDatetime, AfterValidator(_utc)]
-
-
-class MetricName(str, Enum):
-    """Reasoning tokens are included in completion tokens; above-1hr writes are included in cache creation tokens."""
-
-    REQUESTS = "requests"
-    SUCCESSFUL_REQUESTS = "successful_requests"
-    FAILED_REQUESTS = "failed_requests"
-    PROMPT_TOKENS = "prompt_tokens"
-    COMPLETION_TOKENS = "completion_tokens"
-    REASONING_TOKENS = "reasoning_tokens"
-    TOTAL_TOKENS = "total_tokens"
-    CACHE_READ_INPUT_TOKENS = "cache_read_input_tokens"
-    CACHE_CREATION_INPUT_TOKENS = "cache_creation_input_tokens"
-    CACHE_CREATION_INPUT_TOKENS_ABOVE_1HR = "cache_creation_input_tokens_above_1hr"
-    SPEND = "spend"
-
-
-class DimensionName(str, Enum):
-    KEY = "key"
-    USER = "user"
-    TEAM = "team"
-    ORGANIZATION = "organization"
-    END_USER = "end_user"
-    MODEL = "model"
-    MODEL_GROUP = "model_group"
-    DEPLOYMENT = "deployment"
-    PROVIDER = "provider"
-    TOOL = "tool"
-    AGENT = "agent"
-    PROJECT = "project"
-    MODEL_ACCESS_GROUP = "model_access_group"
-    API_BASE = "api_base"
-    SERVICE_TIER = "service_tier"
-    TAG = "tag"
-
-
-class Outcome(str, Enum):
-    SUCCEEDED = "succeeded"
-    FAILED = "failed"
-    CANCELLED = "cancelled"
-
-
-class Granularity(str, Enum):
-    DAY = "day"
-
-
-class EventKind(str, Enum):
-    INBOUND_USAGE = "inbound_usage"
-    OUTBOUND_USAGE = "outbound_usage"
-    COST = "cost"
-
-
-class DeliveryState(str, Enum):
-    PENDING = "pending"
-    APPLIED = "applied"
-    PARKED = "parked"
-
-
-class DeliveryTarget(str, Enum):
-    """One member per existing table the projector writes; each is applied and receipted independently."""
-
-    KEY_SPEND = "LiteLLM_VerificationToken"
-    USER_SPEND = "LiteLLM_UserTable"
-    TEAM_SPEND = "LiteLLM_TeamTable"
-    TEAM_MEMBER_SPEND = "LiteLLM_TeamMembership"
-    ORGANIZATION_SPEND = "LiteLLM_OrganizationTable"
-    ORGANIZATION_MEMBER_SPEND = "LiteLLM_OrganizationMembership"
-    END_USER_SPEND = "LiteLLM_EndUserTable"
-    AGENT_SPEND = "LiteLLM_AgentsTable"
-    TAG_SPEND = "LiteLLM_TagTable"
-    PROJECT_SPEND = "LiteLLM_ProjectTable"
-    MODEL_ACCESS_GROUP_SPEND = "LiteLLM_ModelAccessGroupBudgetTable"
-    BUDGET_SPEND = "LiteLLM_BudgetTable"
-    BUDGET_WINDOW_SPEND = "LiteLLM_BudgetWindowSpend"
-    DAILY_USER_SPEND = "LiteLLM_DailyUserSpend"
-    DAILY_TEAM_SPEND = "LiteLLM_DailyTeamSpend"
-    DAILY_ORGANIZATION_SPEND = "LiteLLM_DailyOrganizationSpend"
-    DAILY_END_USER_SPEND = "LiteLLM_DailyEndUserSpend"
-    DAILY_AGENT_SPEND = "LiteLLM_DailyAgentSpend"
-    DAILY_TAG_SPEND = "LiteLLM_DailyTagSpend"
-    DAILY_MODEL_USAGE = "LiteLLM_DailyModelUsage"
-    DAILY_TOOL_SPEND = "LiteLLM_DailyToolSpend"
-    AUTOROUTER_DAILY_SPEND = "LiteLLM_AutoRouterDailySpend"
-
-
-_SUBSET_PAIRS: Final = (
-    (MetricName.REASONING_TOKENS, MetricName.COMPLETION_TOKENS),
-    (MetricName.CACHE_CREATION_INPUT_TOKENS_ABOVE_1HR, MetricName.CACHE_CREATION_INPUT_TOKENS),
-)
-
-
-def _check_subsets(values: Mapping[str, Decimal]) -> None:
-    for part, whole in _SUBSET_PAIRS:
-        if part.value in values and whole.value in values and values[part.value] > values[whole.value]:
-            raise ValueError(f"{part.value} cannot exceed {whole.value}")
-
-
-class Metric(BaseModel):
-    model_config = _CONFIG
-
-    name: MetricName
-    value: Quantity
-    unit: Literal["count", "USD"] = "count"
-
-    @model_validator(mode="after")
-    def validate_unit(self) -> Self:
-        if (self.name is MetricName.SPEND) != (self.unit == "USD"):
-            raise ValueError("SPEND requires USD; usage metrics require count")
-        if self.unit == "count" and self.value != self.value.to_integral_value():
-            raise ValueError("Count metrics require integral values")
-        return self
-
-
-class Dimension(BaseModel):
-    model_config = _CONFIG
-
-    name: DimensionName
-    value: Identifier
-
-
-_REPEATABLE_DIMENSIONS: Final = frozenset({DimensionName.TAG, DimensionName.MODEL_ACCESS_GROUP})
-
-
-def _dimensions(values: tuple[Dimension, ...]) -> tuple[Dimension, ...]:
-    if len(frozenset(values)) != len(values):
-        raise ValueError("Duplicate dimension values")
-    single_names: Final = tuple(value.name for value in values if value.name not in _REPEATABLE_DIMENSIONS)
-    if len(frozenset(single_names)) != len(single_names):
-        raise ValueError("Only tag and model access group dimensions may have multiple values")
-    return values
-
-
-Dimensions: TypeAlias = Annotated[tuple[Dimension, ...], AfterValidator(_dimensions)]
-_CALLER_METRICS: Final = frozenset((MetricName.REQUESTS, MetricName.SUCCESSFUL_REQUESTS, MetricName.FAILED_REQUESTS))
-
-
-def _metrics(values: tuple[Metric, ...]) -> tuple[Metric, ...]:
-    if len(frozenset(value.name for value in values)) != len(values):
-        raise ValueError("Duplicate metric names")
-    return values
-
-
-Metrics: TypeAlias = Annotated[tuple[Metric, ...], AfterValidator(_metrics)]
-
-
-def _interval(start: datetime, end: datetime, *, allow_empty: bool = False) -> None:
-    if end < start or (end == start and not allow_empty):
-        raise ValueError("End must follow start")
-
-
-class InboundUsageEvent(BaseModel):
-    model_config = _CONFIG
-
-    event_id: Identifier
-    request_id: Identifier
-    call_type: CallTypes
-    started_at: Instant
-    ended_at: Instant
-    outcome: Outcome
-    dimensions: Dimensions
-    metrics: Metrics
-
-    @model_validator(mode="after")
-    def validate_caller_activity(self) -> Self:
-        _interval(self.started_at, self.ended_at, allow_empty=True)
-        if any(metric.name not in _CALLER_METRICS for metric in self.metrics):
-            raise ValueError("Inbound events own only caller request metrics")
-        if not any(metric.name is MetricName.REQUESTS and metric.value == 1 for metric in self.metrics):
-            raise ValueError("Inbound events count exactly one caller request")
-        for metric in self.metrics:
-            if metric.name is MetricName.SUCCESSFUL_REQUESTS and metric.value != int(self.outcome is Outcome.SUCCEEDED):
-                raise ValueError("Successful request count must match outcome")
-            if metric.name is MetricName.FAILED_REQUESTS and metric.value != int(self.outcome is not Outcome.SUCCEEDED):
-                raise ValueError("Failed request count must match outcome")
-        return self
-
-
-class OutboundUsageEvent(BaseModel):
-    model_config = _CONFIG
-
-    event_id: Identifier
-    attempt_id: Identifier
-    request_id: Identifier | None
-    retry_of_attempt_id: Identifier | None
-    call_type: CallTypes
-    started_at: Instant
-    ended_at: Instant
-    outcome: Outcome
-    dimensions: Dimensions
-    metrics: Metrics | None
-
-    @model_validator(mode="after")
-    def validate_attempt_usage(self) -> Self:
-        _interval(self.started_at, self.ended_at, allow_empty=True)
-        if self.retry_of_attempt_id == self.attempt_id:
-            raise ValueError("An attempt cannot retry itself")
-        if self.metrics is not None and any(
-            metric.name in _CALLER_METRICS or metric.name is MetricName.SPEND for metric in self.metrics
-        ):
-            raise ValueError("Outbound events own only attempt usage metrics")
-        if self.metrics is not None:
-            _check_subsets({metric.name.value: metric.value for metric in self.metrics})
-        return self
-
-
-PricingUnit: TypeAlias = Literal["token", "second", "character", "image", "request"]
-
-
-class Rate(BaseModel):
-    model_config = _CONFIG
-
-    measure: Identifier
-    unit: PricingUnit
-    usd_per_unit: Quantity
-
-
-class PricedQuantity(BaseModel):
-    """A pricing input in its billing unit. Only seconds may be fractional."""
-
-    model_config = _CONFIG
-
-    measure: Identifier
-    unit: PricingUnit
-    value: Quantity
-
-    @model_validator(mode="after")
-    def validate_discrete_quantity(self) -> Self:
-        if self.unit != "second" and self.value != self.value.to_integral_value():
-            raise ValueError("Discrete priced quantities require integral values")
-        return self
-
-
-class Multiplier(BaseModel):
-    """A regional or tier factor applied on top of the priced key, e.g. data_residency=eu or vertex_location=us-east5."""
-
-    model_config = _CONFIG
-
-    name: Identifier
-    value: Identifier
-    factor: Annotated[Decimal, Field(strict=True, gt=0, allow_inf_nan=False)]
-
-
-class NamedCost(BaseModel):
-    """A free-form extra charge by name, e.g. azure_model_router_flat_cost."""
-
-    model_config = _CONFIG
-
-    name: Identifier
-    amount: Quantity
-
-
-class PricingSnapshot(BaseModel):
-    """Retained inputs and result of authoritative compilation, reproducible after cost-map rates change.
-
-    Request-scoped inputs, overrides and effective rates come from the usage row's versioned payload.
-    Retries reuse the committed charge and snapshot.
-    Component names match ``CostBreakdown``: reasoning_cost is part of output_cost, not an additional charge.
-    Component sums and float tolerances belong to reconciliation, not this structural contract.
-    """
-
-    model_config = _CONFIG
-
-    schema_version: Literal[1] = 1
-    calculator_version: Identifier
-    model: Identifier
-    provider: Identifier | None
-    pricing_source: Literal["cost_map", "deployment", "per_request", "upstream_header"]
-    priced_model_key: Identifier
-    multipliers: tuple[Multiplier, ...] = ()
-    rates: tuple[Rate, ...] = ()
-    usage: tuple[PricedQuantity, ...] = ()
-    input_cost: Quantity | None = None
-    output_cost: Quantity | None = None
-    cache_read_cost: Quantity | None = None
-    cache_creation_cost: Quantity | None = None
-    reasoning_cost: Quantity | None = None
-    tool_usage_cost: Quantity | None = None
-    guardrail_cost: Quantity | None = None
-    additional_costs: tuple[NamedCost, ...] = ()
-    original_cost: Quantity | None = None
-    discount_percent: Quantity | None = None
-    discount_amount: Quantity | None = None
-    margin_percent: Quantity | None = None
-    margin_fixed_amount: Quantity | None = None
-    margin_total_amount: Quantity | None = None
-    upstream_reported_cost: Quantity | None = None
-    total_cost: Quantity
-
-    @model_validator(mode="after")
-    def validate_snapshot(self) -> Self:
-        rate_keys: Final = frozenset((rate.measure, rate.unit) for rate in self.rates)
-        quantity_keys: Final = frozenset((quantity.measure, quantity.unit) for quantity in self.usage)
-        if len(rate_keys) != len(self.rates):
-            raise ValueError("Duplicate rate measures and units")
-        if len(quantity_keys) != len(self.usage):
-            raise ValueError("Duplicate priced quantity measures and units")
-        if not rate_keys.issubset(quantity_keys):
-            raise ValueError("Every rate requires one priced quantity with the same measure and unit")
-        if len(frozenset(cost.name for cost in self.additional_costs)) != len(self.additional_costs):
-            raise ValueError("Duplicate additional cost names")
-        if len(frozenset(multiplier.name for multiplier in self.multipliers)) != len(self.multipliers):
-            raise ValueError("Duplicate multiplier names")
-        _check_subsets({quantity.measure: quantity.value for quantity in self.usage if quantity.unit == "token"})
-        return self
-
-
-class CostEvent(BaseModel):
-    model_config = _CONFIG
-
-    event_id: Identifier
-    usage_event_id: Identifier
-    occurred_at: Instant
-    dimensions: Dimensions
-    charges: Metrics
-    snapshot: PricingSnapshot
-
-    @model_validator(mode="after")
-    def validate_charge(self) -> Self:
-        if len(self.charges) != 1 or self.charges[0].name is not MetricName.SPEND:
-            raise ValueError("A cost event requires one authoritative USD spend metric")
-        if self.event_id == self.usage_event_id:
-            raise ValueError("Cost and usage events require distinct identities")
-        if self.snapshot.total_cost != self.charges[0].value:
-            raise ValueError("Snapshot total must equal the authoritative spend")
-        return self
-
-
-class Unpriced(BaseModel):
-    model_config = _CONFIG
-
-    usage_event_id: Identifier
-    reason: Identifier
-
-
-class Measurement(BaseModel):
-    model_config = _CONFIG
-
-    source_event_id: Identifier
-    occurred_at: Instant
-    dimensions: Dimensions
-    metric: Metric
-
-
-class DimensionFilter(BaseModel):
-    model_config = _CONFIG
-
-    name: DimensionName
-    values: Annotated[tuple[Identifier, ...], Field(min_length=1)]
-
-
-class AggregateQuery(BaseModel):
-    model_config = _CONFIG
-
-    metric: MetricName
-    filters: tuple[DimensionFilter, ...]
-    group_by: tuple[DimensionName, ...]
-    start: Instant
-    end: Instant
-    granularity: Granularity | None
-
-    @model_validator(mode="after")
-    def validate_query(self) -> Self:
-        _interval(self.start, self.end)
-        if len(frozenset(item.name for item in self.filters)) != len(self.filters):
-            raise ValueError("Use one IN filter per dimension; filters combine with AND")
-        if len(frozenset(self.group_by)) != len(self.group_by):
-            raise ValueError("Duplicate group-by dimensions")
-        if any(
-            (instant.hour, instant.minute, instant.second, instant.microsecond) != (0, 0, 0, 0)
-            for instant in (self.start, self.end)
-        ):
-            raise ValueError("Initial reporting queries require UTC day boundaries")
-        return self
-
-
-class AggregateRow(BaseModel):
-    model_config = _CONFIG
-
-    start: Instant
-    end: Instant
-    dimensions: Dimensions
-    value: Quantity
-
-    @model_validator(mode="after")
-    def validate_bounds(self) -> Self:
-        _interval(self.start, self.end)
-        return self
-
-
-def validate_aggregate_rows(rows: tuple[AggregateRow, ...]) -> tuple[AggregateRow, ...]:
-    """Validate [start, end) buckets per group; overlapping tag groups are not additive."""
-    groups: Final = frozenset(frozenset(row.dimensions) for row in rows)
-    for group in groups:
-        ordered = sorted((row for row in rows if frozenset(row.dimensions) == group), key=lambda row: row.start)
-        if any(left.end > right.start for left, right in zip(ordered, ordered[1:])):
-            raise ValueError("Aggregate buckets overlap within a dimension group")
-    return rows
-
-
-class AdmissionDecision(BaseModel):
-    model_config = _CONFIG
-
-    allowed: bool
-    reason: Identifier | None = None

litellm/accounting/worker.py

@@ -0,0 +1,102 @@
+import os
+import time
+from typing import (
+    TYPE_CHECKING,
+    Final,
+    Literal,
+    Protocol,
+    cast,  # noqa: TID251  # generated Prisma transaction signatures cannot express the borrowed transaction protocol
+)
+
+from litellm._logging import verbose_proxy_logger
+from litellm.accounting.application import apply_deliveries
+from litellm.accounting.metrics import record_deliveries
+from litellm.accounting.projection import Contribution
+from litellm.repositories.accounting_delivery_repository import (
+    AccountingDeliveryRepository,
+    DeliveryProjection,
+    LedgerDatabase,
+    LedgerTransaction,
+)
+from litellm.types.integrations.slack_alerting import AlertType
+
+if TYPE_CHECKING:
+    from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter
+    from litellm.proxy.utils import PrismaClient, ProxyLogging
+
+
+class _AccountingAlerts(Protocol):
+    async def alerting_handler(
+        self, message: str, level: Literal["Low", "Medium", "High"], alert_type: AlertType
+    ) -> None: ...
+
+
+_JOB: Final = "accounting_delivery"
+
+
+async def flush_accounting_deliveries(
+    writer: "DBSpendUpdateWriter", prisma_client: "PrismaClient", proxy_logging_obj: "ProxyLogging"
+) -> None:
+    acquired: Final = await writer.pod_lock_manager.acquire_lock(cronjob_id=_JOB, allow_reentrant=False)
+    if acquired is False:
+        return
+    applied: tuple[Contribution, ...] = ()  # rebind-ok: keep committed contributions for cache invalidation
+
+    async def apply(transaction: LedgerTransaction, deliveries: tuple[DeliveryProjection, ...]) -> None:
+        nonlocal applied
+        applied = await apply_deliveries(  # rebind-ok: invalidate caches only after the outer transaction commits
+            transaction, deliveries, prisma_client=prisma_client, writer=writer, proxy_logging_obj=proxy_logging_obj
+        )
+
+    try:
+        database: Final = cast(  # cast-ok: PrismaClient.tx owns writer routing and yields the generated transaction
+            LedgerDatabase, prisma_client
+        )
+        repository: Final = AccountingDeliveryRepository(database)
+        started: Final = time.monotonic()
+        batch_size: Final = int(os.environ.get("LITELLM_ACCOUNTING_DELIVERY_BATCH_SIZE", "10000"))
+
+        async def run_batch(*, retry_only: bool) -> bool:
+            result: Final = await repository.apply_pending(apply, batch_size=batch_size, retry_only=retry_only)
+            if result.applied:
+                await writer.invalidate_accounting_entities(applied, proxy_logging_obj)
+            if result.parked:
+                verbose_proxy_logger.error(
+                    "Accounting delivery parked %d requests after isolated failures", len(result.parked)
+                )
+                await _alert(
+                    proxy_logging_obj,
+                    message=f"Accounting delivery parked {len(result.parked)} requests. Included spend totals require repair.",
+                )
+            return bool(result.applied or result.failed)
+
+        await run_batch(retry_only=False)
+        retry_deadline: Final = time.monotonic() + 5
+        for _ in range(batch_size):
+            if time.monotonic() >= retry_deadline or not await run_batch(retry_only=True):
+                break
+        await repository.cleanup_applied(
+            retention_days=int(os.environ.get("LITELLM_ACCOUNTING_DELIVERY_RETENTION_DAYS", "1")),
+            batch_size=batch_size * 2,
+        )
+        metrics: Final = await repository.metrics()
+        record_deliveries(metrics, time.monotonic() - started)
+        if metrics.batch_count >= int(
+            os.environ.get("LITELLM_ACCOUNTING_BATCH_DELIVERY_ALERT_COUNT", "1000000")
+        ) or metrics.table_bytes >= int(os.environ.get("LITELLM_ACCOUNTING_DELIVERY_ALERT_BYTES", "1073741824")):
+            await _alert(
+                proxy_logging_obj,
+                message=f"Accounting delivery storage requires review. Retained batches={metrics.batch_count}, table bytes={metrics.table_bytes}.",
+            )
+        if metrics.oldest_pending_seconds > 300:
+            await _alert(
+                proxy_logging_obj,
+                message=f"Accounting deliveries are delayed. Pending={metrics.pending_count}, oldest={metrics.oldest_pending_seconds:.0f}s.",
+            )
+    finally:
+        if acquired:
+            await writer.pod_lock_manager.release_lock(cronjob_id=_JOB)
+
+
+async def _alert(alerts: _AccountingAlerts, message: str) -> None:
+    await alerts.alerting_handler(message=message, level="High", alert_type=AlertType.failed_tracking_spend)

litellm/batches/batch_utils.py

@@ -10,6 +10,7 @@ from pydantic import JsonValue, TypeAdapter
 
 import litellm
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils import batch_utils as core_batch_utils
 from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS
 from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details
 from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo
@@ -38,8 +39,8 @@ class BatchCostUsageResult:
     completion_cost: float = 0.0
 
 
-_COMPLETED_BATCH_STATUSES: Final = frozenset({"completed", "complete"})
 _DECODED_BATCH_LINE: Final = TypeAdapter(object)
+batch_cost_is_final: Final = core_batch_utils.batch_cost_is_final
 
 
 def _uses_native_vertex_output(
@@ -54,26 +55,6 @@ def _uses_native_vertex_output(
     return first_row is not None and is_native_vertex_batch_output_row(first_row)
 
 
-_TERMINAL_BATCH_STATUSES: Final = _COMPLETED_BATCH_STATUSES | frozenset({"failed", "cancelled", "expired"})
-
-
-def batch_cost_is_final(batch: Batch) -> bool:
-    """Whether this retrieve of the batch is the one to account its cost from.
-
-    A batch still in flight has nothing to price, and a "completed" batch can report
-    no output_file_id for a moment before the output populates; pricing either records
-    $0 under the batch's single spend row and pins it there. Final means a completed
-    batch whose output file has arrived or whose counts prove no line succeeded, or
-    any other terminal status (failed, cancelled, expired).
-    """
-    if batch.status not in _TERMINAL_BATCH_STATUSES:
-        return False
-    if batch.status not in _COMPLETED_BATCH_STATUSES or batch.output_file_id is not None:
-        return True
-    request_counts: Final = batch.request_counts
-    return request_counts is not None and request_counts.total > 0 and request_counts.completed == 0
-
-
 async def calculate_batch_cost_and_usage(
     file_content_dictionary: Sequence[dict[str, object]],
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"],

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -1180,11 +1180,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             include_namespace=True,
             replay_openai_history=provider == "openai",
         )
-        # OpenAI's Responses API rejects an empty input. For a system-only
-        # request, carry the system message as a system-role input item instead
-        # of instructions, mirroring how non-string system content is already
-        # handled in convert_chat_completion_messages_to_responses_api.
-        is_system_only_request: Final = not converted_input_items and converted_instructions is not None
+        # Keep the system text in input instead of instructions when the Responses API needs it there.
+        # It rejects an empty input, and json_object requires the word "json" in input, ignoring instructions.
+        typed_optional_params: Final = cast("Mapping[str, object]", optional_params)  # cast-ok: untyped dict
+        system_goes_in_input: Final = converted_instructions is not None and (
+            not converted_input_items or _payload_type(typed_optional_params.get("response_format")) == "json_object"
+        )
         target_input_items: Final = (
             _without_audio_input_parts(converted_input_items)
             if _drops_audio_input(model, litellm_params)
@@ -1196,12 +1197,13 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                     "type": "message",
                     "role": "system",
                     "content": [{"type": "input_text", "text": converted_instructions}],
-                }
+                },
+                *target_input_items,
             ]
-            if is_system_only_request
+            if system_goes_in_input
             else target_input_items
         )
-        instructions: Final = None if is_system_only_request else converted_instructions
+        instructions: Final = None if system_goes_in_input else converted_instructions
 
         # Build responses API request using the reverse transformation logic
         responses_api_request: Final = ResponsesAPIOptionalRequestParams()

litellm/cost_calculator.py

@@ -616,11 +616,17 @@ def cost_per_token(
         )
         if per_second is not None:
             return per_second
+        if custom_llm_provider == "anthropic":
+            return anthropic_cost_per_token(
+                model=model, usage=usage_block, service_tier=service_tier, model_info=requested_model_info_override
+            )
         return generic_cost_per_token(
             model=model,
             usage=usage_block,
             custom_llm_provider=custom_llm_provider or "",
             service_tier=service_tier,
+            data_residency=data_residency,
+            vertex_location=vertex_location,
             model_info=requested_model_info_override,
         )
 
@@ -3323,15 +3329,35 @@ def batch_cost_calculator(
     return total_prompt_cost * vertex_uplift, total_completion_cost * vertex_uplift
 
 
+def _standard_rate(model_info: ModelInfo, key: str, prompt_tokens: int) -> float | None:
+    return get_cost_per_unit(model_info, get_modality_token_cost_key(model_info, key, prompt_tokens), None)
+
+
+def _cached_audio_batch_fallback(
+    model_info: ModelInfo, batch_cache_read_rate: float | None, cache_read_rate: float | None, prompt_tokens: int
+) -> float:
+    if batch_cache_read_rate is not None:
+        return batch_cache_read_rate
+    standard_audio_rate: Final = _standard_rate(model_info, "cache_read_input_audio_token_cost", prompt_tokens)
+    if standard_audio_rate is not None:
+        return standard_audio_rate
+    return 0.0 if cache_read_rate is None else cache_read_rate
+
+
 def _batch_prompt_cost(
     usage: Usage,
     model_info: ModelInfo,
     input_rate: float,
-    cache_read_rate: float | None,
+    batch_cache_read_rate: float | None,
     cache_creation_rate: float | None,
     cache_creation_rates_by_ttl: Mapping[str, float],
 ) -> float:
     details: Final = parse_prompt_tokens_details(usage)
+    cache_read_rate: Final = (
+        _standard_rate(model_info, "cache_read_input_token_cost", usage.prompt_tokens)
+        if batch_cache_read_rate is None
+        else batch_cache_read_rate
+    )
     cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0
     cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cache_read_rate is not None else 0
     bills_cache_writes: Final = cache_creation_rate is not None or bool(cache_creation_rates_by_ttl)
@@ -3352,7 +3378,11 @@ def _batch_prompt_cost(
         + video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate)
         + (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0)
         + cached_audio_tokens
-        * _batch_rate(model_info, "cache_read_input_audio_token_cost_batches", cache_read_rate or 0.0)
+        * _batch_rate(
+            model_info,
+            "cache_read_input_audio_token_cost_batches",
+            _cached_audio_batch_fallback(model_info, batch_cache_read_rate, cache_read_rate, usage.prompt_tokens),
+        )
         + (
             calculate_cache_writing_cost(
                 written_tokens,

litellm/integrations/otel/model/spans.py

@@ -340,6 +340,7 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset(
         "LiteLLM_AccessGroupTable",
         "LiteLLM_AccountingDelivery",
         "LiteLLM_AccountingEvent",
+        "LiteLLM_SpendLogDelivery",
         "LiteLLM_ClaudeCodePluginTable",
         "LiteLLM_ClaudeCodePluginUpload",
         "LiteLLM_MemoryTable",

litellm/integrations/prometheus.py

@@ -54,6 +54,7 @@ from litellm.litellm_core_utils.get_llm_provider_logic import is_registered_cust
 from litellm.litellm_core_utils.service_tier_utils import (
     get_service_tier_from_standard_logging_payload,
 )
+from litellm.llms.azure.common_utils import caused_by_entra_rejection
 from litellm.llms.github_copilot.per_user_auth import github_copilot_user_session_from
 from litellm.llms.openai_like.json_loader import JSONProviderRegistry
 from litellm.models.end_user import LiteLLM_EndUserTable
@@ -448,7 +449,7 @@ def _is_deployment_failure(exception: openai.APIError, logged_call: Mapping[str,
     return not (
         isinstance(exception, CallerCredentialAuthenticationError)
         or bool(logged_call.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL))
-        or logged_call.get("api_call_start_time") is None
+        or (logged_call.get("api_call_start_time") is None and not caused_by_entra_rejection(exception))
         or _used_a_caller_credential(logged_call)
     )
 

litellm/litellm_core_utils/batch_utils.py

@@ -3,21 +3,32 @@ import json
 from collections.abc import Callable, Generator, Iterable, Mapping, Sequence
 from contextlib import contextmanager
 from contextvars import ContextVar
+from dataclasses import dataclass
 from os import PathLike
-from typing import Final, TypeVar
+from typing import Final, Literal, TypeVar
 
 from openai.types import Batch
 from pydantic import TypeAdapter, ValidationError
 
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.core_helpers import finite_cost
+from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_information_cost
 from litellm.types.llms.openai import FileTypes, OpenAIFilesPurpose
 from litellm.types.utils import CallTypes, LlmProviders
+from litellm.types.videos.main import VideoObject
+from litellm.types.videos.utils import POLLED_VIDEO_STATUS_KWARG, canonical_video_id, is_video_status_call_type
 
 _DECODED_JSONL_ROW: Final = TypeAdapter(object)
 _DEPLOYMENT_FIELD: Final = TypeAdapter(Mapping[str, object])
+_PAYLOAD_FIELDS: Final = TypeAdapter(Mapping[str, object])
 _ENDPOINT_PATHS: Final = TypeAdapter(tuple[str, ...])
 
+_COMPLETED_BATCH_STATUSES: Final = frozenset({"completed", "complete"})
+_TERMINAL_BATCH_STATUSES: Final = _COMPLETED_BATCH_STATUSES | frozenset({"failed", "cancelled", "expired"})
 BATCH_ENDPOINT: Final = "/v1/batch"
+POLLED_BATCH_ID_KWARG: Final = "polled_batch_id"
+POLLED_BATCH_COST_IS_FINAL_KWARG: Final = "polled_batch_cost_is_final"
+BATCH_COST_REQUEST_ID_SUFFIX: Final = "_batch_cost"
 _AZURE_PROVIDERS: Final = frozenset({LlmProviders.AZURE.value, LlmProviders.AZURE_AI.value})
 
 _batch_create_routing: Final[ContextVar[bool]] = ContextVar("batch_create_routing", default=False)
@@ -289,12 +300,151 @@ def is_batch_retrieve_call_type(call_type: object) -> bool:
     return isinstance(call_type, str) and call_type in BATCH_RETRIEVE_CALL_TYPES
 
 
-def batch_charged_once_marker_key(spend_key: str, batch_id: str) -> str:
-    return f"{spend_key}:batch:{batch_id}"
+@dataclass(frozen=True, slots=True)
+class ChargeOnceJob:
+    kind: Literal["batch", "video"]
+    job_id: str
+    repeat_poll_cost: float = 0.0
 
 
-def batch_id_to_charge_once(call_type: object, response_obj: object, response_cost: float) -> str | None:
-    """A finished batch reports its whole cost on every poll, so its id is charged once per counter."""
-    if response_cost <= 0 or not is_batch_retrieve_call_type(call_type):
+def job_charged_once_marker_key(spend_key: str, job: ChargeOnceJob) -> str:
+    return f"{spend_key}:{job.kind}:{job.job_id}"
+
+
+def _guardrail_cost_of_poll(standard_logging_payload: object) -> float:
+    guardrail_information: Final = (
+        _PAYLOAD_FIELDS.validate_python(standard_logging_payload).get("guardrail_information")
+        if isinstance(standard_logging_payload, Mapping)
+        else None
+    )
+    return guardrail_information_cost(guardrail_information)
+
+
+def charged_poll_cost(kwargs: Mapping[str, object]) -> object:
+    """The poll's whole cost as the cost callback charges it, guardrails included, which the logging kwargs' own
+    response_cost leaves out."""
+    standard_logging_payload: Final = kwargs.get("standard_logging_object")
+    logged_cost: Final = (
+        _PAYLOAD_FIELDS.validate_python(standard_logging_payload).get("response_cost")
+        if isinstance(standard_logging_payload, Mapping)
+        else None
+    )
+    return kwargs.get("response_cost") if logged_cost is None else logged_cost
+
+
+def job_cost_of_poll(response_cost: object, standard_logging_payload: object) -> float:
+    """What the polled job itself cost, since a guardrail on the poll charges every poll, running or finished."""
+    return (finite_cost(response_cost) or 0.0) - _guardrail_cost_of_poll(standard_logging_payload)
+
+
+def repeat_poll_cost(call_type: object, response_cost: object, standard_logging_payload: object) -> float:
+    """What a poll still costs once an earlier poll charged its job: a video poll's guardrails, which every video
+    poll has always paid, and nothing for a batch poll, whose repeats have always been free."""
+    if not is_video_status_call_type(call_type):
+        return 0.0
+    return max(0.0, min(finite_cost(response_cost) or 0.0, _guardrail_cost_of_poll(standard_logging_payload)))
+
+
+def batch_cost_is_final(batch: Batch) -> bool:
+    """Whether this retrieve of the batch is the one to account its cost from.
+
+    A batch still in flight has nothing to price, and a "completed" batch can report
+    no output_file_id for a moment before the output populates; pricing either records
+    $0 under the batch's single spend row and pins it there. Final means a completed
+    batch whose output file has arrived or whose counts prove no line succeeded, or
+    any other terminal status (failed, cancelled, expired).
+    """
+    if batch.status not in _TERMINAL_BATCH_STATUSES:
+        return False
+    if batch.status not in _COMPLETED_BATCH_STATUSES or batch.output_file_id is not None:
+        return True
+    request_counts: Final = batch.request_counts
+    return request_counts is not None and request_counts.total > 0 and request_counts.completed == 0
+
+
+@dataclass(frozen=True, slots=True)
+class PolledBatch:
+    id: str
+    cost_is_final: bool
+
+
+def _polled_batch(kwargs: Mapping[str, object], response_obj: object) -> PolledBatch | None:
+    """The retrieved batch, read from the logging kwargs when message redaction replaced the logged response."""
+    if not is_batch_retrieve_call_type(kwargs.get("call_type")):
         return None
-    return response_obj.id if isinstance(response_obj, Batch) else None
+    if isinstance(response_obj, Batch):
+        return PolledBatch(id=response_obj.id, cost_is_final=batch_cost_is_final(response_obj))
+    stamped_id: Final = kwargs.get(POLLED_BATCH_ID_KWARG)
+    stamped_cost_is_final: Final = kwargs.get(POLLED_BATCH_COST_IS_FINAL_KWARG)
+    if not isinstance(stamped_id, str) or not stamped_id or not isinstance(stamped_cost_is_final, bool):
+        return None
+    return PolledBatch(id=stamped_id, cost_is_final=stamped_cost_is_final)
+
+
+def defers_batch_cost(kwargs: Mapping[str, object], response_obj: object) -> bool:
+    """A batch retrieve before the batch's cost is final charges nothing, guardrails included, since the retrieve
+    that settles the batch charges its whole cost once."""
+    polled: Final = _polled_batch(kwargs, response_obj)
+    return polled is not None and not polled.cost_is_final
+
+
+def pays_only_its_guardrail(call_type: object, kwargs: Mapping[str, object], response_cost: object) -> bool:
+    """A video poll whose whole cost is its guardrails', charged like any other request."""
+    if not is_video_status_call_type(call_type):
+        return False
+    return (
+        job_cost_of_poll(response_cost, kwargs.get("standard_logging_object"))
+        <= 0
+        < (finite_cost(response_cost) or 0.0)
+    )
+
+
+def _requested_video_id(kwargs: Mapping[str, object]) -> str | None:
+    litellm_params: Final = kwargs.get("litellm_params")
+    requested: Final = (
+        _PAYLOAD_FIELDS.validate_python(litellm_params).get("video_id") if isinstance(litellm_params, Mapping) else None
+    )
+    return requested if isinstance(requested, str) and requested else None
+
+
+def charged_video_id(
+    call_type: object, kwargs: Mapping[str, object], fallback_video_id: str | None, response_cost: object
+) -> str | None:
+    """The id a finished video's own cost is charged once under.
+
+    That is the id the client polled, which names the deployment that made the video, since a
+    custom provider's own id can repeat across deployments. The status comes from the logging
+    kwargs because message redaction replaces the logged video object.
+    """
+    if not is_video_status_call_type(call_type) or kwargs.get(POLLED_VIDEO_STATUS_KWARG) != "completed":
+        return None
+    if job_cost_of_poll(response_cost, kwargs.get("standard_logging_object")) <= 0:
+        return None
+    video_id: Final = _requested_video_id(kwargs) or fallback_video_id
+    return None if video_id is None else canonical_video_id(video_id)
+
+
+def _charged_batch(kwargs: Mapping[str, object], response_obj: object, response_cost: float) -> ChargeOnceJob | None:
+    polled: Final = _polled_batch(kwargs, response_obj)
+    if polled is None or not polled.cost_is_final or response_cost <= 0:
+        return None
+    return ChargeOnceJob(kind="batch", job_id=polled.id)
+
+
+def job_to_charge_once(
+    kwargs: Mapping[str, object], response_obj: object, response_cost: float
+) -> ChargeOnceJob | None:
+    """A finished batch or video reports its whole cost on every poll, so it is charged once per counter."""
+    call_type: Final = kwargs.get("call_type")
+    if not is_video_status_call_type(call_type):
+        return _charged_batch(kwargs, response_obj, response_cost)
+    video_id: Final = charged_video_id(
+        call_type, kwargs, response_obj.id if isinstance(response_obj, VideoObject) else None, response_cost
+    )
+    if video_id is None:
+        return None
+    return ChargeOnceJob(
+        kind="video",
+        job_id=video_id,
+        repeat_poll_cost=repeat_poll_cost(call_type, response_cost, kwargs.get("standard_logging_object")),
+    )

litellm/litellm_core_utils/litellm_logging.py

@@ -19,7 +19,7 @@ from datetime import datetime as dt_object
 from enum import Enum
 from functools import lru_cache
 from types import MappingProxyType, TracebackType
-from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NamedTuple, TypeVar, cast
+from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, TypeVar, cast
 
 from httpx import Response, Timeout
 from openai import AsyncStream as OpenAIAsyncStream
@@ -38,7 +38,7 @@ from litellm._logging import (
     verbose_logger,
 )
 from litellm._uuid import uuid
-from litellm.batches.batch_utils import batch_cost_is_final, handle_completed_batch
+from litellm.batches.batch_utils import handle_completed_batch
 from litellm.caching.caching import DualCache
 from litellm.caching.caching_handler import LLMCachingHandler
 from litellm.caching.redis_batch import flush_post_call_redis_batches
@@ -79,6 +79,11 @@ from litellm.integrations.mlflow import MlflowLogger
 from litellm.integrations.sqs import SQSLogger
 from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
 from litellm.litellm_core_utils.agentic_followup_failure import ServedRound
+from litellm.litellm_core_utils.batch_utils import (
+    POLLED_BATCH_COST_IS_FINAL_KWARG,
+    POLLED_BATCH_ID_KWARG,
+    batch_cost_is_final,
+)
 from litellm.litellm_core_utils.classifier_logging import (
     classifier_audit_fields,
     classifier_input_snapshot,
@@ -220,6 +225,7 @@ from litellm.types.utils import (
     batch_chat_usage,
 )
 from litellm.types.videos.main import VideoObject
+from litellm.types.videos.utils import POLLED_VIDEO_STATUS_KWARG
 from litellm.utils import (
     get_base_model_from_metadata,
     print_verbose,
@@ -283,8 +289,6 @@ if TYPE_CHECKING:
     from litellm.llms.base_llm.passthrough.transformation import PassthroughStreamCollector
     from litellm.proxy.hooks.autorouter_baseline_cache import BaselineCacheContext, CapturedBaselineObservation
     from litellm.proxy.hooks.search_free_tier import SearchAllowanceAllocation
-    from litellm.proxy.spend_tracking.usage_capture import CapturedUsage
-    from litellm.proxy.spend_tracking.usage_meter import UsageCaptureError
 try:
     from litellm_enterprise.enterprise_callbacks.callback_controls import (
         EnterpriseCallbackControls,
@@ -662,54 +666,7 @@ class UsageObservation(Enum):
     ESTIMATED = "estimated"
 
 
-@dataclass(frozen=True, slots=True)
-class UsageCaptureObservation:
-    usage_json: str | None
-    provenance: UsageObservation
-    response_cost: float | None = None
-
-    @classmethod
-    def from_usage(
-        cls, usage: Usage | None, provenance: UsageObservation, response_cost: float | None = None
-    ) -> "UsageCaptureObservation | UsageCaptureError":
-        from litellm.proxy.spend_tracking.usage_meter import UsageCaptureError
-
-        try:
-            serialized: Final = (
-                None if usage is None or provenance is UsageObservation.UNKNOWN else usage.model_dump_json()
-            )
-        except Exception as error:  # noqa: BLE001  # observation failures must not interrupt legacy retry handling
-            return UsageCaptureError(str(error))
-        else:
-            return cls(serialized, provenance, response_cost)
-
-
 class Logging(LiteLLMLoggingBaseClass):
-    usage_capture_callback: ClassVar[
-        Callable[
-            ["Logging", object, datetime.datetime, datetime.datetime, bool], "CapturedUsage | UsageCaptureError | None"
-        ]
-        | None
-    ] = None
-
-    def capture_usage_before_logging(
-        self,
-        response: object,
-        start_time: datetime.datetime | None = None,
-        end_time: datetime.datetime | None = None,
-        *,
-        freeze_only: bool = False,
-    ) -> None:
-        capture: Final = type(self).usage_capture_callback
-        if capture is not None and (not self.stream or self._is_assembled_stream_success(response)):
-            capture(
-                self,
-                response,
-                start_time or self.start_time,
-                end_time or datetime.datetime.now(datetime.timezone.utc),
-                freeze_only,
-            )
-
     global \
         supabaseClient, \
         promptLayerLogger, \
@@ -949,7 +906,6 @@ class Logging(LiteLLMLoggingBaseClass):
         self._stash_owed_on_success: bool = False
         self._stash_billed_by_continuation: bool = False
         self._failed_attempts_owed_on_success: tuple[ServedRound, ...] = ()
-        self._failed_attempt_ids_owed_on_success: tuple[str, ...] = ()
         self.failed_attempts_billed_on_success: tuple[ServedRound, ...] = ()
         self.shadow_eval_request_snapshot: GuardrailRequestSnapshot | None = None
 
@@ -2582,11 +2538,6 @@ class Logging(LiteLLMLoggingBaseClass):
             )
             return
 
-        self.capture_usage_before_logging(
-            TypeAdapter(object).validate_python(result),
-            TypeAdapter[datetime.datetime | None](datetime.datetime | None).validate_python(start_time),
-            TypeAdapter[datetime.datetime | None](datetime.datetime | None).validate_python(end_time),
-        )
         await self.async_success_handler(
             result,
             start_time=start_time,
@@ -2698,20 +2649,12 @@ class Logging(LiteLLMLoggingBaseClass):
         self.finalized_cost = finalized
         self.model_call_details["response_cost"] = finalized.amount
 
-    def record_usage_capture(self, captured: "CapturedUsage") -> None:
-        self.model_call_details["_accounting_capture"] = captured
-
-    def record_usage_pricing(self, pricing: object) -> None:
-        self.model_call_details["_accounting_pricing_snapshot"] = pricing
-
     def record_partial_usage_for_failure(
         self, usage: Usage, response_cost: float, *, observation: UsageObservation = UsageObservation.KNOWN
     ) -> None:
         """Stash what an interrupted stream already consumed so the failure log bills it instead of zero."""
         self.model_call_details["combined_usage_object"] = usage
         self.model_call_details["response_cost"] = response_cost
-        if type(self).usage_capture_callback is not None:
-            self.model_call_details["_accounting_usage_observed"] = observation
 
     def record_failure_after_earlier_rounds(
         self, earlier: BilledRoundsSpend, failed_usage: Usage, failed_cost: float
@@ -2795,12 +2738,6 @@ class Logging(LiteLLMLoggingBaseClass):
         owed_on_success: Final = self._stash_owed_on_success
         self._stash_billed_by_continuation = False
         self._stash_owed_on_success = False
-        captured_attempt_ids: Final = (
-            self.capture_failed_attempt_usage(usage)
-            if isinstance(usage, Usage) and type(self).usage_capture_callback is not None
-            else ()
-        )
-        self.model_call_details.pop("_accounting_capture", None)
         self.model_call_details.pop("_accounting_pricing_snapshot", None)
         self.model_call_details.pop("_accounting_usage_observed", None)
         self.model_call_details.pop("combined_usage_object", None)
@@ -2816,60 +2753,10 @@ class Logging(LiteLLMLoggingBaseClass):
         if not owed_on_success or billed_by_continuation:
             return
         self._failed_attempts_owed_on_success = (*self._failed_attempts_owed_on_success, attempt)
-        self._failed_attempt_ids_owed_on_success = (*self._failed_attempt_ids_owed_on_success, *captured_attempt_ids)
 
     def mark_stash_billed_by_continuation(self) -> None:
         self._stash_billed_by_continuation = True
 
-    def capture_failed_attempt_usage(self, usage: Usage) -> tuple[str, ...]:
-        from litellm.constants import BILLED_RETRY_ATTEMPT_METADATA_KEY
-        from litellm.proxy.spend_tracking.usage_capture import CapturedUsage
-
-        capture: Final = type(self).usage_capture_callback
-        if capture is None:
-            return ()
-        attempt: Final = self.attempt_scoped_copy()
-        attempt.prepared_cost = None
-        attempt.finalized_cost = None
-        identity: Final = (
-            f"{self.litellm_call_id}-attempt-{len(self.model_call_details.get('failed_attempts_usage', ()))}"
-        )
-        params: Final = _LITELLM_PARAMS.validate_python(self.model_call_details.get("litellm_params") or {})
-        metadata: Final = StandardLoggingPayloadSetup.merge_litellm_metadata(params)
-        observed: Final[object] = self.model_call_details.get("_accounting_usage_observed")
-        cost: Final[object] = self.model_call_details.get("response_cost")
-        attempt.model_call_details = {
-            **attempt.model_call_details,
-            "litellm_call_id": identity,
-            "parent_request_id": self.litellm_call_id,
-            "_accounting_capture": None,
-            "_accounting_prior_captures": (),
-            "_accounting_usage_snapshot": UsageCaptureObservation.from_usage(
-                usage,
-                observed if isinstance(observed, UsageObservation) else UsageObservation.KNOWN,
-                float(cost) if isinstance(cost, (int, float)) else None,
-            ),
-            "litellm_params": {
-                **params,
-                "metadata": {**metadata, BILLED_RETRY_ATTEMPT_METADATA_KEY: True, "status": "failure"},
-                "litellm_metadata": {**metadata, BILLED_RETRY_ATTEMPT_METADATA_KEY: True, "status": "failure"},
-            },
-        }
-        captured: Final = capture(
-            attempt,
-            {"id": identity, "usage": usage.model_dump(), "status": "failed"},
-            self.start_time,
-            datetime.datetime.now(datetime.timezone.utc),
-            False,
-        )
-        if not isinstance(captured, CapturedUsage):
-            return ()
-        self.model_call_details["_accounting_prior_captures"] = (
-            *self.model_call_details.get("_accounting_prior_captures", ()),
-            captured,
-        )
-        return tuple(write.event.event_id for write in captured.writes)
-
     def record_continued_partial(self, usage: Usage, response_cost: float, response_id: str) -> None:
         """Bill what a cut stream already consumed on this continuation's success log, under the id the caller saw."""
         self.model_call_details["continued_partial_usage"] = usage
@@ -2909,8 +2796,6 @@ class Logging(LiteLLMLoggingBaseClass):
         self.model_call_details.pop("failed_attempts_usage", None)
         self.failed_attempts_billed_on_success = failed_attempts
         self.model_call_details["failed_attempts_billed_on_success"] = failed_attempts
-        if self._failed_attempt_ids_owed_on_success:
-            self.model_call_details["_accounting_included_attempt_ids"] = self._failed_attempt_ids_owed_on_success
         self.model_call_details["combined_usage_object"] = BaseTokenUsageProcessor.combine_usage_objects(
             usage_objects=[own_usage, *(usage for usage, _ in failed_attempts)]
         )
@@ -3073,45 +2958,6 @@ class Logging(LiteLLMLoggingBaseClass):
             return self._handle_a2a_response_logging(result=result)
         return result
 
-    def usage_capture_response(self, result: object) -> object:
-        if self.stream and not self._is_assembled_stream_success(result):
-            return None
-        copied: Final = result.model_copy(deep=True) if isinstance(result, BaseModel) else result
-        normalized: Final = self.normalize_logging_result(self.normalize_endpoint_logging_result(copied))
-        if self.model_call_details.get("call_type") == CallTypes.arealtime.value and isinstance(result, list):
-            frames: Final = TypeAdapter(tuple[dict[str, object], ...]).validate_python(result)
-            measured: Final = any(
-                isinstance(_LITELLM_PARAMS.validate_python(frame.get("response") or {}).get("usage"), Mapping)
-                for frame in frames
-                if frame.get("type") == "response.done"
-            )
-            if not measured:
-                metered: Final = self.model_call_details.get(REALTIME_METERED_USAGE_KEY)
-                return {
-                    "id": self.litellm_call_id,
-                    "usage": {"metered_usage": TypeAdapter(object).dump_python(metered, mode="json")}
-                    if metered is not None
-                    else None,
-                }
-        assembled: Final = self._get_assembled_streaming_response(
-            normalized, self.start_time, self.start_time, True, []
-        )
-        return normalized if assembled is None else assembled
-
-    def usage_capture_pricing(self, result: object) -> Mapping[str, JsonValue]:
-        from litellm.utils import count_characters
-
-        return TypeAdapter(dict[str, JsonValue]).validate_python(
-            {
-                "custom_pricing": self.custom_pricing_for(result),
-                "custom_cost_per_token": self.custom_cost_per_token,
-                "custom_cost_per_second": self.custom_cost_per_second,
-                "prompt_characters": count_characters(self._prompt_for_cost_calculation())
-                if self.model_call_details.get("call_type") in (CallTypes.speech.value, CallTypes.aspeech.value)
-                else None,
-            }
-        )
-
     def normalize_logging_result(self, result: object) -> object:
         """
         Some endpoints return a different type of result than what is expected by the logging system.
@@ -3468,6 +3314,11 @@ class Logging(LiteLLMLoggingBaseClass):
             result = self.normalize_endpoint_logging_result(result)
             if self.stream is not True:
                 result = self._as_served_response(result)  # rebind-ok: bridged chat calls log the served response
+            if isinstance(result, VideoObject):
+                self.model_call_details[POLLED_VIDEO_STATUS_KWARG] = result.status
+            if isinstance(result, LiteLLMBatch):
+                self.model_call_details[POLLED_BATCH_ID_KWARG] = result.id
+                self.model_call_details[POLLED_BATCH_COST_IS_FINAL_KWARG] = batch_cost_is_final(result)
 
             logging_result: Final = self.normalize_logging_result(result=result)
             if cache_hit is None and _served_from_cache_at_no_charge(logging_result):
@@ -3826,7 +3677,6 @@ class Logging(LiteLLMLoggingBaseClass):
                     self._as_served_chat_response(complete_streaming_response)
                 )
                 self.model_call_details["complete_streaming_response"] = logged_streaming_response
-                self.capture_usage_before_logging(logged_streaming_response, start_time, end_time)
                 ## STANDARDIZED LOGGING PAYLOAD
                 if not for_async_request:
                     self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
@@ -4352,8 +4202,6 @@ class Logging(LiteLLMLoggingBaseClass):
                     cost_for_built_in_tools_cost_usd_dollar=0.0,
                 )
 
-        self.capture_usage_before_logging(result, start_time, end_time)
-
         self.truncated_messages_for_logging = await truncate_base64_in_messages_async(
             StandardLoggingPayloadSetup.append_system_prompt_messages(
                 kwargs=self.model_call_details, messages=self.model_call_details.get("messages")
@@ -7295,6 +7143,7 @@ class StandardLoggingPayloadSetup:
         init_response_obj: object,
         api_base: str | None = None,
         router_model_id: str | None = None,
+        custom_model_info: ModelInfo | None = None,
     ) -> StandardLoggingModelInformation:
         model_cost_name: Final = select_model_name_for_cost_calc(
             model=base_model if custom_pricing else None,
@@ -7313,6 +7162,8 @@ class StandardLoggingPayloadSetup:
             if router_model_id is not None
             else model_cost_name
         )
+        if model_cost_name is not None and custom_model_info is not None:
+            return StandardLoggingModelInformation(model_map_key=model_cost_name, model_map_value=custom_model_info)
         if model_cost_name is None:
             model_cost_information = StandardLoggingModelInformation(model_map_key="", model_map_value=None)
         else:
@@ -8063,6 +7914,7 @@ def get_standard_logging_object_payload(
             init_response_obj=init_response_obj,
             api_base=litellm_params.get("api_base"),
             router_model_id=logging_obj.get_router_model_id(),
+            custom_model_info=logging_obj.custom_model_info if custom_pricing else None,
         )
 
         error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload(

litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py

@@ -224,7 +224,20 @@ class StandardBuiltInToolCostTracking:
             web_search_options=standard_built_in_tools_params.get("web_search_options", None),
             model_info=model_info,
         )
-        return per_call_cost * StandardBuiltInToolCostTracking._count_web_search_calls(response_object)
+        return per_call_cost * StandardBuiltInToolCostTracking._billable_web_search_calls(
+            resolved_usage, response_object
+        )
+
+    @staticmethod
+    def _billable_web_search_calls(usage: Usage | None, response_object: object) -> int:
+        """
+        Providers without their own calculator but whose usage carries server_tool_use.web_search_requests
+        (OpenAI-compatible providers that report a per-tool count) are billed that many calls on the per-call path.
+        """
+        reported: Final = None if usage is None else get_web_search_requests_from_usage(usage)
+        if isinstance(reported, int) and not isinstance(reported, bool) and reported > 0:
+            return reported
+        return StandardBuiltInToolCostTracking._count_web_search_calls(response_object)
 
     @staticmethod
     def _count_web_search_calls(response_object: object) -> int:

litellm/litellm_core_utils/realtime_streaming.py

@@ -836,8 +836,6 @@ class RealTimeStreaming:
             if self.session_tools or self.tool_calls:
                 self.logging_obj.model_call_details["realtime_tools"] = self.session_tools
                 self.logging_obj.model_call_details["realtime_tool_calls"] = self.tool_calls
-            if LiteLLMLogging.usage_capture_callback is not None:
-                self.logging_obj.capture_usage_before_logging(self.messages)
             # Route through the bounded logging worker (per-coroutine timeout +
             # concurrency cap) instead of a bare create_task, so a slow callback
             # can't leave suspended tasks pinning each call's response in memory.

litellm/litellm_core_utils/redact_messages.py

@@ -272,15 +272,31 @@ def _redacted_provider_specific_fields(fields: Mapping[str, object]) -> Mapping[
     return {
         **fields,
         **({"reasoning_content": REDACTED_BY_LITELLM} if fields.get("reasoning_content") is not None else {}),
+        **({"reasoning": REDACTED_BY_LITELLM} if fields.get("reasoning") is not None else {}),
+        **({"reasoning_text": REDACTED_BY_LITELLM} if fields.get("reasoning_text") is not None else {}),
         **{
             key: None
             for key in (
                 "thinking_blocks",
                 "reasoningContentBlocks",
+                "reasoningContent",
+                "annotations",
                 "citations",
+                "citation",
+                "citationsContent",
                 "compaction_blocks",
                 "compaction_start",
                 "compaction_delta",
+                "reasoning_details",
+                "web_search_calls",
+                "web_search_results",
+                "tool_results",
+                "code_interpreter_results",
+                "search_results",
+                "rerank_results",
+                "executed_tools",
+                "custom_outputs",
+                "tool_search_items",
                 "token_ids",
             )
             if key in fields

litellm/litellm_core_utils/tool_fields.py

@@ -10,7 +10,7 @@ from typing_extensions import ReadOnly, TypedDict
 import litellm
 from litellm._logging import verbose_logger
 from litellm.exceptions import UnsupportedParamsError
-from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params
+from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, normalize_drop_params
 
 ROUTES_WITH_TOOLS: Final = frozenset(
     {
@@ -58,6 +58,19 @@ _TOOL_ENTRY: Final = TypeAdapter(_ToolEntry)
 _WEB_SEARCH_OPTIONS: Final = TypeAdapter(dict[str, object] | None, config=ConfigDict(strict=True))
 
 
+def declared_function_name(tool: object) -> str | None:
+    fields: Final = as_str_mapping(tool)
+    if fields is None:
+        return None
+    function: Final = as_str_mapping(fields.get("function"))
+    name: Final = (fields if function is None else function).get("name")
+    return name if isinstance(name, str) and name else None
+
+
+def qualified_member_name(namespace: str, name: str) -> str:
+    return f"{namespace}__{name}"
+
+
 @dataclass(frozen=True, slots=True)
 class DropPolicy:
     drop_params: bool

litellm/litellm_core_utils/url_utils.py

@@ -102,6 +102,14 @@ class SSRFError(ValueError):
     """Raised when a URL targets a blocked network."""
 
 
+def is_https_url_with_host(url: str) -> bool:
+    try:
+        parsed_url: Final = urlsplit(url)
+        return parsed_url.scheme == "https" and bool(parsed_url.hostname)
+    except ValueError:
+        return False
+
+
 def encode_url_path_segment(value: object, *, field_name: str = "path parameter") -> str:
     """Percent-encode one user-controlled URL path segment.
 

litellm/llms/anthropic/chat/guardrail_translation/handler.py

@@ -781,12 +781,17 @@ class AnthropicMessagesHandler(BaseTranslation):
             for event in AnthropicMessagesHandler._parse_sse_data_line(line)
         )
 
-    def _translate_to_openai(self, data: dict) -> ChatCompletionRequest:
+    def _translate_to_openai(
+        self,
+        data: dict,  # mutable-ok: the request body the adapter reads
+        *,
+        keep_cache_control: bool = False,
+    ) -> ChatCompletionRequest:
         """Translate Anthropic request to OpenAI chat completion format."""
         (
             chat_completion_compatible_request,
             _tool_name_mapping,
-        ) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
+        ) = LiteLLMAnthropicMessagesAdapter(keep_cache_control=keep_cache_control).translate_anthropic_to_openai(
             anthropic_message_request=cast(AnthropicMessagesRequest, scannable_tool_result_request(data)),
             preserve_midturn_system=True,
         )
@@ -813,9 +818,13 @@ class AnthropicMessagesHandler(BaseTranslation):
         data: dict,
         guardrail_to_apply: "CustomGuardrail",
         litellm_logging_obj: "LiteLLMLoggingObj | None" = None,
+        *,
+        request_data: dict[str, object] | None = None,  # mutable-ok: the request dict guardrails receive
+        keep_cache_control: bool = False,
     ) -> Mapping[str, object]:
         """
-        Process input messages by applying guardrails to text content.
+        Process input messages by applying guardrails to text content. ``request_data`` is what the guardrail sees as
+        the request when ``data`` is only its body, e.g. a provider pass-through route that nests the body
         """
         messages: Final = data.get("messages")
         if messages is None:
@@ -827,8 +836,17 @@ class AnthropicMessagesHandler(BaseTranslation):
 
         # The top-level prompt is translated on its own below so it can be hoisted in front of
         # any mid-turn system entries and scanned first, aligned with that structured position.
-        translation_source: Final = {key: value for key, value in data.items() if key != "system"}
-        chat_completion_compatible_request: Final = self._translate_to_openai(translation_source)
+        body_model: Final = data.get("model")
+        request_model: Final = (request_data or {}).get("model")
+        model: Final[str] = (
+            body_model
+            if isinstance(body_model, str) and body_model
+            else (request_model if isinstance(request_model, str) else "")
+        )
+        translation_source: Final = {**{key: value for key, value in data.items() if key != "system"}, "model": model}
+        chat_completion_compatible_request: Final = self._translate_to_openai(
+            translation_source, keep_cache_control=keep_cache_control
+        )
 
         full_structured_messages: Final = cast(
             list[AllMessageValues],
@@ -837,7 +855,9 @@ class AnthropicMessagesHandler(BaseTranslation):
         has_midturn_system_message: Final = any(
             str(message.get("role") or "").lower() == "system" for message in full_structured_messages
         )
-        hoisted_system_message: Final = None if skip_system else self._hoisted_top_level_system_message(data)
+        hoisted_system_message: Final = (
+            None if skip_system else self._hoisted_top_level_system_message(data, model, keep_cache_control)
+        )
         if hoisted_system_message is not None:
             full_structured_messages.insert(0, hoisted_system_message)
         # skip_system already excluded the trusted top-level prompt (it is simply not hoisted);
@@ -899,7 +919,6 @@ class AnthropicMessagesHandler(BaseTranslation):
             if structured_messages:
                 inputs["structured_messages"] = structured_messages
             # Include model information if available
-            model: Final = data.get("model")
             if model:
                 inputs["model"] = model
             original_tool_names: Final = _original_tool_names(
@@ -907,7 +926,7 @@ class AnthropicMessagesHandler(BaseTranslation):
             )
             guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail(
                 inputs=inputs,
-                request_data=data,
+                request_data=data if request_data is None else request_data,
                 input_type="request",
                 logging_obj=litellm_logging_obj,
             )
@@ -955,6 +974,7 @@ class AnthropicMessagesHandler(BaseTranslation):
                     hoisted_system_message=hoisted_system_message,
                     preserve_system_messages=has_midturn_system_message,
                     original_tool_names=original_tool_names.used,
+                    model=model,
                 )
             else:
                 if guardrailed_texts and len(guardrailed_texts) != len(scanned):
@@ -977,17 +997,20 @@ class AnthropicMessagesHandler(BaseTranslation):
 
         return data
 
-    def _hoisted_top_level_system_message(self, data: Mapping[str, object]) -> AllMessageValues | None:
+    def _hoisted_top_level_system_message(
+        self, data: Mapping[str, object], model: str, keep_cache_control: bool
+    ) -> AllMessageValues | None:
         """Return the system message produced by translating the top-level prompt."""
         system: Final = data.get("system")
         if not system:
             return None
         probe: Final = self._translate_to_openai(
             {
-                "model": data.get("model") or "",
+                "model": model,
                 "messages": [],
                 "system": system,
-            }
+            },
+            keep_cache_control=keep_cache_control,
         )
         hoisted: Final = probe.get("messages") or []
         return hoisted[0] if hoisted else None
@@ -1121,6 +1144,7 @@ class AnthropicMessagesHandler(BaseTranslation):
         preserve_system_messages: bool = False,
         *,
         original_tool_names: Mapping[str, str],
+        model: str,
     ) -> None:
         """Write a guardrail's structured-message rewrite back without losing corrections."""
         from litellm.litellm_core_utils.prompt_templates.factory import (
@@ -1129,7 +1153,6 @@ class AnthropicMessagesHandler(BaseTranslation):
         )
 
         _is_system: Final = AnthropicMessagesHandler._is_system
-        model: Final = str(data.get("model") or "")
         converted: Final[list] = []  # mutable-ok: API message payload
 
         def _convert_run(run: list) -> None:  # mutable-ok: API message payload

litellm/llms/anthropic/messages/adapters/transformation.py

@@ -380,8 +380,8 @@ _CLAUDE_OVER_REASONING_EFFORT_PROVIDERS: Final = frozenset({LlmProviders.KYMA.va
 
 
 class LiteLLMAnthropicMessagesAdapter:
-    def __init__(self):
-        pass
+    def __init__(self, *, keep_cache_control: bool = False) -> None:
+        self.keep_cache_control: Final = keep_cache_control
 
     ### FOR [BETA] `/v1/messages` endpoint support
 
@@ -437,7 +437,7 @@ class LiteLLMAnthropicMessagesAdapter:
         cache_control: Final = (
             source.get("cache_control") if isinstance(source, dict) else getattr(source, "cache_control", None)
         )
-        if cache_control and model and self.target_consumes_cache_control(model):
+        if cache_control and (self.keep_cache_control or (model and self.target_consumes_cache_control(model))):
             # TypedDict objects support dict operations at runtime
             # Use type ignore consistent with codebase pattern (see anthropic/chat/transformation.py:432)
             if isinstance(target, dict):

litellm/llms/azure/common_utils.py

@@ -1,15 +1,16 @@
 import asyncio
 import hashlib
+import itertools
 import json
 import os
 import re
-from collections.abc import Awaitable, Callable, Mapping
+from collections.abc import Awaitable, Callable, Iterator, Mapping
 from contextlib import closing
 from types import MappingProxyType
-from typing import Final, Literal, NamedTuple, cast
+from typing import Final, Literal, NamedTuple, NoReturn, cast
 
 import httpx
-from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI
+from openai import APIError, AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI
 from pydantic import ConfigDict, TypeAdapter, with_config
 from typing_extensions import ReadOnly, TypedDict
 
@@ -20,7 +21,7 @@ from litellm.constants import DEFAULT_MAX_RETRIES
 from litellm.litellm_core_utils.single_flight import SingleFlight, single_flight_lru_cache
 from litellm.litellm_core_utils.url_utils import ensure_api_route
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
-from litellm.llms.custom_httpx.http_handler import HTTPHandler
+from litellm.llms.custom_httpx.http_handler import HTTPHandler, MaskedHTTPStatusError
 from litellm.llms.openai.common_utils import BaseOpenAILLM, OpenAIAsyncHTTPClient, OpenAIHTTPClient
 from litellm.secret_managers.get_azure_ad_token_provider import (
     get_azure_ad_token_provider,
@@ -52,6 +53,13 @@ class _AsyncAzureClientParams(TypedDict, total=False, extra_items=object):
 
 
 _ASYNC_AZURE_CLIENT_PARAMS_ADAPTER: Final = TypeAdapter(_AsyncAzureClientParams)
+_AZURE_AD_TOKEN_PROVIDER_ADAPTER: Final = TypeAdapter[Callable[[], str | Awaitable[str]] | None](
+    Callable[[], str | Awaitable[str]] | None
+)
+
+
+def _callable_azure_ad_token_provider(provider: object) -> Callable[[], str | Awaitable[str]] | None:
+    return _AZURE_AD_TOKEN_PROVIDER_ADAPTER.validate_python(provider) if callable(provider) else None
 
 
 class _AzureGatewayClientParams(TypedDict, total=False, extra_items=object):
@@ -81,6 +89,160 @@ class AzureOpenAIError(BaseLLMException):
         )
 
 
+class EntraRejectionError(AzureOpenAIError):
+    pass
+
+
+_GUID: Final = re.compile(r"[0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}")
+_AADSTS_REASON: Final = re.compile(r"\bAADSTS(\d+)\b.*?(?:\.(?=[\s'\"\\]|$)|$)", re.MULTILINE)
+_ENTRA_CREDENTIAL_REJECTION_CODES: Final = frozenset(
+    {
+        50012,
+        50013,
+        50027,
+        50034,
+        50053,
+        50055,
+        50057,
+        50059,
+        50076,
+        50079,
+        50126,
+        50128,
+        53003,
+        65001,
+        70001,
+        70002,
+        70011,
+        70021,
+        70025,
+        70052,
+        90002,
+        90094,
+        500011,
+        500014,
+        700016,
+        700023,
+        700024,
+        700025,
+        700027,
+        700211,
+        700213,
+        700223,
+        700238,
+        7000112,
+        7000215,
+        7000218,
+        7000222,
+    }
+)
+
+
+def _entra_rejection_reason(message: str) -> str | None:
+    reason: Final = _AADSTS_REASON.search(message)
+    if reason is None or int(reason.group(1)) not in _ENTRA_CREDENTIAL_REJECTION_CODES:
+        return None
+    return _GUID.sub("<redacted>", reason.group(0))
+
+
+def _entra_rejection(reason: str) -> EntraRejectionError:
+    return EntraRejectionError(status_code=401, message=f"Failed to get Azure AD token: {reason}")
+
+
+def entra_rejection_as_auth_error(error: Exception) -> EntraRejectionError | None:
+    try:
+        from azure.core.exceptions import ClientAuthenticationError
+    except ImportError:
+        return None
+    if not isinstance(error, ClientAuthenticationError):
+        return None
+    status_code: Final = error.status_code
+    if status_code is not None and (status_code == 429 or status_code >= 500):
+        return None
+    reason: Final = _entra_rejection_reason(str(error))
+    return None if reason is None else _entra_rejection(reason)
+
+
+def _raise_entra_rejection_as_auth_error(error: Exception) -> NoReturn:
+    auth_error: Final = entra_rejection_as_auth_error(error)
+    if auth_error is None:
+        raise error
+    raise auth_error from error
+
+
+def entra_token(provider: Callable[[], str]) -> str:
+    try:
+        return provider()
+    except entra_rejection_types() as error:
+        _raise_entra_rejection_as_auth_error(error)
+
+
+def entra_rejection_types() -> tuple[type[Exception], ...]:
+    try:
+        from azure.core.exceptions import ClientAuthenticationError
+    except ImportError:
+        return ()
+    return (ClientAuthenticationError,)
+
+
+_EXCEPTION_CHAIN_MAX_DEPTH: Final = 16
+
+
+def _exception_chain(error: BaseException) -> Iterator[BaseException]:
+    link: BaseException | None = error  # rebind-ok: advances one link of the cause chain per iteration
+    for _ in range(_EXCEPTION_CHAIN_MAX_DEPTH):
+        if link is None:
+            return
+        yield link
+        link = link.__cause__ or link.__context__
+
+
+_REQUEST_FAILURE_TYPES: Final = (APIError, httpx.HTTPError)
+
+
+def _ends_the_failed_attempt(link: BaseException, failed_attempt: BaseException) -> bool:
+    return not isinstance(link, EntraRejectionError) and (
+        isinstance(link, _REQUEST_FAILURE_TYPES) or type(link) is type(failed_attempt)
+    )
+
+
+def _links_within_the_failed_attempt(error: BaseException) -> Iterator[BaseException]:
+    yield error
+    yield from itertools.takewhile(
+        lambda link: not _ends_the_failed_attempt(link, error), itertools.islice(_exception_chain(error), 1, None)
+    )
+
+
+def entra_rejection_in(error: BaseException) -> EntraRejectionError | None:
+    return next(
+        (link for link in _links_within_the_failed_attempt(error) if isinstance(link, EntraRejectionError)), None
+    )
+
+
+def caused_by_entra_rejection(error: BaseException) -> bool:
+    return entra_rejection_in(error) is not None
+
+
+async def _awaited_token(token: Awaitable[str]) -> str:
+    try:
+        return await token
+    except entra_rejection_types() as error:
+        _raise_entra_rejection_as_auth_error(error)
+
+
+def _entra_rejections_as_auth_errors(
+    provider: Callable[[], str | Awaitable[str]],
+) -> Callable[[], str | Awaitable[str]]:
+    def call_provider() -> str | Awaitable[str]:
+        try:
+            token: Final = provider()
+        except entra_rejection_types() as error:
+            _raise_entra_rejection_as_auth_error(error)
+        return token if isinstance(token, str) else _awaited_token(token)
+
+    return call_provider
+
+
 def process_azure_headers(headers: httpx.Headers | dict) -> dict:
     openai_headers: Final = {}
     if "x-ratelimit-limit-requests" in headers:
@@ -91,7 +253,7 @@ def process_azure_headers(headers: httpx.Headers | dict) -> dict:
         openai_headers["x-ratelimit-limit-tokens"] = headers["x-ratelimit-limit-tokens"]
     if "x-ratelimit-remaining-tokens" in headers:
         openai_headers["x-ratelimit-remaining-tokens"] = headers["x-ratelimit-remaining-tokens"]
-    llm_response_headers: Final = {"{}-{}".format("llm_provider", k): v for k, v in headers.items()}
+    llm_response_headers: Final = {f"llm_provider-{k}": v for k, v in headers.items()}
 
     return {**llm_response_headers, **openai_headers}
 
@@ -257,22 +419,34 @@ def _cached_oidc_token(cache_key: str) -> str | None:
     return cached if isinstance(cached, str) else None
 
 
+def _oidc_exchange_rejection_reason(error: MaskedHTTPStatusError) -> str | None:
+    if error.response.status_code not in (400, 401):
+        return None
+    return _entra_rejection_reason(error.text or error.response.text)
+
+
 def _exchange_oidc_token(cache_key: str, token_url: str, client_id: str, scope: str, oidc_token: str) -> str:
     cached_access_token: Final = _cached_oidc_token(cache_key)
     if cached_access_token is not None:
         return cached_access_token
 
     with closing(HTTPHandler(timeout=litellm.request_timeout)) as client:
-        req_token: Final = client.post(
-            token_url,
-            data={
-                "client_id": client_id,
-                "grant_type": "client_credentials",
-                "scope": scope,
-                "client_assertion_type": "urn:ietf:params:oauth:client-assertion-type:jwt-bearer",
-                "client_assertion": oidc_token,
-            },
-        )
+        try:
+            req_token: Final = client.post(
+                token_url,
+                data={
+                    "client_id": client_id,
+                    "grant_type": "client_credentials",
+                    "scope": scope,
+                    "client_assertion_type": "urn:ietf:params:oauth:client-assertion-type:jwt-bearer",
+                    "client_assertion": oidc_token,
+                },
+            )
+        except MaskedHTTPStatusError as error:
+            reason: Final = _oidc_exchange_rejection_reason(error)
+            if reason is None:
+                raise
+            raise _entra_rejection(reason) from error
 
     if req_token.status_code != 200:
         raise AzureOpenAIError(
@@ -483,6 +657,9 @@ def get_azure_ad_token(
             raise
         except Exception as e:
             verbose_logger.error("Error calling Azure AD token provider: %s", e)
+            auth_error: Final = entra_rejection_as_auth_error(e)
+            if auth_error is not None:
+                raise auth_error from e
             raise RuntimeError(f"Failed to get Azure AD token: {e}") from e
 
     return azure_ad_token
@@ -696,7 +873,7 @@ class BaseAzureLLM(BaseOpenAILLM):
         api_version: str | None,
         is_async: bool,
     ) -> dict:
-        azure_ad_token_provider = litellm_params.get("azure_ad_token_provider")
+        azure_ad_token_provider = _callable_azure_ad_token_provider(litellm_params.get("azure_ad_token_provider"))
         # If we have api_key, then we have higher priority
         azure_ad_token = litellm_params.get("azure_ad_token")
 
@@ -775,7 +952,7 @@ class BaseAzureLLM(BaseOpenAILLM):
             azure_client_params["timeout"] = timeout
 
         if azure_ad_token_provider is not None:
-            azure_client_params["azure_ad_token_provider"] = azure_ad_token_provider
+            azure_client_params["azure_ad_token_provider"] = _entra_rejections_as_auth_errors(azure_ad_token_provider)
         # this decides if we should set azure_endpoint or base_url on Azure OpenAI Client
         # required to support GPT-4 vision enhancements, since base_url needs to be set on Azure OpenAI Client
 

litellm/llms/azure/decisions/transformation.py

@@ -8,6 +8,7 @@ import litellm
 from litellm.llms.azure.common_utils import (
     BaseAzureLLM,
     cached_azure_ad_token_refresh_provider,
+    entra_token,
 )
 from litellm.llms.openai.decisions.transformation import OpenAIDecisionsConfig
 from litellm.secret_managers.main import get_secret_str
@@ -61,7 +62,7 @@ class AzureDecisionsConfig(OpenAIDecisionsConfig):
         scope: Final = (
             params.azure_scope or get_secret_str("AZURE_SCOPE") or "https://cognitiveservices.azure.com/.default"
         )
-        azure_headers["Authorization"] = f"Bearer {cached_azure_ad_token_refresh_provider(scope)()}"
+        azure_headers["Authorization"] = f"Bearer {entra_token(cached_azure_ad_token_refresh_provider(scope))}"
         return azure_headers
 
     async def avalidate_environment(

litellm/llms/azure/responses/transformation.py

@@ -3,11 +3,10 @@ from collections.abc import Callable, Mapping
 from copy import deepcopy
 from functools import partial
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, Literal
+from typing import TYPE_CHECKING, Any, Final, Literal, cast
 from urllib.parse import urlsplit
 
 import httpx
-from openai.types.responses import ResponseReasoningItem
 from pydantic import TypeAdapter
 
 from litellm._logging import verbose_logger
@@ -16,6 +15,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import prompt_cach
 from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config
 from litellm.llms.azure.common_utils import BaseAzureLLM
+from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
 from litellm.llms.base_llm.responses.transformation import SUMMARY_ONLY_REASONING_FIELD_VALUES, ReasoningFieldValues
 from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
 from litellm.types.llms.openai import *
@@ -72,6 +72,21 @@ def _with_responses_api_version(api_base: str | None, litellm_params: Mapping[st
     return MappingProxyType({**litellm_params, "api_version": _resolve_responses_api_version(api_base, litellm_params)})
 
 
+_AZURE_REJECTED_INPUT_FIELDS: Final = MappingProxyType(
+    {"message": frozenset({"status"}), "reasoning": frozenset({"status"})}
+)
+_AZURE_REJECTED_INPUT_FIELDS_UNDER_DROP_PARAMS: Final = MappingProxyType(
+    {**_AZURE_REJECTED_INPUT_FIELDS, "reasoning": frozenset({"status", "content"})}
+)
+
+
+def _without_fields_azure_rejects(item: object, rejected_fields: Mapping[str, frozenset[str]]) -> object:
+    if not is_str_keyed_mapping(item):
+        return item
+    rejected: Final = rejected_fields.get(str(item.get("type")), frozenset())
+    return {key: value for key, value in item.items() if key not in rejected} if rejected else item
+
+
 class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
     # Parameters not supported by Azure Responses API
     AZURE_UNSUPPORTED_PARAMS = ["context_management"]
@@ -122,68 +137,17 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
     def get_stripped_model_name(self, model: str) -> str:
         return model.replace("responses/", "").replace("o_series/", "").replace("azure_ai/", "")
 
-    def _handle_reasoning_item(self, item: dict[str, Any]) -> Mapping[str, object]:
-        """
-        Handle reasoning items to filter out the status field.
-        Issue: https://github.com/BerriAI/litellm/issues/13484
-
-        Azure OpenAI API does not accept 'status' field in reasoning input items.
-        """
-        if item.get("type") == "reasoning":
-            try:
-                # Ensure required fields are present for ResponseReasoningItem
-                item_data: Final = dict(item)
-                if "summary" not in item_data:
-                    item_data["summary"] = (
-                        item_data.get("reasoning_content", "")[:100] + "..."
-                        if len(item_data.get("reasoning_content", "")) > 100
-                        else item_data.get("reasoning_content", "")
-                    )
-
-                # Create ResponseReasoningItem object from the item data
-                reasoning_item: Final = ResponseReasoningItem.model_validate(item_data)
-
-                # Convert back to dict with exclude_none=True to exclude None fields
-                dict_reasoning_item: Final = reasoning_item.model_dump(exclude_none=True)
-                dict_reasoning_item.pop("status", None)
-
-                return dict_reasoning_item
-            except Exception as e:
-                verbose_logger.debug("Failed to create ResponseReasoningItem, falling back to manual filtering: %s", e)
-                # Fallback: manually filter out known None fields
-                filtered_item: Final = {
-                    k: v
-                    for k, v in item.items()
-                    if v is not None or k not in {"status", "content", "encrypted_content"}
-                }
-                return filtered_item
-        return item
-
     def _validate_input_param(
-        self, input: str | ResponseInputParam, *, drops_set_created_by: bool = False
+        self, input: str | ResponseInputParam, *, drop_params: bool = False
     ) -> str | ResponseInputParam:
-        """
-        Override parent method to also filter out 'status' field from message items.
-        Azure OpenAI API does not accept 'status' field in input messages.
-        """
-        from typing import cast
-
-        # First call parent's validation
-        validated_input: Final = super()._validate_input_param(input, drops_set_created_by=drops_set_created_by)
-
-        # Then filter out status from message items
-        if isinstance(validated_input, list):
-            filtered_input: Final[list[object]] = []
-            for item in validated_input:
-                if isinstance(item, dict) and item.get("type") == "message":
-                    # Filter out status field from message items
-                    filtered_item = {k: v for k, v in item.items() if k != "status"}
-                    filtered_input.append(filtered_item)
-                else:
-                    filtered_input.append(item)
-            return cast(ResponseInputParam, filtered_input)
-
-        return validated_input
+        validated_input: Final = super()._validate_input_param(input, drop_params=drop_params)
+        if not isinstance(validated_input, list):
+            return validated_input
+        rejected_fields: Final = (
+            _AZURE_REJECTED_INPUT_FIELDS_UNDER_DROP_PARAMS if drop_params else _AZURE_REJECTED_INPUT_FIELDS
+        )
+        sanitized_items: Final = [_without_fields_azure_rejects(item, rejected_fields) for item in validated_input]
+        return cast("ResponseInputParam", sanitized_items)  # cast-ok: items keep their shape minus rejected fields
 
     def transform_responses_api_request(
         self,

litellm/llms/azure/search/transformation.py

@@ -36,7 +36,7 @@ import httpx
 from pydantic import ConfigDict, ValidationError
 
 from litellm.litellm_core_utils.url_utils import ensure_api_route
-from litellm.llms.azure.common_utils import cached_azure_ad_token_refresh_provider
+from litellm.llms.azure.common_utils import cached_azure_ad_token_refresh_provider, entra_rejection_as_auth_error
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.base_llm.search.responses import (
     ResponsesSearchAnnotation,
@@ -270,6 +270,9 @@ class BingGroundingSearchConfig(BaseSearchConfig):
         try:
             return minter()
         except Exception as e:
+            rejection: Final = entra_rejection_as_auth_error(e)
+            if rejection is not None:
+                raise rejection from e
             raise ValueError(
                 f"Grounding with Bing Search: no credential available. Pass api_key, set {TOKEN_ENV} "
                 f"to an Entra bearer token, or configure azure-identity (AZURE_CLIENT_ID / "

litellm/llms/azure_ai/anthropic/count_tokens/handler.py

@@ -16,6 +16,7 @@ from litellm.anthropic_beta_headers_manager import aload_beta_headers_config
 from litellm.litellm_core_utils.asyncify import asyncify
 from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
 from litellm.llms.anthropic.common_utils import AnthropicError
+from litellm.llms.azure.common_utils import EntraRejectionError
 from litellm.llms.azure_ai.anthropic.count_tokens.transformation import (
     AzureAIAnthropicCountTokensConfig,
 )
@@ -135,6 +136,8 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
         except AnthropicError:
             # Re-raise Anthropic exceptions as-is
             raise
+        except EntraRejectionError as rejection:
+            raise AnthropicError(status_code=rejection.status_code, message=rejection.message) from rejection
         except httpx.HTTPStatusError as e:
             # HTTP errors - preserve the actual status code
             verbose_logger.error("HTTP error in CountTokens handler: %s", e)

litellm/llms/azure_ai/common_utils.py

@@ -1,5 +1,7 @@
 import asyncio
-from collections.abc import Mapping
+import os
+from collections.abc import Callable, Mapping
+from functools import partial
 from types import MappingProxyType
 from typing import Final, Literal
 from urllib.parse import urlparse
@@ -52,6 +54,28 @@ AZURE_ENTRA_CREDENTIAL_PARAM_KEYS: Final = frozenset({"azure_ad_token", "client_
 AZURE_ENTRA_LITELLM_PARAM_KEYS: Final = AZURE_ENTRA_CREDENTIAL_PARAM_KEYS | frozenset(
     {"tenant_id", "client_id", "azure_username", "azure_scope"}
 )
+AZURE_DEFAULT_SCOPE: Final = "https://cognitiveservices.azure.com/.default"
+_AZURE_ENTRA_ENVIRONMENT_VARIABLES: Final = MappingProxyType(
+    {
+        "tenant_id": "AZURE_TENANT_ID",
+        "client_id": "AZURE_CLIENT_ID",
+        "client_secret": "AZURE_CLIENT_SECRET",
+        "azure_username": "AZURE_USERNAME",
+        "azure_password": "AZURE_PASSWORD",
+        "azure_ad_token": "AZURE_AD_TOKEN",
+    }
+)
+
+
+def azure_entra_environment_defaults() -> Mapping[str, object]:
+    return MappingProxyType(
+        {
+            **{key: os.getenv(variable) for key, variable in _AZURE_ENTRA_ENVIRONMENT_VARIABLES.items()},
+            "azure_scope": os.getenv("AZURE_SCOPE", AZURE_DEFAULT_SCOPE),
+        }
+    )
+
+
 AZURE_ENTRA_CREDENTIAL_HELP: Final = (
     "Set `tenant_id` + `client_id` + `client_secret`, `azure_ad_token` (an `oidc/` token also needs "
     "`tenant_id` + `client_id`), or `client_id` + `azure_username` + `azure_password` in the agent's `litellm_params`"
@@ -70,6 +94,17 @@ def _resolve_config_secret(value: object) -> str | None:
     return get_secret_str(value) if value.startswith("os.environ/") else value
 
 
+def _agent_entra_token(provider: Callable[[], str], model: str) -> str:
+    from litellm.llms.azure.common_utils import EntraRejectionError, entra_token
+
+    try:
+        return entra_token(provider)
+    except EntraRejectionError as rejection:
+        raise litellm.AuthenticationError(
+            message=rejection.message, llm_provider="azure_ai", model=model
+        ) from rejection
+
+
 def get_azure_ai_agent_entra_token(litellm_params: Mapping[str, object]) -> str:
     """Mints the Entra bearer from the agent's own litellm_params, never from process-wide AZURE_* env vars."""
     from litellm.llms.azure.common_utils import (
@@ -88,18 +123,32 @@ def get_azure_ai_agent_entra_token(litellm_params: Mapping[str, object]) -> str:
     azure_username: Final = resolved["azure_username"]
     azure_password: Final = resolved["azure_password"]
     azure_ad_token: Final = resolved["azure_ad_token"]
+    model: Final = str(litellm_params.get("model") or "")
     if tenant_id and client_id and client_secret:
-        return get_azure_ad_token_from_entra_id(
-            tenant_id=tenant_id, client_id=client_id, client_secret=client_secret, scope=scope
-        )()
+        return _agent_entra_token(
+            get_azure_ad_token_from_entra_id(
+                tenant_id=tenant_id, client_id=client_id, client_secret=client_secret, scope=scope
+            ),
+            model,
+        )
     if client_id and azure_username and azure_password:
-        return get_azure_ad_token_from_username_password(
-            client_id=client_id, azure_username=azure_username, azure_password=azure_password, scope=scope
-        )()
+        return _agent_entra_token(
+            get_azure_ad_token_from_username_password(
+                client_id=client_id, azure_username=azure_username, azure_password=azure_password, scope=scope
+            ),
+            model,
+        )
     federated: Final = azure_ad_token is not None and azure_ad_token.startswith("oidc/")
     if azure_ad_token and federated and tenant_id and client_id:
-        return get_azure_ad_token_from_oidc(
-            azure_ad_token=azure_ad_token, azure_client_id=client_id, azure_tenant_id=tenant_id, scope=scope
+        return _agent_entra_token(
+            partial(
+                get_azure_ad_token_from_oidc,
+                azure_ad_token=azure_ad_token,
+                azure_client_id=client_id,
+                azure_tenant_id=tenant_id,
+                scope=scope,
+            ),
+            model,
         )
     if azure_ad_token and not federated:
         return azure_ad_token

litellm/llms/bedrock/passthrough/guardrail_translation/handler.py

@@ -14,6 +14,7 @@ from pydantic import InstanceOf, TypeAdapter, ValidationError
 from typing_extensions import ReadOnly, TypedDict
 
 from litellm._logging import verbose_proxy_logger
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
 from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicMessagesHandler
 from litellm.llms.base_llm.guardrail_translation.base_translation import (
     BaseTranslation,
@@ -60,6 +61,8 @@ if TYPE_CHECKING:
 
 _CONVERSE_ACTIONS: Final = frozenset({"converse", "converse-stream"})
 _INVOKE_STREAM_ACTION: Final = "invoke-with-response-stream"
+_INVOKE_ACTIONS: Final = frozenset({"invoke", _INVOKE_STREAM_ACTION})
+_BLOCK_TYPES_THE_ANTHROPIC_HANDLER_DOES_NOT_SCAN: Final = frozenset({"document", "search_result"})
 _EVENT_STREAM_CONTENT_TYPE: Final = "vnd.amazon.eventstream"
 _EVENT_STREAM_MEDIA_TYPE: Final = "application/vnd.amazon.eventstream"
 
@@ -73,6 +76,83 @@ def _is_invoke_stream_endpoint(endpoint: str) -> bool:
     return endpoint.rstrip("/").split("/")[-1] == _INVOKE_STREAM_ACTION
 
 
+def _anthropic_invoke_body(
+    data: Mapping[str, object],
+) -> dict[str, object] | None:  # mutable-ok: the body guardrails write back into
+    """The body of an InvokeModel call in Anthropic Messages format, which Bedrock requires to carry anthropic_version"""
+    endpoint: Final = data.get("endpoint")
+    body: Final = data.get("data")
+    if not isinstance(endpoint, str) or endpoint.rstrip("/").split("/")[-1] not in _INVOKE_ACTIONS:
+        return None
+    if not isinstance(body, dict):
+        return None
+    invoke_body: Final = _BODY_BY_REFERENCE.validate_python(body)
+    if not isinstance(invoke_body.get("anthropic_version"), str) or not isinstance(invoke_body.get("messages"), list):
+        return None
+    messages: Final = _ARRAY_BY_REFERENCE.validate_python(invoke_body["messages"])
+    if not all(_is_well_formed_message(message) for message in messages):
+        return None
+    blocks: Final = _anthropic_message_blocks(messages)
+    if any(block.get("type") in _BLOCK_TYPES_THE_ANTHROPIC_HANDLER_DOES_NOT_SCAN for block in blocks):
+        return None
+    return invoke_body
+
+
+def _body_level_request_data(
+    passthrough_request: Mapping[str, object],
+) -> dict[str, object]:  # mutable-ok: the request dict guardrails receive
+    """What `/v1/messages` hands a guardrail as `request_data`: the Anthropic body at the top level, with the proxy's own keys (model, metadata, headers) on top"""
+    body: Final = as_str_mapping(passthrough_request.get("data")) or {}
+    proxy_keys: Final = {key: value for key, value in passthrough_request.items() if key != "data"}
+    return {**body, **proxy_keys}
+
+
+def _is_well_formed_message(message: object) -> bool:
+    """Whether the Messages translation can read `message`, so a malformed body still reaches Bedrock to be rejected there"""
+    fields: Final = as_str_mapping(message)
+    if fields is None or not isinstance(fields.get("role"), str):
+        return False
+    content: Final = fields.get("content")
+    if isinstance(content, str):
+        return True
+    return isinstance(content, list) and all(
+        as_str_mapping(block) is not None for block in _ARRAY_BY_REFERENCE.validate_python(content)
+    )
+
+
+def _anthropic_message_blocks(messages: Sequence[object]) -> tuple[Mapping[str, object], ...]:
+    fields: Final = (as_str_mapping(message) for message in messages)
+    return tuple(chain.from_iterable(_anthropic_content_blocks(field.get("content")) for field in fields if field))
+
+
+def _anthropic_content_blocks(content: object) -> tuple[Mapping[str, object], ...]:
+    blocks: Final[list[Mapping[str, object]]] = []  # mutable-ok: collected while walking the content
+    pending: Final[list[object]] = [content]  # mutable-ok: content lists still to walk, nested tool results included
+    while pending:
+        current = pending.pop()
+        if not isinstance(current, list):
+            continue
+        for item in _ARRAY_BY_REFERENCE.validate_python(current):
+            block = as_str_mapping(item)
+            if block is None:
+                continue
+            blocks.append(block)
+            if block.get("type") == "tool_result":
+                pending.append(block.get("content"))
+    return tuple(blocks)
+
+
+def _anthropic_invoke_reply(
+    response: object,
+    request_data: object,
+) -> dict[str, object] | None:  # mutable-ok: the reply guardrails write back into
+    if not isinstance(response, dict) or not isinstance(request_data, dict):
+        return None
+    if _anthropic_invoke_body(_OBJECT_BY_REFERENCE.validate_python(request_data)) is None:
+        return None
+    return _BODY_BY_REFERENCE.validate_python(response)
+
+
 def _generic_passthrough_handler() -> "PassThroughEndpointHandler":
     """
     Fallback for non-Converse Bedrock routes (e.g. invoke). The generic
@@ -1285,6 +1365,17 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
         body: Final = data.get("data")
 
         if not _is_converse_endpoint(endpoint):
+            passthrough_request: Final = _BODY_BY_REFERENCE.validate_python(data)
+            anthropic_body: Final = _anthropic_invoke_body(passthrough_request)
+            if anthropic_body is not None:
+                await AnthropicMessagesHandler().process_input_messages(  # pyright: ignore[reportUnknownMemberType]  # takes the untyped request dict
+                    data=anthropic_body,
+                    guardrail_to_apply=guardrail_to_apply,
+                    litellm_logging_obj=litellm_logging_obj,
+                    request_data=_body_level_request_data(passthrough_request),
+                    keep_cache_control=True,
+                )
+                return passthrough_request
             return await _generic_passthrough_handler().process_input_messages(
                 data=data,
                 guardrail_to_apply=guardrail_to_apply,
@@ -1410,6 +1501,15 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
             and not _is_converse_endpoint(endpoint)
             and not _is_invoke_stream_endpoint(endpoint)
         ):
+            anthropic_reply: Final = _anthropic_invoke_reply(response, request_data)  # pyright: ignore[reportUnknownArgumentType]  # the base signature types request_data as a bare dict
+            if anthropic_reply is not None:
+                return await AnthropicMessagesHandler().process_output_response(  # pyright: ignore[reportUnknownMemberType]  # takes the untyped request dict
+                    response=anthropic_reply,  # pyright: ignore[reportArgumentType]  # the Anthropic reply dict, written back in place
+                    guardrail_to_apply=guardrail_to_apply,
+                    litellm_logging_obj=litellm_logging_obj,
+                    user_api_key_dict=user_api_key_dict,
+                    request_data=_body_level_request_data(_OBJECT_BY_REFERENCE.validate_python(request_data or {})),
+                )
             return await _generic_passthrough_handler().process_output_response(
                 response=response,
                 guardrail_to_apply=guardrail_to_apply,

litellm/llms/empiriolabs/decisions/transformation.py

@@ -0,0 +1,10 @@
+from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
+
+
+class EmpirioLabsDecisionsConfig(BaseDecisionsConfig):
+    path = "/v1/decisions"
+    api_key_env = ("EMPIRIOLABS_API_KEY",)
+    api_base_env = ("EMPIRIOLABS_API_BASE",)
+
+    def get_default_api_base(self) -> str | None:
+        return "https://api.empiriolabs.ai"

litellm/llms/github_copilot/github_host.py

@@ -21,8 +21,10 @@ class GithubHostEndpoints:
     device_code_url: str
     access_token_url: str
     api_key_url: str
+    copilot_user_url: str
     user_url: str
     client_id: str
+    exchanges_copilot_token: bool
     copilot_api_hosts: frozenset[str]
     copilot_api_host_suffix: str | None
     default_copilot_api_base: str
@@ -49,14 +51,17 @@ def github_host_endpoints(host: str) -> GithubHostEndpoints:
     if normalized is None:
         raise ValueError("host must be github.com or a single-subdomain GHE.com hostname")
     client_id: Final = os.getenv("GITHUB_COPILOT_CLIENT_ID", DEFAULT_GITHUB_CLIENT_ID)
+    exchanges_copilot_token: Final = client_id == DEFAULT_GITHUB_CLIENT_ID
     if normalized == GITHUB_COPILOT_DEFAULT_HOST:
         return GithubHostEndpoints(
             host=normalized,
             device_code_url=os.getenv("GITHUB_COPILOT_DEVICE_CODE_URL", DEFAULT_GITHUB_DEVICE_CODE_URL),
             access_token_url=os.getenv("GITHUB_COPILOT_ACCESS_TOKEN_URL", DEFAULT_GITHUB_ACCESS_TOKEN_URL),
             api_key_url=os.getenv("GITHUB_COPILOT_API_KEY_URL", DEFAULT_GITHUB_API_KEY_URL),
+            copilot_user_url="https://api.github.com/copilot_internal/user",
             user_url="https://api.github.com/user",
             client_id=client_id,
+            exchanges_copilot_token=exchanges_copilot_token,
             copilot_api_hosts=frozenset({"githubcopilot.com"}),
             copilot_api_host_suffix=".githubcopilot.com",
             default_copilot_api_base="https://api.githubcopilot.com",
@@ -70,8 +75,10 @@ def github_host_endpoints(host: str) -> GithubHostEndpoints:
         device_code_url=f"https://{normalized}/login/device/code",
         access_token_url=f"https://{normalized}/login/oauth/access_token",
         api_key_url=f"https://{api_host}/copilot_internal/v2/token",
+        copilot_user_url=f"https://{api_host}/copilot_internal/user",
         user_url=f"https://{api_host}/user",
         client_id=client_id,
+        exchanges_copilot_token=exchanges_copilot_token,
         copilot_api_hosts=frozenset({copilot_api_host}),
         copilot_api_host_suffix=None,
         default_copilot_api_base=f"https://{copilot_api_host}",

litellm/llms/github_copilot/per_user_auth.py

@@ -46,6 +46,7 @@ _LLM_PROVIDER: Final = "github_copilot"
 _DEVICE_FLOW_SCOPE: Final = "read:user"
 _SESSION_CACHE_MAX_SIZE: Final = 1000
 _SYNC_LOCK_COUNT: Final = 64
+_ACCOUNT_LOOKUP_TTL_SECONDS: Final = 1800
 
 
 class _SessionCache(Protocol):
@@ -125,12 +126,20 @@ class _CopilotEndpointsPayload(TypedDict):
     api: ReadOnly[NotRequired[object]]
 
 
+@with_config(ConfigDict(extra="allow", strict=True))
+class _CopilotAccountPayload(TypedDict):
+    access_type_sku: ReadOnly[NotRequired[object]]
+    chat_enabled: ReadOnly[NotRequired[object]]
+    endpoints: ReadOnly[NotRequired[object]]
+
+
 @with_config(ConfigDict(extra="allow", strict=True))
 class _GithubUserPayload(TypedDict):
     login: ReadOnly[str]
 
 
 _COPILOT_TOKEN: Final = TypeAdapter(_CopilotTokenPayload)
+_COPILOT_ACCOUNT: Final = TypeAdapter(_CopilotAccountPayload)
 _DEVICE_CODE: Final = TypeAdapter(_DeviceCodePayload)
 _ACCESS_TOKEN_POLL: Final = TypeAdapter(_AccessTokenPollPayload)
 _GITHUB_USER: Final = TypeAdapter(_GithubUserPayload)
@@ -273,6 +282,8 @@ def _session_from_response(
             llm_provider=_LLM_PROVIDER,
             model="",
         )
+    if not endpoints.exchanges_copilot_token:
+        return _session_from_account(response, github_token, credential_name, cache_key, endpoints)
     try:
         payload: Final = _COPILOT_TOKEN.validate_python(response.json())
     except Exception as e:
@@ -282,13 +293,7 @@ def _session_from_response(
             model="",
         ) from e
 
-    payload_endpoints: Final[object] = payload.get("endpoints")
-    endpoints_api: Final[object] = (
-        _COPILOT_ENDPOINTS.validate_python(payload_endpoints).get("api")
-        if isinstance(payload_endpoints, Mapping)
-        else None
-    )
-    api_base: Final = validated_copilot_api_base(endpoints_api, endpoints)
+    api_base: Final = validated_copilot_api_base(_endpoints_api(payload.get("endpoints")), endpoints)
     session: Final = GithubCopilotUserSession(token=payload["token"], api_base=api_base)
     expires_at: Final = _expires_at_seconds(payload.get("expires_at"))
     if expires_at > 0:
@@ -296,6 +301,41 @@ def _session_from_response(
     return session
 
 
+def _session_from_account(
+    response: httpx.Response,
+    github_token: str,
+    credential_name: str,
+    cache_key: str,
+    endpoints: GithubHostEndpoints,
+) -> GithubCopilotUserSession:
+    try:
+        payload: Final = _COPILOT_ACCOUNT.validate_python(response.json())
+    except Exception as e:
+        raise ServiceUnavailableError(
+            message="GitHub Copilot account lookup returned an invalid response",
+            llm_provider=_LLM_PROVIDER,
+            model="",
+        ) from e
+    if payload.get("chat_enabled") is False or payload.get("access_type_sku") == "no_access":
+        _SESSION_CACHE.delete_cache(cache_key)
+        raise _reject_connection(credential_name)
+
+    api_base: Final = validated_copilot_api_base(_endpoints_api(payload.get("endpoints")), endpoints)
+    session: Final = GithubCopilotUserSession(token=github_token, api_base=api_base)
+    _cache_session(cache_key, github_token, session, time.time() + _ACCOUNT_LOOKUP_TTL_SECONDS)
+    return session
+
+
+def _endpoints_api(payload_endpoints: object) -> object:
+    if not isinstance(payload_endpoints, Mapping):
+        return None
+    return _COPILOT_ENDPOINTS.validate_python(payload_endpoints).get("api")
+
+
+def _session_lookup_url(endpoints: GithubHostEndpoints) -> str:
+    return endpoints.api_key_url if endpoints.exchanges_copilot_token else endpoints.copilot_user_url
+
+
 def _to_int(raw: object) -> int:
     if isinstance(raw, bool):
         return 0
@@ -337,7 +377,7 @@ def _do_exchange(
             _SyncGetClient, litellm.module_level_client
         )
         response: Final = client.get(
-            endpoints.api_key_url,
+            _session_lookup_url(endpoints),
             headers=github_api_headers(github_token),
         )
     except Exception as e:
@@ -367,7 +407,7 @@ async def _do_aexchange(
     client: Final = _new_async_github_client()
     try:
         response: Final = await client.get(
-            endpoints.api_key_url,
+            _session_lookup_url(endpoints),
             headers=github_api_headers(github_token),
         )
     except Exception as e:

litellm/llms/nvidia_riva/audio_transcription/handler.py

@@ -13,24 +13,26 @@ The handler is intentionally a thin orchestration layer:
 2. Build ``RecognitionConfig`` / ``StreamingRecognitionConfig`` protobufs
    from the structured dict produced by
    :class:`NvidiaRivaAudioTranscriptionConfig`.
-3. Construct ``riva.client.Auth`` honoring NVCF (function-id metadata + TLS)
-   vs self-hosted (any host:port, optional TLS) modes.
-4. Stream the audio through Riva's ``streaming_response_generator`` and
-   aggregate ``is_final`` results into a single transcript, or, when
-   ``riva_offline`` is set, send the whole buffer through the unary
-   ``offline_recognize`` RPC for NIMs that only ship offline engines.
+3. Open a gRPC channel honoring NVCF (TLS by default) vs self-hosted (any
+   host:port, optional TLS) modes, and attach the ``authorization`` and NVCF
+   ``function-id`` metadata to each call exactly once.
+4. Stream the audio through Riva's ``StreamingRecognize`` RPC and aggregate
+   ``is_final`` results into a single transcript, or, when ``riva_offline``
+   is set, send the whole buffer through the unary ``Recognize`` RPC for NIMs
+   that only ship offline engines.
 5. Return a normalized ``TranscriptionResponse`` with ``duration`` exposed
    on ``_hidden_params`` so cost calculation works.
 
-``riva-client`` is imported lazily so ``litellm`` core remains usable
-without the optional STT extras installed.
+``grpcio`` and ``protobuf`` are imported lazily so ``litellm`` core remains
+usable without the optional STT extras installed.
 """
 
+from __future__ import annotations
+
 import asyncio
-import inspect
-from collections.abc import Callable, Iterable, Mapping, Sequence
-from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, Protocol
+from collections.abc import Iterable, Sequence
+from contextlib import closing
+from typing import TYPE_CHECKING, Final
 
 import httpx
 
@@ -64,89 +66,19 @@ if TYPE_CHECKING:
         Logging as LiteLLMLoggingObj,
     )
 
-# Stream audio to Riva in ~50 ms slices (1600 samples at 16 kHz). Matches
-# NVIDIA's recommended chunk size for streaming ASR — small enough for
-# responsive endpointing, large enough to keep per-RPC overhead low.
-_DEFAULT_CHUNK_SAMPLES: Final = 1600
-_DEFAULT_CHUNK_BYTES: Final = _DEFAULT_CHUNK_SAMPLES * 2  # int16 = 2 bytes/sample
-
-
 _RIVA_INSTALL_HINT = "NVIDIA Riva client is not installed. Install with `pip install 'litellm[stt-nvidia-riva]'`."
 
 
-class _RivaAuth(Protocol):
-    """Opaque ``riva.client.Auth`` handle."""
-
-
-class _RecognizeFuture(Protocol):
-    def result(self, timeout: float | None = None) -> object: ...
-
-    def done(self) -> bool: ...
-
-    def cancel(self) -> bool: ...
-
-
-class _AsrService(Protocol):
-    @property
-    def streaming_response_generator(self) -> Callable[..., Iterable[object]]: ...
-
-    def offline_recognize(self, audio_bytes: bytes, config: object, future: bool) -> _RecognizeFuture: ...
-
-
-class _EndpointingConfig(Protocol):
-    """Opaque ``EndpointingConfig`` protobuf message."""
-
-
-class _EndpointingConfigField(Protocol):
-    CopyFrom: Callable[[_EndpointingConfig], None]
-
-
-class _RecognitionConfig(Protocol):
-    @property
-    def endpointing_config(self) -> _EndpointingConfigField: ...
-
-
-class _StreamingRecognitionConfig(Protocol):
-    """Opaque ``StreamingRecognitionConfig`` protobuf message."""
-
-
-class _AudioEncoding(Protocol):
-    @property
-    def LINEAR_PCM(self) -> object: ...
-
-
-class _RivaClientModule(Protocol):
-    """The ``riva.client`` entry points this handler calls."""
-
-    @property
-    def Auth(self) -> Callable[..., _RivaAuth]: ...
-
-    @property
-    def ASRService(self) -> Callable[[_RivaAuth], _AsrService]: ...
-
-
-class _RivaAsrModule(Protocol):
-    """The protobuf constructors this handler calls, from whichever module exposes them."""
-
-    @property
-    def AudioEncoding(self) -> _AudioEncoding: ...
-
-    @property
-    def RecognitionConfig(self) -> Callable[..., _RecognitionConfig]: ...
-
-    @property
-    def StreamingRecognitionConfig(self) -> Callable[..., _StreamingRecognitionConfig]: ...
-
-    @property
-    def EndpointingConfig(self) -> Callable[..., _EndpointingConfig]: ...
-
-
-def _auth_factory(riva_module: _RivaClientModule) -> Callable[..., _RivaAuth]:
-    return riva_module.Auth
-
-
-def _audio_encoding(riva_asr_module: _RivaAsrModule) -> _AudioEncoding:
-    return riva_asr_module.AudioEncoding
+def _use_ssl(optional_params: dict) -> bool:
+    """
+    NVCF endpoints are TLS-only, so ``nvcf_function_id`` defaults ``use_ssl``
+    to True. Self-hosted Riva defaults to plaintext but honors an explicit
+    ``use_ssl``, since TLS-terminating ingresses in front of it are common.
+    """
+    use_ssl_override: Final = optional_params.get("use_ssl")
+    if use_ssl_override is not None:
+        return bool(use_ssl_override)
+    return bool(optional_params.get("nvcf_function_id"))
 
 
 def _parse_riva_offline(value: object, drop_params: bool) -> bool:
@@ -192,7 +124,7 @@ class NvidiaRivaAudioTranscription:
         litellm_params: dict,
         model_response: TranscriptionResponse,
         timeout: float | str | httpx.Timeout | None,
-        logging_obj: "LiteLLMLoggingObj",
+        logging_obj: LiteLLMLoggingObj,
         api_key: str | None,
         api_base: str | None,
         atranscription: bool = False,
@@ -237,13 +169,11 @@ class NvidiaRivaAudioTranscription:
         litellm_params: dict,
         model_response: TranscriptionResponse,
         timeout: float | str | httpx.Timeout | None,
-        logging_obj: "LiteLLMLoggingObj",
+        logging_obj: LiteLLMLoggingObj,
         api_key: str | None,
         api_base: str | None,
         provider_config: NvidiaRivaAudioTranscriptionConfig | None = None,
     ) -> TranscriptionResponse:
-        # ``riva-client`` exposes a sync streaming generator, so we offload
-        # the blocking call to a worker thread to keep the event loop free.
         return await asyncio.to_thread(
             self._run_sync,
             model=model,
@@ -267,7 +197,7 @@ class NvidiaRivaAudioTranscription:
         litellm_params: dict,
         model_response: TranscriptionResponse,
         timeout: float | str | httpx.Timeout | None,
-        logging_obj: "LiteLLMLoggingObj",
+        logging_obj: LiteLLMLoggingObj,
         api_key: str | None,
         api_base: str | None,
         provider_config: NvidiaRivaAudioTranscriptionConfig,
@@ -317,18 +247,12 @@ class NvidiaRivaAudioTranscription:
         ) or litellm.drop_params is True
         riva_offline: Final = _parse_riva_offline(optional_params.get("riva_offline"), drop_params=drop_params)
 
-        riva_module, riva_asr_module = _import_riva()
-        auth_obj: Final = self._construct_auth(
-            riva_module=riva_module,
-            api_base=api_base,
-            api_key=api_key,
-            optional_params=optional_params,
-        )
+        try:
+            from litellm.llms.nvidia_riva.audio_transcription import riva_grpc
+        except ImportError as e:
+            raise NvidiaRivaException(status_code=500, message=_RIVA_INSTALL_HINT) from e
 
-        recognition_config: Final = self._build_recognition_config_proto(
-            riva_asr_module=riva_asr_module,
-            recognition_config_dict=recognition_config_dict,
-        )
+        recognition_config: Final = riva_grpc.recognition_config(recognition_config_dict)
         logging_obj.pre_call(
             input=None,
             api_key=api_key,
@@ -344,26 +268,18 @@ class NvidiaRivaAudioTranscription:
             },
         )
 
+        metadata: Final = riva_grpc.call_metadata(api_key, optional_params.get("nvcf_function_id"))
         try:
-            asr_service: Final[_AsrService] = riva_module.ASRService(auth_obj)
-            responses: Final = (
-                self._offline_responses(
-                    asr_service=asr_service,
-                    pcm_bytes=resampled.pcm_bytes,
-                    recognition_config=recognition_config,
-                    timeout=None if timeout is None else _deadline_seconds(timeout, drop_params),
+            deadline: Final = None if timeout is None else _deadline_seconds(timeout, drop_params)
+            with closing(riva_grpc.open_channel(api_base, _use_ssl(optional_params))) as channel:
+                responses: Final = (
+                    riva_grpc.recognize(channel, recognition_config, resampled.pcm_bytes, metadata, deadline)
+                    if riva_offline
+                    else riva_grpc.streaming_recognize(
+                        channel, recognition_config, resampled.pcm_bytes, metadata, deadline
+                    )
                 )
-                if riva_offline
-                else self._streaming_responses(
-                    asr_service=asr_service,
-                    riva_asr_module=riva_asr_module,
-                    pcm_bytes=resampled.pcm_bytes,
-                    recognition_config=recognition_config,
-                    timeout=timeout,
-                    drop_params=drop_params,
-                )
-            )
-            final_results: Final = self._collect_final_results(responses, require_final=not riva_offline)
+                final_results: Final = self._collect_final_results(responses, require_final=not riva_offline)
         except NvidiaRivaException:
             raise
         except Exception as e:
@@ -400,142 +316,6 @@ class NvidiaRivaAudioTranscription:
 
         return final_response
 
-    def _construct_auth(
-        self,
-        riva_module: _RivaClientModule,
-        api_base: str,
-        api_key: str | None,
-        optional_params: dict,
-    ) -> _RivaAuth:
-        """
-        Build a ``riva.client.Auth`` object.
-
-        - When ``nvcf_function_id`` is provided we attach the NVCF
-          ``function-id`` and bearer ``authorization`` metadata, and default
-          ``use_ssl`` to True (NVCF endpoints are TLS-only).
-        - Otherwise (self-hosted) we default ``use_ssl`` to False but still
-          honor an explicit override — self-hosted Riva behind an ingress
-          with TLS termination is a real deployment topology.
-        """
-        nvcf_function_id: Final = optional_params.get("nvcf_function_id")
-        use_ssl_override: Final = optional_params.get("use_ssl")
-        use_ssl: Final = bool(use_ssl_override) if use_ssl_override is not None else bool(nvcf_function_id)
-
-        metadata: Final[list[tuple[str, str]]] = []
-        if nvcf_function_id:
-            metadata.append(("function-id", str(nvcf_function_id)))
-        if api_key:
-            metadata.append(("authorization", f"Bearer {api_key}"))
-
-        try:
-            return _auth_factory(riva_module)(uri=api_base, use_ssl=use_ssl, metadata_args=metadata)
-        except TypeError:
-            # Older riva-client signatures used positional-only args.
-            return _auth_factory(riva_module)(None, use_ssl, api_base, metadata)
-
-    def _build_recognition_config_proto(
-        self, riva_asr_module: _RivaAsrModule, recognition_config_dict: dict[str, Any]
-    ) -> _RecognitionConfig:
-        encoding_name: Final = (recognition_config_dict.get("encoding") or "LINEAR_PCM").upper()
-        encoding_enum: Final[object] = getattr(
-            _audio_encoding(riva_asr_module),
-            encoding_name,
-            _audio_encoding(riva_asr_module).LINEAR_PCM,
-        )
-
-        config: Final[_RecognitionConfig] = riva_asr_module.RecognitionConfig(
-            encoding=encoding_enum,
-            sample_rate_hertz=int(recognition_config_dict["sample_rate_hertz"]),
-            language_code=recognition_config_dict["language_code"],
-            audio_channel_count=int(recognition_config_dict["audio_channel_count"]),
-            enable_automatic_punctuation=bool(recognition_config_dict.get("enable_automatic_punctuation", True)),
-            enable_word_time_offsets=bool(recognition_config_dict.get("enable_word_time_offsets", False)),
-            max_alternatives=int(recognition_config_dict.get("max_alternatives", 1)),
-            model=recognition_config_dict.get("model", "") or "",
-            verbatim_transcripts=bool(recognition_config_dict.get("verbatim_transcripts", False)),
-            profanity_filter=bool(recognition_config_dict.get("profanity_filter", False)),
-        )
-
-        endpointing: Final = recognition_config_dict.get("endpointing_config")
-        if isinstance(endpointing, dict) and endpointing:
-            try:
-                ep: Final[_EndpointingConfig] = riva_asr_module.EndpointingConfig(**endpointing)
-                config.endpointing_config.CopyFrom(ep)
-            except Exception:
-                # If the user supplied an unknown EndpointingConfig field
-                # (older Riva server), fall back to Riva's defaults rather
-                # than failing the whole request.
-                pass
-
-        return config
-
-    def _streaming_responses(
-        self,
-        asr_service: _AsrService,
-        riva_asr_module: _RivaAsrModule,
-        pcm_bytes: bytes,
-        recognition_config: _RecognitionConfig,
-        timeout: float | str | httpx.Timeout | None,
-        drop_params: bool,
-    ) -> Iterable[object]:
-        streaming_config: Final[_StreamingRecognitionConfig] = riva_asr_module.StreamingRecognitionConfig(
-            config=recognition_config, interim_results=False
-        )
-        return asr_service.streaming_response_generator(
-            audio_chunks=self._iter_audio_chunks(pcm_bytes),
-            streaming_config=streaming_config,
-            **self._timeout_kwargs(asr_service.streaming_response_generator, timeout, drop_params),
-        )
-
-    def _offline_responses(
-        self,
-        asr_service: _AsrService,
-        pcm_bytes: bytes,
-        recognition_config: _RecognitionConfig,
-        timeout: float | None,
-    ) -> Iterable[object]:
-        pending: Final = asr_service.offline_recognize(audio_bytes=pcm_bytes, config=recognition_config, future=True)
-        try:
-            return (pending.result(timeout=timeout),)
-        except Exception as e:
-            if pending.done():
-                raise
-            pending.cancel()
-            raise NvidiaRivaException(
-                status_code=408,
-                message=f"NVIDIA Riva offline recognition did not finish within {timeout} seconds",
-            ) from e
-
-    @classmethod
-    def _timeout_kwargs(
-        cls, rpc: Callable[..., object], timeout: float | str | httpx.Timeout | None, drop_params: bool
-    ) -> Mapping[str, float]:
-        # Forward the deadline so the call cannot block forever if the
-        # server stalls. Older riva-client versions do not accept a
-        # ``timeout`` kwarg, so pass it only when supported.
-        if timeout is None or not cls._supports_timeout_kwarg(rpc):
-            return MappingProxyType({})
-        return MappingProxyType({"timeout": _deadline_seconds(timeout, drop_params)})
-
-    @staticmethod
-    def _supports_timeout_kwarg(callable_obj: Callable[..., object]) -> bool:
-        try:
-            sig: Final = inspect.signature(callable_obj)
-        except (TypeError, ValueError):
-            return False
-        params: Final = sig.parameters
-        if "timeout" in params:
-            return True
-        return any(p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values())
-
-    @staticmethod
-    def _iter_audio_chunks(pcm_bytes: bytes):
-        for offset in range(0, len(pcm_bytes), _DEFAULT_CHUNK_BYTES):
-            chunk = pcm_bytes[offset : offset + _DEFAULT_CHUNK_BYTES]
-            if not chunk:
-                continue
-            yield chunk
-
     @staticmethod
     def _collect_final_results(stream: Iterable[object], require_final: bool = True) -> Sequence[RivaFinalResult]:
         """
@@ -570,27 +350,3 @@ class NvidiaRivaAudioTranscription:
                     }
                 )
         return final_results
-
-
-def _import_riva() -> tuple[_RivaClientModule, _RivaAsrModule]:
-    """
-    Lazy import of ``riva.client`` and ``riva.client.proto.riva_asr_pb2``.
-
-    We try the SDK first (preferred) and fall back to importing the proto
-    module separately when the SDK packaging changes between versions.
-    """
-    try:
-        import riva.client as riva_client
-    except ImportError as e:
-        raise NvidiaRivaException(status_code=500, message=_RIVA_INSTALL_HINT) from e
-
-    riva_asr_module = riva_client
-    if not hasattr(riva_asr_module, "RecognitionConfig"):
-        try:
-            from riva.client.proto import riva_asr_pb2
-
-            riva_asr_module = riva_asr_pb2
-        except ImportError as e:
-            raise NvidiaRivaException(status_code=500, message=_RIVA_INSTALL_HINT) from e
-
-    return riva_client, riva_asr_module

litellm/llms/nvidia_riva/audio_transcription/riva_asr_schema.py

@@ -0,0 +1,199 @@
+"""
+The part of NVIDIA Riva's speech recognition gRPC schema that LiteLLM sends and reads.
+
+Names and numbers match ``riva/proto/riva_asr.proto`` and ``riva/proto/riva_audio.proto`` in
+https://github.com/nvidia-riva/common. Fields LiteLLM never sets or reads are left out, and the
+wire format skips them. The messages live in their own descriptor pool so they never collide
+with nvidia-riva-client's copies when both are imported in one process.
+"""
+
+from dataclasses import dataclass
+from enum import Enum, IntEnum
+from typing import Final
+
+from google.protobuf import descriptor_pb2, descriptor_pool, message_factory
+from google.protobuf.message import Message
+
+_PACKAGE: Final = "nvidia.riva.asr"
+_SERVICE: Final = f"/{_PACKAGE}.RivaSpeechRecognition"
+RECOGNIZE_METHOD: Final = f"{_SERVICE}/Recognize"
+STREAMING_RECOGNIZE_METHOD: Final = f"{_SERVICE}/StreamingRecognize"
+
+
+class _Kind(Enum):
+    FLOAT = "TYPE_FLOAT"
+    INT32 = "TYPE_INT32"
+    BOOL = "TYPE_BOOL"
+    STRING = "TYPE_STRING"
+    MESSAGE = "TYPE_MESSAGE"
+    BYTES = "TYPE_BYTES"
+    ENUM = "TYPE_ENUM"
+
+
+class AudioEncoding(IntEnum):
+    ENCODING_UNSPECIFIED = 0
+    LINEAR_PCM = 1
+
+
+LINEAR_PCM: Final = AudioEncoding.LINEAR_PCM.value
+
+
+@dataclass(frozen=True, slots=True)
+class _Field:
+    name: str
+    number: int
+    kind: _Kind
+    type_name: str = ""
+    repeated: bool = False
+    optional: bool = False
+    oneof: str = ""
+
+
+_MESSAGES: Final[tuple[tuple[str, tuple[_Field, ...]], ...]] = (
+    (
+        "RecognitionConfig",
+        (
+            _Field("encoding", 1, _Kind.ENUM, "AudioEncoding"),
+            _Field("sample_rate_hertz", 2, _Kind.INT32),
+            _Field("language_code", 3, _Kind.STRING),
+            _Field("max_alternatives", 4, _Kind.INT32),
+            _Field("profanity_filter", 5, _Kind.BOOL),
+            _Field("audio_channel_count", 7, _Kind.INT32),
+            _Field("enable_word_time_offsets", 8, _Kind.BOOL),
+            _Field("enable_automatic_punctuation", 11, _Kind.BOOL),
+            _Field("model", 13, _Kind.STRING),
+            _Field("verbatim_transcripts", 14, _Kind.BOOL),
+            _Field("endpointing_config", 25, _Kind.MESSAGE, "EndpointingConfig", optional=True),
+        ),
+    ),
+    (
+        "EndpointingConfig",
+        (
+            _Field("start_history", 1, _Kind.INT32, optional=True),
+            _Field("start_threshold", 2, _Kind.FLOAT, optional=True),
+            _Field("stop_history", 3, _Kind.INT32, optional=True),
+            _Field("stop_threshold", 4, _Kind.FLOAT, optional=True),
+            _Field("stop_history_eou", 5, _Kind.INT32, optional=True),
+            _Field("stop_threshold_eou", 6, _Kind.FLOAT, optional=True),
+        ),
+    ),
+    (
+        "StreamingRecognitionConfig",
+        (
+            _Field("config", 1, _Kind.MESSAGE, "RecognitionConfig"),
+            _Field("interim_results", 2, _Kind.BOOL),
+        ),
+    ),
+    (
+        "StreamingRecognizeRequest",
+        (
+            _Field("streaming_config", 1, _Kind.MESSAGE, "StreamingRecognitionConfig", oneof="streaming_request"),
+            _Field("audio_content", 2, _Kind.BYTES, oneof="streaming_request"),
+        ),
+    ),
+    (
+        "WordInfo",
+        (
+            _Field("start_time", 1, _Kind.INT32),
+            _Field("end_time", 2, _Kind.INT32),
+            _Field("word", 3, _Kind.STRING),
+        ),
+    ),
+    (
+        "SpeechRecognitionAlternative",
+        (
+            _Field("transcript", 1, _Kind.STRING),
+            _Field("words", 3, _Kind.MESSAGE, "WordInfo", repeated=True),
+        ),
+    ),
+    (
+        "StreamingRecognitionResult",
+        (
+            _Field("alternatives", 1, _Kind.MESSAGE, "SpeechRecognitionAlternative", repeated=True),
+            _Field("is_final", 2, _Kind.BOOL),
+        ),
+    ),
+    (
+        "StreamingRecognizeResponse",
+        (_Field("results", 1, _Kind.MESSAGE, "StreamingRecognitionResult", repeated=True),),
+    ),
+    (
+        "RecognizeRequest",
+        (
+            _Field("config", 1, _Kind.MESSAGE, "RecognitionConfig"),
+            _Field("audio", 2, _Kind.BYTES),
+        ),
+    ),
+    (
+        "SpeechRecognitionResult",
+        (_Field("alternatives", 1, _Kind.MESSAGE, "SpeechRecognitionAlternative", repeated=True),),
+    ),
+    (
+        "RecognizeResponse",
+        (_Field("results", 1, _Kind.MESSAGE, "SpeechRecognitionResult", repeated=True),),
+    ),
+)
+
+
+def _oneof_names(fields: tuple[_Field, ...]) -> tuple[str, ...]:
+    declared: Final = tuple(dict.fromkeys(field.oneof for field in fields if field.oneof))
+    synthetic: Final = tuple(f"_{field.name}" for field in fields if field.optional)
+    return declared + synthetic
+
+
+def _field_proto(field: _Field, oneofs: tuple[str, ...]) -> descriptor_pb2.FieldDescriptorProto:
+    oneof: Final = f"_{field.name}" if field.optional else field.oneof
+    return descriptor_pb2.FieldDescriptorProto(
+        name=field.name,
+        number=field.number,
+        type=descriptor_pb2.FieldDescriptorProto.Type.Value(field.kind.value),
+        label=(
+            descriptor_pb2.FieldDescriptorProto.LABEL_REPEATED
+            if field.repeated
+            else descriptor_pb2.FieldDescriptorProto.LABEL_OPTIONAL
+        ),
+        type_name=f".{_PACKAGE}.{field.type_name}" if field.type_name else None,
+        oneof_index=oneofs.index(oneof) if oneof else None,
+        proto3_optional=field.optional or None,
+    )
+
+
+def _message_proto(name: str, fields: tuple[_Field, ...]) -> descriptor_pb2.DescriptorProto:
+    oneofs: Final = _oneof_names(fields)
+    return descriptor_pb2.DescriptorProto(
+        name=name,
+        field=[_field_proto(field, oneofs) for field in fields],
+        oneof_decl=[descriptor_pb2.OneofDescriptorProto(name=oneof) for oneof in oneofs],
+    )
+
+
+_FILE: Final = descriptor_pb2.FileDescriptorProto(
+    name="litellm/nvidia_riva/riva_asr.proto",
+    package=_PACKAGE,
+    syntax="proto3",
+    enum_type=[
+        descriptor_pb2.EnumDescriptorProto(
+            name="AudioEncoding",
+            value=[
+                descriptor_pb2.EnumValueDescriptorProto(name=encoding.name, number=encoding.value)
+                for encoding in AudioEncoding
+            ],
+        )
+    ],
+    message_type=[_message_proto(name, fields) for name, fields in _MESSAGES],
+)
+_POOL: Final = descriptor_pool.DescriptorPool()
+_POOL.AddSerializedFile(_FILE.SerializeToString())
+
+
+def _message_class(name: str) -> type[Message]:
+    return message_factory.GetMessageClass(_POOL.FindMessageTypeByName(f"{_PACKAGE}.{name}"))
+
+
+RecognitionConfig: Final = _message_class("RecognitionConfig")
+EndpointingConfig: Final = _message_class("EndpointingConfig")
+StreamingRecognitionConfig: Final = _message_class("StreamingRecognitionConfig")
+StreamingRecognizeRequest: Final = _message_class("StreamingRecognizeRequest")
+StreamingRecognizeResponse: Final = _message_class("StreamingRecognizeResponse")
+RecognizeRequest: Final = _message_class("RecognizeRequest")
+RecognizeResponse: Final = _message_class("RecognizeResponse")

litellm/llms/nvidia_riva/audio_transcription/riva_grpc.py

@@ -0,0 +1,113 @@
+"""
+gRPC calls to NVIDIA Riva's speech recognition service.
+
+The ``authorization`` and NVCF ``function-id`` metadata ride on each call, never on the channel, so every
+request carries them exactly once over both plaintext and TLS.
+"""
+
+from collections.abc import Iterator, Mapping, Sequence
+from typing import Any, Final, TypeAlias
+
+import grpc
+from google.protobuf.message import Message
+from pydantic import TypeAdapter
+
+from litellm.llms.nvidia_riva.audio_transcription import riva_asr_schema as schema
+from litellm.llms.nvidia_riva.common_utils import NvidiaRivaException
+
+# Stream audio to Riva in ~50 ms slices (1600 samples at 16 kHz). Matches
+# NVIDIA's recommended chunk size for streaming ASR, small enough for
+# responsive endpointing, large enough to keep per-RPC overhead low.
+_CHUNK_SAMPLES: Final = 1600
+_CHUNK_BYTES: Final = _CHUNK_SAMPLES * 2
+
+CallMetadata: TypeAlias = tuple[tuple[str, str], ...]
+
+_ENDPOINTING_FIELDS: Final = TypeAdapter(Mapping[str, object])
+
+
+def call_metadata(api_key: str | None, nvcf_function_id: object) -> CallMetadata:
+    function_id: Final = (("function-id", str(nvcf_function_id)),) if nvcf_function_id else ()
+    authorization: Final = (("authorization", f"Bearer {api_key}"),) if api_key else ()
+    return function_id + authorization
+
+
+def open_channel(api_base: str, use_ssl: bool) -> grpc.Channel:
+    if use_ssl:
+        return grpc.secure_channel(api_base, grpc.ssl_channel_credentials())
+    return grpc.insecure_channel(api_base)
+
+
+def recognition_config(config: Mapping[str, Any]) -> Message:
+    endpointing: Final = _endpointing_config(config.get("endpointing_config"))
+    return schema.RecognitionConfig(
+        encoding=schema.LINEAR_PCM,
+        sample_rate_hertz=int(config["sample_rate_hertz"]),
+        language_code=config["language_code"],
+        audio_channel_count=int(config["audio_channel_count"]),
+        enable_automatic_punctuation=bool(config.get("enable_automatic_punctuation", True)),
+        enable_word_time_offsets=bool(config.get("enable_word_time_offsets", False)),
+        max_alternatives=int(config.get("max_alternatives", 1)),
+        model=config.get("model", "") or "",
+        verbatim_transcripts=bool(config.get("verbatim_transcripts", False)),
+        profanity_filter=bool(config.get("profanity_filter", False)),
+        **({} if endpointing is None else {"endpointing_config": endpointing}),
+    )
+
+
+def _endpointing_config(endpointing: object) -> Message | None:
+    try:
+        fields: Final = _ENDPOINTING_FIELDS.validate_python(endpointing)
+        return schema.EndpointingConfig(**fields) if fields else None
+    except (ValueError, TypeError):
+        # A missing, unknown or mistyped field falls back to Riva's default
+        # endpointing rather than failing the whole request.
+        return None
+
+
+def _streaming_requests(config: Message, pcm_bytes: bytes) -> Iterator[Message]:
+    yield schema.StreamingRecognizeRequest(
+        streaming_config=schema.StreamingRecognitionConfig(config=config, interim_results=False)
+    )
+    for offset in range(0, len(pcm_bytes), _CHUNK_BYTES):
+        yield schema.StreamingRecognizeRequest(audio_content=pcm_bytes[offset : offset + _CHUNK_BYTES])
+
+
+def streaming_recognize(
+    channel: grpc.Channel,
+    config: Message,
+    pcm_bytes: bytes,
+    metadata: CallMetadata,
+    timeout: float | None,
+) -> Iterator[Message]:
+    rpc: Final = channel.stream_stream(
+        schema.STREAMING_RECOGNIZE_METHOD,
+        request_serializer=schema.StreamingRecognizeRequest.SerializeToString,
+        response_deserializer=schema.StreamingRecognizeResponse.FromString,
+    )
+    return rpc(_streaming_requests(config, pcm_bytes), metadata=metadata, timeout=timeout)
+
+
+def recognize(
+    channel: grpc.Channel,
+    config: Message,
+    pcm_bytes: bytes,
+    metadata: CallMetadata,
+    timeout: float | None,
+) -> Sequence[Message]:
+    rpc: Final = channel.unary_unary(
+        schema.RECOGNIZE_METHOD,
+        request_serializer=schema.RecognizeRequest.SerializeToString,
+        response_deserializer=schema.RecognizeResponse.FromString,
+    )
+    pending: Final = rpc.future(schema.RecognizeRequest(config=config, audio=pcm_bytes), metadata=metadata)
+    try:
+        return (pending.result(timeout=timeout),)
+    except Exception as e:
+        if pending.done():
+            raise
+        pending.cancel()
+        raise NvidiaRivaException(
+            status_code=408,
+            message=f"NVIDIA Riva offline recognition did not finish within {timeout} seconds",
+        ) from e

litellm/llms/openai/responses/transformation.py

@@ -669,7 +669,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         prompt_cache_options: object = None,
     ) -> tuple[str | ResponseInputParam, Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None]:
         validated_input: Final = self._validate_input_param(
-            self._drop_bridge_minted_reasoning_items(input), drops_set_created_by=drops_params(litellm_params)
+            self._drop_bridge_minted_reasoning_items(input), drop_params=drops_params(litellm_params)
         )
         reads_breakpoints: Final = self._reads_prompt_cache_breakpoints(
             model=model, litellm_params=litellm_params, prompt_cache_options=prompt_cache_options
@@ -951,7 +951,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         return tools if all(new is old for new, old in zip(sanitized, tools, strict=True)) else sanitized
 
     def _validate_input_param(
-        self, input: str | ResponseInputParam, *, drops_set_created_by: bool = False
+        self, input: str | ResponseInputParam, *, drop_params: bool = False
     ) -> str | ResponseInputParam:
         """
         Ensure all input fields if pydantic are converted to dict
@@ -965,9 +965,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
                 # if it's pydantic, convert to dict
                 if isinstance(item, BaseModel):
                     validated_input.append(
-                        replayable_responses_item(
-                            item.model_dump(exclude_none=True), drops_set_created_by=drops_set_created_by
-                        )
+                        replayable_responses_item(item.model_dump(exclude_none=True), drops_set_created_by=drop_params)
                     )
                 elif isinstance(item, dict):
                     validated_input.append(
@@ -975,7 +973,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
                             self._handle_reasoning_item(cast(dict[str, object], item))
                             if item.get("type") == "reasoning"
                             else item,
-                            drops_set_created_by=drops_set_created_by,
+                            drops_set_created_by=drop_params,
                         )
                     )
                 else:

litellm/llms/openai_like/dynamic_config.py

@@ -2,9 +2,11 @@
 Dynamic configuration class generator for JSON-based providers.
 """
 
-from collections.abc import Coroutine
+from collections.abc import Coroutine, Mapping
 from typing import TYPE_CHECKING, Final, Literal, overload
 
+from pydantic import TypeAdapter, ValidationError
+
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     handle_messages_with_content_list_to_str_conversion,
@@ -20,6 +22,26 @@ from .json_loader import SimpleProviderConfig
 if TYPE_CHECKING:
     from litellm.llms.openai_like.responses.transformation import OpenAILikeResponsesConfig
 
+_PAID_SEARCH_TOOLS: Final = frozenset({"web_search", "web_search_image", "image_search"})
+_PAYLOAD_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
+_TOOL_USAGE_ADAPTER: Final[TypeAdapter[Mapping[str, int]]] = TypeAdapter(Mapping[str, int])
+
+
+def web_search_requests_from_tool_usage(payload: object) -> int | None:
+    """
+    Billable search calls in a parsed OpenAI-compatible chat response whose usage carries a
+    tool_usage {tool_name: count} map, as EmpirioLabs reports built-in tool invocations.
+    Free tools (code_interpreter, web_extractor) are not counted.
+    """
+    try:
+        body: Final = _PAYLOAD_ADAPTER.validate_python(payload)
+        usage: Final = _PAYLOAD_ADAPTER.validate_python(body.get("usage"))
+        tool_usage: Final = _TOOL_USAGE_ADAPTER.validate_python(usage.get("tool_usage"), strict=True)
+    except ValidationError:
+        return None
+    calls: Final = sum(count for tool, count in tool_usage.items() if tool in _PAID_SEARCH_TOOLS and count > 0)
+    return calls if calls > 0 else None
+
 
 def create_config_class(provider: SimpleProviderConfig):
     """Generate config class dynamically from JSON configuration"""
@@ -179,6 +201,25 @@ def create_config_class(provider: SimpleProviderConfig):
 
             return optional_params
 
+        def transform_parsed_response_dict(
+            self,
+            parsed_response: dict[str, object],  # mutable-ok: BaseConfig signature
+        ) -> dict[str, object]:  # mutable-ok: BaseConfig signature
+            """
+            Carry the provider's usage.tool_usage search count as usage.server_tool_use.web_search_requests,
+            the field the built-in tool cost path bills per call.
+            """
+            repaired: Final = _PAYLOAD_ADAPTER.validate_python(
+                super().transform_parsed_response_dict(parsed_response)  # pyright: ignore[reportUnknownMemberType]  # base_class is picked at runtime
+            )
+            if not provider.special_handling.get("web_search_requests_from_tool_usage"):
+                return {**repaired}
+            web_search_requests: Final = web_search_requests_from_tool_usage(repaired)
+            if web_search_requests is None:
+                return {**repaired}
+            usage: Final = _PAYLOAD_ADAPTER.validate_python(repaired["usage"])
+            return {**repaired, "usage": {**usage, "server_tool_use": {"web_search_requests": web_search_requests}}}
+
         @property
         def custom_llm_provider(self) -> str | None:
             return provider.slug

litellm/llms/openai_like/providers.json

@@ -178,7 +178,10 @@
     "param_mappings": {
       "max_completion_tokens": "max_tokens"
     },
-    "supported_endpoints": ["/v1/chat/completions", "/v1/responses"]
+    "special_handling": {
+      "web_search_requests_from_tool_usage": true
+    },
+    "supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/embeddings"]
   },
   "meta": {
     "base_url": "https://api.meta.ai/v1",

litellm/llms/perplexity/responses/transformation.py

@@ -89,7 +89,7 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
         """Handle preset/ model prefix: send as {"preset": name} instead of {"model": name}."""
         input = self._ensure_message_type(input)
         if model.startswith("preset/"):
-            input = self._validate_input_param(input, drops_set_created_by=drops_params(litellm_params))
+            input = self._validate_input_param(input, drop_params=drops_params(litellm_params))
             data: Final[dict] = {
                 "preset": model[len("preset/") :],
                 "input": input,

litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py

@@ -5,9 +5,11 @@ import json
 import re
 import threading
 import time
+from collections import Counter
 from collections.abc import Awaitable, Callable, Mapping, Sequence
 from copy import deepcopy
 from functools import lru_cache, partial
+from itertools import chain, repeat
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, Protocol, Union, cast, get_args
 
@@ -34,7 +36,11 @@ from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAcc
 from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id
 from litellm.litellm_core_utils.prompt_templates.factory import encode_tool_call_id_with_signature
 from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
-from litellm.litellm_core_utils.tool_fields import tool_params_without_hosted_mcp
+from litellm.litellm_core_utils.tool_fields import (
+    declared_function_name,
+    qualified_member_name,
+    tool_params_without_hosted_mcp,
+)
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
@@ -544,6 +550,138 @@ def mapped_gemini_speech_config(
     }
 
 
+def _as_chat_tool(member: JsonValue) -> JsonValue:
+    if not isinstance(member, dict) or "function" in member or "name" not in member:
+        return member
+    return {
+        "type": member.get("type", "function"),
+        "function": {key: value for key, value in member.items() if key != "type"},
+    }
+
+
+def _namespace_members_or_tool(tool: JsonValue) -> tuple[JsonValue, ...]:
+    if not isinstance(tool, dict) or tool.get("type") != "namespace":
+        return (tool,)
+    members: Final = tool.get("tools")
+    return _members_named_once(tuple(map(_as_chat_tool, members))) if isinstance(members, list) else ()
+
+
+def _members_named_once(
+    members: tuple[JsonValue, ...], repeatable: frozenset[str] = frozenset()
+) -> tuple[JsonValue, ...]:
+    names: Final = tuple(map(declared_function_name, members))
+    first_index: Final = {name: index for index, name in reversed(tuple(enumerate(names)))}
+    return tuple(
+        member
+        for index, (member, name) in enumerate(zip(members, names, strict=True))
+        if name is None or name in repeatable or first_index[name] == index
+    )
+
+
+def _is_namespace(tool: JsonValue) -> bool:
+    return isinstance(tool, dict) and tool.get("type") == "namespace"
+
+
+def _top_level_name(tool: JsonValue) -> str | None:
+    name: Final = tool.get("name") if isinstance(tool, dict) else None
+    return name if isinstance(name, str) and name else None
+
+
+def _listed(tools: JsonValue) -> tuple[JsonValue, ...]:
+    return tuple(tools) if isinstance(tools, list) else ()
+
+
+def _declarable_names(tool: JsonValue) -> tuple[str | None, ...]:
+    members: Final = _namespace_members_or_tool(tool) if _is_namespace(tool) else ()
+    return (declared_function_name(tool), *map(declared_function_name, members))
+
+
+def _declarable_names_in(tools: tuple[JsonValue, ...]) -> tuple[str | None, ...]:
+    return tuple(chain.from_iterable(map(_declarable_names, tools)))
+
+
+def _inlines_members(tool: JsonValue, declarable: Counter[str | None]) -> bool:
+    members: Final = _namespace_members_or_tool(tool)
+    return _is_namespace(tool) and all(
+        name is None or declarable[name] == 1 for name in map(declared_function_name, members)
+    )
+
+
+def _qualified_member(namespace: str, member: JsonValue) -> JsonValue:
+    function: Final = member.get("function") if isinstance(member, dict) else None
+    name: Final = declared_function_name(member)
+    if not isinstance(member, dict) or not isinstance(function, dict) or name is None:
+        return member
+    return {**member, "function": {**function, "name": qualified_member_name(namespace, name)}}
+
+
+def _as_declared(
+    tool: JsonValue, declarable: Counter[str | None], top_level_names: frozenset[str]
+) -> tuple[JsonValue, ...]:
+    if _inlines_members(tool, declarable):
+        return _namespace_members_or_tool(tool)
+    namespace: Final = _top_level_name(tool)
+    if not _is_namespace(tool) or namespace is None:
+        return (tool,)
+    qualified: Final = (_qualified_member(namespace, member) for member in _namespace_members_or_tool(tool))
+    return tuple(member for member in qualified if declared_function_name(member) not in top_level_names)
+
+
+def _top_level_names_in(tools: tuple[JsonValue, ...]) -> frozenset[str]:
+    top_level_tools: Final = (tool for tool in tools if not _is_namespace(tool))
+    return frozenset(name for name in map(declared_function_name, top_level_tools) if name is not None)
+
+
+def _with_namespace_members_inlined(tools: JsonValue, outside_declarations: JsonValue = None) -> JsonValue:
+    if not isinstance(tools, list) or not any(map(_is_namespace, tools)):
+        return tools
+    candidates: Final = (*tools, *_listed(outside_declarations))
+    declarable: Final = Counter(_declarable_names_in(candidates))
+    top_level_names: Final = _top_level_names_in(candidates)
+    declared: Final = chain.from_iterable(_as_declared(tool, declarable, top_level_names) for tool in tools)
+    return list(_members_named_once(tuple(declared), top_level_names))
+
+
+def _names_each_tool_declares(candidates: tuple[JsonValue, ...]) -> tuple[tuple[str, ...], ...]:
+    declarable: Final = Counter(_declarable_names_in(candidates))
+    top_level_names: Final = _top_level_names_in(candidates)
+    declared: Final = tuple(
+        tuple(filter(None, map(declared_function_name, _as_declared(tool, declarable, top_level_names))))
+        for tool in candidates
+    )
+    owned_names: Final = chain.from_iterable(zip(repeat(index), names) for index, names in enumerate(declared))
+    first_owner: Final = {name: index for index, name in reversed(tuple(owned_names))}
+    return tuple(tuple(name for name in names if first_owner[name] == index) for index, names in enumerate(declared))
+
+
+def _qualified_names_of(
+    member_name: str, tool: JsonValue, declarable: Counter[str | None], declared: tuple[str, ...]
+) -> tuple[str, ...]:
+    namespace: Final = _top_level_name(tool)
+    if not _is_namespace(tool) or namespace is None or _inlines_members(tool, declarable):
+        return ()
+    qualified: Final = qualified_member_name(namespace, member_name)
+    return (qualified,) if qualified in declared else ()
+
+
+def _forced_function_names(name: str, tools: JsonValue) -> tuple[str, ...]:
+    candidates: Final = _listed(tools)
+    if any(not _is_namespace(tool) and declared_function_name(tool) == name for tool in candidates):
+        return (name,)
+    declarable: Final = Counter(_declarable_names_in(candidates))
+    declared_by_tool: Final = tuple(zip(candidates, _names_each_tool_declares(candidates), strict=True))
+    named: Final = next(
+        (declared for tool, declared in declared_by_tool if _is_namespace(tool) and _top_level_name(tool) == name),
+        None,
+    )
+    if named is not None:
+        return named
+    qualified: Final = chain.from_iterable(
+        _qualified_names_of(name, tool, declarable, declared) for tool, declared in declared_by_tool
+    )
+    return tuple(qualified) or (name,)
+
+
 class VertexAIBaseConfig:
     def get_mapped_special_auth_params(self) -> dict:
         """
@@ -815,7 +953,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
 
         return supported_params
 
-    def map_tool_choice_values(self, model: str, tool_choice: str | dict) -> ToolConfig | None:
+    def map_tool_choice_values(
+        self, model: str, tool_choice: str | dict, tools: JsonValue = None, drop_params: bool = False
+    ) -> ToolConfig | None:
         if tool_choice == "none":
             return ToolConfig(functionCallingConfig=FunctionCallingConfig(mode="NONE"))
         elif tool_choice == "required":
@@ -825,7 +965,17 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
         elif isinstance(tool_choice, dict):
             # only supported for anthropic + mistral models - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html
             name: Final = tool_choice.get("function", {}).get("name", "")
-            return ToolConfig(functionCallingConfig=FunctionCallingConfig(mode="ANY", allowed_function_names=[name]))
+            forced: Final = _forced_function_names(name, tools)
+            if forced:
+                return ToolConfig(
+                    functionCallingConfig=FunctionCallingConfig(mode="ANY", allowed_function_names=list(forced))
+                )
+            if drops_unsupported_params(drop_params):
+                return None
+            raise litellm.utils.UnsupportedParamsError(
+                message=f"VertexAI can't force tool_choice '{name}', since none of that namespace's members can be sent to Gemini as a function. Name one of its functions instead, or set `litellm.drop_params = True` to drop tool_choice.",
+                status_code=400,
+            )
         else:
             raise litellm.utils.UnsupportedParamsError(
                 message=f"VertexAI doesn't support tool_choice={tool_choice}. Supported tool_choice values=['auto', 'required', json object]. To drop it from the call, set `litellm.drop_params = True.",
@@ -1068,7 +1218,13 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
 
         return googleSearch, googleSearchRetrieval, enterpriseWebSearch, urlContext
 
-    def _map_function(self, value: list[dict], optional_params: dict, send_integer_enums: bool = False) -> list[Tools]:
+    def _map_function(
+        self,
+        value: list[dict],
+        optional_params: dict,
+        send_integer_enums: bool = False,
+        outside_declarations: JsonValue = None,
+    ) -> list[Tools]:
         """
         Map OpenAI-style tools/functions to Vertex AI format.
 
@@ -1099,7 +1255,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
         # remove 'strict' from tools
         remove_strict_from_schema(schema_value)
 
-        for tool in cast(list[dict], schema_value):  # cast-ok: a deep copy keeps the caller's tool shape
+        inlined_tools: Final = _with_namespace_members_inlined(schema_value, outside_declarations)
+        for tool in cast(list[dict], inlined_tools):  # cast-ok: a deep copy keeps the caller's tool shape
             openai_function_object: ChatCompletionToolParamFunctionChunk | None = None
             if "function" in tool:  # tools list
                 _openai_function_object = ChatCompletionToolParamFunctionChunk(**tool["function"])
@@ -1734,7 +1891,12 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
             elif (param == "tools" or param == "functions") and isinstance(value, list) and value:
                 # Pass optional_params so _map_function can add toolConfig if needed
                 mapped_tools = with_integer_enums_that_fit(
-                    partial(self._map_function, value, optional_params),
+                    partial(
+                        self._map_function,
+                        value,
+                        optional_params,
+                        outside_declarations=request_params.get("functions" if param == "tools" else "tools"),
+                    ),
                     allow_integer_enums and self._is_gemini_3_or_newer(model),
                 )
                 optional_params = self._add_tools_to_optional_params(optional_params, mapped_tools)
@@ -1742,6 +1904,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
                 _tool_choice_value = self.map_tool_choice_values(
                     model=model,
                     tool_choice=value,
+                    tools=[*_listed(non_default_params.get("tools")), *_listed(non_default_params.get("functions"))],
+                    drop_params=drop_params,
                 )
                 if _tool_choice_value is not None:
                     optional_params["tool_choice"] = _tool_choice_value

litellm/provider_endpoints_support_backup.json

@@ -5277,13 +5277,13 @@
       }
     },
     "empiriolabs": {
-      "display_name": "EmpirioLabs (`empiriolabs`)",
+      "display_name": "EmpirioLabs AI (`empiriolabs`)",
       "url": "https://docs.litellm.ai/docs/providers/empiriolabs",
       "endpoints": {
-        "chat_completions": false,
-        "responses": false,
-        "text_completion": false,
-        "embeddings": false,
+        "chat_completions": true,
+        "responses": true,
+        "text_completion": true,
+        "embeddings": true,
         "image_generations": false,
         "image_edits": false,
         "audio_transcriptions": false,
@@ -5295,11 +5295,11 @@
         "realtime": false,
         "video_generations": false,
         "batches": false,
-        "messages": false,
-        "interactions": false,
+        "messages": true,
+        "interactions": true,
         "systemone": false,
-        "a2a": false,
-        "generateContent": false
+        "a2a": true,
+        "generateContent": true
       }
     },
     "openinfer": {

litellm/proxy/_lazy_openapi_snapshot.json

@@ -20233,7 +20233,7 @@
                   "type": "null"
                 }
               ],
-              "description": "Regex pattern applied to the tool's function name",
+              "description": "Regex pattern applied to the tool's name, or to its type when the tool has no name",
               "title": "Tool Name"
             },
             "tool_type": {

litellm/proxy/_types.py

@@ -38,7 +38,6 @@ from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
     validate_no_callback_env_reference,
 )
 from litellm.proxy.mcp_server.stdio_gate import MCP_STDIO_DISABLED_MESSAGE, is_mcp_stdio_enabled
-from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings
 from litellm.types.agents import (
     AgentCaller as AgentCaller,  # noqa: PLC0414  # public re-export
 )
@@ -2481,20 +2480,9 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
     Documents all the fields supported by `general_settings` in config.yaml
     """
 
-    accounting_capture: UsageCaptureSettings | None = Field(
-        None,
-        description="Opt-in durable usage capture. Legacy spend updates remain active. Requires local Prisma writes.",
-    )
-    accounting_sync_ack: bool = Field(
+    use_spend_logs_ledger: bool = Field(
         False,
-        description="Wait for usage commit on foreground non-stream chat, text, embedding and Responses requests. "
-        "Other requests remain supported with asynchronous usage capture.",
-    )
-    accounting_sync_ack_timeout_seconds: float = Field(
-        20.0, gt=0, description="Maximum response wait for usage commit, including the batch flush interval"
-    )
-    accounting_sync_ack_failure_policy: Literal["fail_closed", "log"] = Field(
-        "fail_closed", description="Return 503 or log the persistence failure after the response wait expires"
+        description="Apply request accounting from stored SpendLogs deliveries. Enable only after all accounting writers are upgraded and ledger activation gates pass.",
     )
 
     request_tag_mappings: Sequence[RequestTagMapping] | None = Field(
@@ -3817,6 +3805,9 @@ class SpendLogsMetadata(TypedDict):
 
 
 class SpendLogsPayload(TypedDict):
+    accounting_payload: NotRequired[Mapping[str, object]]  # writable-ok: capture before buffering
+    accounting_log_disabled: NotRequired[bool]  # writable-ok: capture logging mode before buffering
+    accounting_delivery_id: NotRequired[str]  # writable-ok: assigned once before buffering
     request_id: str
     call_type: str
     api_key: str

litellm/proxy/agent_endpoints/a2a_endpoints.py

@@ -26,6 +26,7 @@ from litellm._logging import verbose_proxy_logger
 from litellm.litellm_core_utils.core_helpers import as_str_mapping
 from litellm.litellm_core_utils.url_utils import SSRFError, validate_url
 from litellm.llms.a2a.common_utils import resolve_a2a_hop_auth_header
+from litellm.llms.azure.common_utils import caused_by_entra_rejection
 from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines
 from litellm.proxy._types import ProxyException, UserAPIKeyAuth
 from litellm.proxy.a2a.agent_card import point_agent_card_at_proxy
@@ -1202,6 +1203,8 @@ async def invoke_agent_a2a(
             pass
         if isinstance(e, litellm.BadRequestError):
             return _jsonrpc_error(body.get("id"), -32602, e.message, 400)
+        if isinstance(e, litellm.AuthenticationError) and caused_by_entra_rejection(e):
+            return _jsonrpc_error(body.get("id"), -32000, e.message, 401)
         return _jsonrpc_error(body.get("id"), -32603, f"Internal error: {e}", 500)
 
 

litellm/proxy/anthropic_endpoints/answered_status.py

@@ -1,10 +1,29 @@
 from typing import Final, TypeVar
 
-from litellm.exceptions import InternalServerError, MidStreamFallbackError
+from litellm.exceptions import APIError, InternalServerError, MidStreamFallbackError
+
+_PROVIDER_OVERLOAD_STATUS_CODE: Final = 529
+_OPENAI_OVERLOAD_ANSWERED_STATUS_CODE: Final = 503
 
 _Raised = TypeVar("_Raised", bound=Exception)
 
 
+def is_provider_overload(exc: Exception) -> bool:
+    return _provider_status_code(exc) == _PROVIDER_OVERLOAD_STATUS_CODE
+
+
+def with_openai_overload_status(exc: _Raised) -> _Raised:
+    return (
+        _copy_carrying(
+            _copy_carrying(exc, "provider_status_code", _PROVIDER_OVERLOAD_STATUS_CODE),
+            "status_code",
+            _OPENAI_OVERLOAD_ANSWERED_STATUS_CODE,
+        )
+        if is_provider_overload(exc)
+        else exc
+    )
+
+
 def _provider_status_code(exc: Exception) -> int | None:
     unwrapped = exc  # rebind-ok: the recursion gate requires iterative wrapper traversal
     while isinstance(unwrapped, MidStreamFallbackError) and unwrapped.original_exception is not None:
@@ -12,6 +31,10 @@ def _provider_status_code(exc: Exception) -> int | None:
     match unwrapped:
         case InternalServerError(provider_status_code=int() as provider_status):
             return provider_status
+        case APIError(provider_status_code=int() as provider_status):
+            return provider_status
+        case APIError(status_code=int() as provider_status):
+            return provider_status
         case _:
             return None
 

litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace.py

@@ -26,7 +26,6 @@ from collections.abc import Mapping, Sequence
 from datetime import datetime, timezone
 from types import MappingProxyType
 from typing import Annotated, Final, Protocol, TypedDict, get_args
-from urllib.parse import urlsplit
 
 from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, UploadFile
 from fastapi.responses import JSONResponse, Response
@@ -34,6 +33,7 @@ from pydantic import Json, TypeAdapter
 from typing_extensions import ReadOnly
 
 from litellm._logging import verbose_proxy_logger
+from litellm.litellm_core_utils.url_utils import is_https_url_with_host
 from litellm.models.plugin_upload import PluginUpload
 from litellm.proxy._types import CommonProxyErrors, ProxyErrorTypes, ProxyException, UserAPIKeyAuth
 from litellm.proxy.anthropic_endpoints.claude_code_endpoints.claude_code_skill_access import (
@@ -377,14 +377,6 @@ _VALID_GIT_SUBDIR_PATH_RE: Final = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9._-]*(/[a-
 _VALID_SHA256_RE: Final = re.compile(r"^[0-9a-fA-F]{64}$")
 
 
-def _is_https_url_with_host(url: str) -> bool:
-    try:
-        parts: Final = urlsplit(url)
-    except ValueError:
-        return False
-    return parts.scheme == "https" and bool(parts.hostname)
-
-
 def _validate_plugin_source(source: Mapping[str, str]) -> None:
     """Validate plugin source format, raising HTTPException on invalid input."""
     source_type: Final = source.get("source")
@@ -421,7 +413,7 @@ def _validate_plugin_source(source: Mapping[str, str]) -> None:
                 },
             )
     elif source_type == "archive":
-        if not _is_https_url_with_host(source.get("url", "")):
+        if not is_https_url_with_host(source.get("url", "")):
             raise HTTPException(
                 status_code=400,
                 detail={