Merge remote-tracking branch 'origin/main' into litellm_azure_create_no_blind_retry

# Conflicts:
#	litellm/proxy/common_utils/openai_error_payload.py

.github/workflows/osv-scan.yml

@@ -6,6 +6,8 @@ on:
       - main
       - "litellm_**"
       - "rc/**"
+      - "stable/**"
+      - "lts/**"
   schedule:
     - cron: "23 6 * * *"
   workflow_dispatch:

.github/workflows/test-unit.yml

@@ -5,6 +5,9 @@ on:
     branches:
       - main
       - "litellm_**"
+      - "rc/**"
+      - "stable/**"
+      - "lts/**"
   push:
     branches:
       - main
@@ -280,6 +283,7 @@ jobs:
               tests/unit/llms/e2b
               tests/unit/llms/edenai
               tests/unit/llms/elevenlabs
+              tests/unit/llms/empiriolabs
               tests/unit/llms/exa_ai
               tests/unit/llms/fal_ai
               tests/unit/llms/fastcrw
@@ -973,10 +977,10 @@ jobs:
             dist: loadscope
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        timeout-minutes: ${{ matrix.shard == 'db-and-spend' && 10 || 3 }}
+        timeout-minutes: 3
         with:
           persist-credentials: false
-          fetch-depth: ${{ matrix.shard == 'db-and-spend' && '0' || '1' }}
+          fetch-depth: 1
 
       - name: Detect relevant changes
         id: changes
@@ -1009,13 +1013,11 @@ jobs:
         with:
           artifact: ${{ needs.rust-bridge.outputs.artifact }}
 
-      - name: Install dependencies and check schema
+      - name: Install dependencies
         if: steps.changes.outputs.decision != 'skip'
-        timeout-minutes: 8
+        timeout-minutes: 5
         env:
           RUST_BRIDGE_ARTIFACT: ${{ needs.rust-bridge.outputs.artifact }}
-          UNIT_SHARD: ${{ matrix.shard }}
-          SCHEMA_BASE_REF: origin/${{ github.event.repository.default_branch }}
         run: |
           diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
           if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
@@ -1027,9 +1029,21 @@ jobs:
             uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))"
           fi
           uv run --no-sync python -c 'import os, sys; print(sys.version); assert f"{sys.version_info.major}.{sys.version_info.minor}" == os.environ["UV_PYTHON"]'
-          if [ "$UNIT_SHARD" = "db-and-spend" ]; then
-            uv run --no-sync python scripts/check_schema_compatibility.py --base-ref "$SCHEMA_BASE_REF"
-          fi
+
+      - name: Check schema.prisma against the merge base
+        if: steps.changes.outputs.decision != 'skip' && matrix.shard == 'db-and-spend' && github.event_name == 'pull_request'
+        timeout-minutes: 3
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+          EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/branches/${EVENT_BASE_REF}" --jq .name) || base_ref=$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" --jq .base.ref)
+          merge_base=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${base_ref}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
+          test -n "$merge_base"
+          git fetch --no-tags --depth=1 origin "$merge_base"
+          uv run --no-sync python scripts/check_schema_compatibility.py --base-commit "$merge_base"
 
       - name: Cache Prisma binaries
         if: steps.changes.outputs.decision != 'skip'

README.md

@@ -398,6 +398,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
 | [Deepseek (`deepseek`)](https://docs.litellm.ai/docs/providers/deepseek) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [Eden AI (`edenai`)](https://docs.litellm.ai/docs/providers/edenai) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |  |  |  |
 | [ElevenLabs (`elevenlabs`)](https://docs.litellm.ai/docs/providers/elevenlabs) | ✅ | ✅ | ✅ |  |  | ✅ | ✅ |  |  |  |
+| [EmpirioLabs AI (`empiriolabs`)](https://docs.litellm.ai/docs/providers/empiriolabs) | ✅ |  | ✅ |  |  |  |  |  |  |  |
 | [Empower (`empower`)](https://docs.litellm.ai/docs/providers/empower) | ✅ | ✅ | ✅ |  |  |  |  |  |  |  |
 | [eu/jev Decisions API (`eu_jev`)](https://docs.litellm.ai/docs/pass_through/eu_jev) |  |  |  |  |  |  |  |  |  |  |
 | [Fal AI (`fal_ai`)](https://docs.litellm.ai/docs/providers/fal_ai) | ✅ | ✅ | ✅ |  | ✅ |  |  |  |  |  |

enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py

@@ -200,7 +200,6 @@ async def log_billed_background_response(billed: BilledBackgroundResponse) -> Bi
     unpriced: Final = _without_read_price(billed.response)
     cost: Final = logging_obj.response_cost_calculator(result=unpriced)
     end_time: Final = datetime.now()
-    logging_obj.capture_usage_before_logging(unpriced, logging_obj.start_time, end_time)
     GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
         async_coroutine=logging_obj.async_success_handler(
             result=unpriced, start_time=logging_obj.start_time, end_time=end_time

helm/litellm-helm/Chart.yaml

@@ -18,7 +18,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.1.3
+version: 1.1.4
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm/litellm-helm/README.md

@@ -56,8 +56,8 @@ If `db.useStackgresOperator` is used (not yet implemented):
 
 | `billingMetrics.enabled`    | Enable enterprise billable-request metering. Requires an enterprise license.                                                                                                                                                                  | `false`                   |
 | `billingMetrics.endpoint`   | Collector that the billable-request counter is pushed to.                                                                                                                                                                                    | `https://telemetry.litellm.ai` |
-| `billingMetrics.secretName` | Name of an existing Secret holding the mTLS client certificate, under the keys `tls.crt` and `tls.key`.                                                                                                                                       | `litellm-billing-metrics-mtls` |
-| `billingMetrics.caSecretName` | Name of an existing Secret holding a CA bundle under the key `ca.crt`. Only needed for a private or test collector whose server certificate is not on the public web PKI.                                                                    | `""`                      |
+| `billingMetrics.tokenUrl`   | Cognito OAuth2 client-credentials token endpoint.                                                                                                                                                                                               | `https://litellm-prod.auth.us-west-1.amazoncognito.com/oauth2/token` |
+| `billingMetrics.secretName` | Name of an existing Secret with `client-id` and `client-secret` keys.                                                                                                                                                                           | `litellm-billing-metrics` |
 | `billingMetrics.exportIntervalMs` | How often the counter is pushed, in milliseconds. The proxy defaults to `60000` when unset.                                                                                                                                             | `""`                      |
 
 #### Example `proxy_config` ConfigMap from values (default):
@@ -109,10 +109,10 @@ type: Opaque
 
 #### Enterprise billable-request metering
 
-Enterprise licenses meter billable requests by pushing a counter to LiteLLM's collector over mutual TLS. The chart does not create the client certificate; it mounts one you already hold, read-only, so the private key is never exposed through the environment. Create the Secret under the name the chart expects, then turn the block on:
+Enterprise licenses meter billable requests by pushing a counter to LiteLLM's collector with Cognito bearer authentication. Create a Secret with `client-id` and `client-secret` keys, then enable the chart values:
 
 ```
-kubectl create secret tls litellm-billing-metrics-mtls --cert=client.crt --key=client.key
+kubectl create secret generic litellm-billing-metrics --from-literal=client-id=... --from-literal=client-secret=...
 ```
 
 ```
@@ -120,7 +120,8 @@ billingMetrics:
   enabled: true
 ```
 
-Set `billingMetrics.caSecretName` only when the collector is a private or test one whose server certificate is not on the public web PKI; the production collector needs no CA override. The chart fails the render rather than deploying a proxy that silently never exports, so a missing `secretName` or an emptied `endpoint` surfaces at `helm install` time.
+The endpoint and token URL must use HTTPS. The chart requires both URLs and
+the Secret name when billing metrics are enabled
 
 ### Database Settings
 

helm/litellm-helm/templates/_helpers.tpl

@@ -50,53 +50,27 @@ app.kubernetes.io/name: {{ include "litellm.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
 
-{{/*
-Enterprise billable-request metering. The client certificate identifies the
-deployment to LiteLLM's collector, so it is mounted read-only from an existing
-Secret rather than passed through the environment.
-*/}}
-{{- define "litellm.billingMetrics.certDir" -}}/etc/litellm/billing-mtls{{- end -}}
-{{- define "litellm.billingMetrics.caDir" -}}/etc/litellm/billing-mtls-ca{{- end -}}
-
 {{- define "litellm.billingMetricsEnv" -}}
 - name: LITELLM_BILLING_METRICS_ENDPOINT
   value: {{ required "billingMetrics.endpoint is required when billingMetrics.enabled is true" .Values.billingMetrics.endpoint | quote }}
-- name: LITELLM_BILLING_METRICS_CLIENT_CERT
-  value: {{ printf "%s/tls.crt" (include "litellm.billingMetrics.certDir" .) | quote }}
-- name: LITELLM_BILLING_METRICS_CLIENT_KEY
-  value: {{ printf "%s/tls.key" (include "litellm.billingMetrics.certDir" .) | quote }}
-{{- if .Values.billingMetrics.caSecretName }}
-- name: LITELLM_BILLING_METRICS_CA_CERT
-  value: {{ printf "%s/ca.crt" (include "litellm.billingMetrics.caDir" .) | quote }}
-{{- end }}
+- name: LITELLM_BILLING_METRICS_TOKEN_URL
+  value: {{ required "billingMetrics.tokenUrl is required when billingMetrics.enabled is true" .Values.billingMetrics.tokenUrl | quote }}
+- name: LITELLM_BILLING_METRICS_CLIENT_ID
+  valueFrom:
+    secretKeyRef:
+      name: {{ required "billingMetrics.secretName is required when billingMetrics.enabled is true" .Values.billingMetrics.secretName }}
+      key: client-id
+- name: LITELLM_BILLING_METRICS_CLIENT_SECRET
+  valueFrom:
+    secretKeyRef:
+      name: {{ required "billingMetrics.secretName is required when billingMetrics.enabled is true" .Values.billingMetrics.secretName }}
+      key: client-secret
 {{- with .Values.billingMetrics.exportIntervalMs }}
 - name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS
   value: {{ . | quote }}
 {{- end }}
 {{- end -}}
 
-{{- define "litellm.billingMetricsVolumes" -}}
-- name: billing-metrics-mtls
-  secret:
-    secretName: {{ required "billingMetrics.secretName is required when billingMetrics.enabled is true (an existing Secret with tls.crt and tls.key)" .Values.billingMetrics.secretName }}
-{{- if .Values.billingMetrics.caSecretName }}
-- name: billing-metrics-mtls-ca
-  secret:
-    secretName: {{ .Values.billingMetrics.caSecretName }}
-{{- end }}
-{{- end -}}
-
-{{- define "litellm.billingMetricsVolumeMounts" -}}
-- name: billing-metrics-mtls
-  mountPath: {{ include "litellm.billingMetrics.certDir" . }}
-  readOnly: true
-{{- if .Values.billingMetrics.caSecretName }}
-- name: billing-metrics-mtls-ca
-  mountPath: {{ include "litellm.billingMetrics.caDir" . }}
-  readOnly: true
-{{- end }}
-{{- end -}}
-
 {{/*
 Create the name of the service account to use
 */}}

helm/litellm-helm/templates/deployment.yaml

@@ -148,9 +148,6 @@ spec:
             - name: npm
               mountPath: /.npm
           {{- end }}
-          {{- if .Values.billingMetrics.enabled }}
-            {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }}
-          {{- end }}
           {{- if include "litellm.collector.socketDir" . }}
             - name: collector-socket
               mountPath: {{ include "litellm.collector.socketDir" . }}
@@ -234,9 +231,6 @@ spec:
             items:
               - key: {{ .Values.proxyConfigMap.key | default "config.yaml" }}
                 path: "config.yaml"
-        {{- if .Values.billingMetrics.enabled }}
-        {{- include "litellm.billingMetricsVolumes" . | nindent 8 }}
-        {{- end }}
         {{- if include "litellm.collector.socketDir" . }}
         - name: collector-socket
           emptyDir:

helm/litellm-helm/tests/billing_metrics_tests.yaml

@@ -1,31 +1,21 @@
-suite: test billingMetrics wiring on the proxy deployment
+suite: test bearer-only billingMetrics wiring
 templates:
   - deployment.yaml
   - configmap-litellm.yaml
-  - migrations-job.yaml
 tests:
-  - it: is off by default, adding no env, volume, or mount
+  - it: is disabled by default
     template: deployment.yaml
     asserts:
-      - notContains:
-          path: spec.template.spec.volumes
-          content:
-            name: billing-metrics-mtls
-            secret:
-              secretName: litellm-billing-metrics-mtls
-      - notContains:
-          path: spec.template.spec.containers[0].volumeMounts
-          content:
-            name: billing-metrics-mtls
-            mountPath: /etc/litellm/billing-mtls
-            readOnly: true
       - notContains:
           path: spec.template.spec.containers[0].env
           content:
             name: LITELLM_BILLING_METRICS_ENDPOINT
-            value: https://telemetry.litellm.ai
+      - notContains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics
 
-  - it: renders the endpoint and the mounted cert paths when enabled
+  - it: renders required URLs and Cognito secret references when enabled
     template: deployment.yaml
     set:
       billingMetrics:
@@ -39,119 +29,34 @@ tests:
       - contains:
           path: spec.template.spec.containers[0].env
           content:
-            name: LITELLM_BILLING_METRICS_CLIENT_CERT
-            value: /etc/litellm/billing-mtls/tls.crt
+            name: LITELLM_BILLING_METRICS_TOKEN_URL
+            value: https://litellm-prod.auth.us-west-1.amazoncognito.com/oauth2/token
       - contains:
           path: spec.template.spec.containers[0].env
           content:
-            name: LITELLM_BILLING_METRICS_CLIENT_KEY
-            value: /etc/litellm/billing-mtls/tls.key
-
-  # The conventional Secret name is the default, so enabling the block is enough.
-  - it: mounts the default cert secret read-only alongside the config volume
-    template: deployment.yaml
-    set:
-      billingMetrics:
-        enabled: true
-    asserts:
-      - contains:
-          path: spec.template.spec.volumes
-          content:
-            name: billing-metrics-mtls
-            secret:
-              secretName: litellm-billing-metrics-mtls
-      - contains:
-          path: spec.template.spec.containers[0].volumeMounts
-          content:
-            name: billing-metrics-mtls
-            mountPath: /etc/litellm/billing-mtls
-            readOnly: true
-
-  - it: honours a secretName override
-    template: deployment.yaml
-    set:
-      billingMetrics:
-        enabled: true
-        secretName: my-billing-mtls
-    asserts:
-      - contains:
-          path: spec.template.spec.volumes
-          content:
-            name: billing-metrics-mtls
-            secret:
-              secretName: my-billing-mtls
-      - notContains:
-          path: spec.template.spec.volumes
-          content:
-            name: billing-metrics-mtls
-            secret:
-              secretName: litellm-billing-metrics-mtls
-
-  - it: honours an endpoint override
-    template: deployment.yaml
-    set:
-      billingMetrics:
-        enabled: true
-        endpoint: https://collector.internal:4318
-    asserts:
+            name: LITELLM_BILLING_METRICS_CLIENT_ID
+            valueFrom:
+              secretKeyRef:
+                name: litellm-billing-metrics
+                key: client-id
       - contains:
           path: spec.template.spec.containers[0].env
           content:
-            name: LITELLM_BILLING_METRICS_ENDPOINT
-            value: https://collector.internal:4318
-
-  # The production collector presents a public web-PKI certificate, so the CA
-  # override must stay absent unless a private collector is configured.
-  - it: omits the CA env, volume, and mount when no caSecretName is set
-    template: deployment.yaml
-    set:
-      billingMetrics:
-        enabled: true
-    asserts:
+            name: LITELLM_BILLING_METRICS_CLIENT_SECRET
+            valueFrom:
+              secretKeyRef:
+                name: litellm-billing-metrics
+                key: client-secret
       - notContains:
           path: spec.template.spec.volumes
           content:
-            name: billing-metrics-mtls-ca
-            secret:
-              secretName: billing-ca
+            name: billing-metrics
       - notContains:
           path: spec.template.spec.containers[0].volumeMounts
           content:
-            name: billing-metrics-mtls-ca
-            mountPath: /etc/litellm/billing-mtls-ca
-            readOnly: true
-      - notContains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_BILLING_METRICS_CA_CERT
-            value: /etc/litellm/billing-mtls-ca/ca.crt
+            name: billing-metrics
 
-  - it: mounts the CA secret when caSecretName is set
-    template: deployment.yaml
-    set:
-      billingMetrics:
-        enabled: true
-        caSecretName: billing-ca
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_BILLING_METRICS_CA_CERT
-            value: /etc/litellm/billing-mtls-ca/ca.crt
-      - contains:
-          path: spec.template.spec.volumes
-          content:
-            name: billing-metrics-mtls-ca
-            secret:
-              secretName: billing-ca
-      - contains:
-          path: spec.template.spec.containers[0].volumeMounts
-          content:
-            name: billing-metrics-mtls-ca
-            mountPath: /etc/litellm/billing-mtls-ca
-            readOnly: true
-
-  - it: passes the export interval through only when set
+  - it: renders the export interval only when set
     template: deployment.yaml
     set:
       billingMetrics:
@@ -164,129 +69,7 @@ tests:
             name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS
             value: "5000"
 
-  - it: omits the export interval when unset
-    template: deployment.yaml
-    set:
-      billingMetrics:
-        enabled: true
-    asserts:
-      - notContains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS
-            value: "60000"
-
-  # Kubernetes resolves duplicate env names last-wins, so the chart-owned billing
-  # entries must render after .Values.envVars or a user could silently redirect
-  # the metering export. The three billing entries are the last ones emitted here
-  # (migrationJob, which appends DISABLE_SCHEMA_UPDATE, is off for this case).
-  - it: renders the billing endpoint after envVars so it cannot be shadowed
-    template: deployment.yaml
-    set:
-      migrationJob:
-        enabled: false
-      billingMetrics:
-        enabled: true
-      envVars:
-        LITELLM_BILLING_METRICS_ENDPOINT: https://shadowed.example
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_BILLING_METRICS_ENDPOINT
-            value: https://shadowed.example
-      - equal:
-          path: spec.template.spec.containers[0].env[-3]
-          value:
-            name: LITELLM_BILLING_METRICS_ENDPOINT
-            value: https://telemetry.litellm.ai
-      - equal:
-          path: spec.template.spec.containers[0].env[-2].name
-          value: LITELLM_BILLING_METRICS_CLIENT_CERT
-      - equal:
-          path: spec.template.spec.containers[0].env[-1].name
-          value: LITELLM_BILLING_METRICS_CLIENT_KEY
-
-  - it: keeps user-supplied volumes and mounts alongside the billing secret
-    template: deployment.yaml
-    set:
-      billingMetrics:
-        enabled: true
-      volumes:
-        - name: custom-callbacks
-          configMap:
-            name: my-callbacks
-      volumeMounts:
-        - name: custom-callbacks
-          mountPath: /app/callbacks
-    asserts:
-      - contains:
-          path: spec.template.spec.volumes
-          content:
-            name: custom-callbacks
-            configMap:
-              name: my-callbacks
-      - contains:
-          path: spec.template.spec.volumes
-          content:
-            name: billing-metrics-mtls
-            secret:
-              secretName: litellm-billing-metrics-mtls
-      - contains:
-          path: spec.template.spec.containers[0].volumeMounts
-          content:
-            name: custom-callbacks
-            mountPath: /app/callbacks
-      - contains:
-          path: spec.template.spec.containers[0].volumeMounts
-          content:
-            name: billing-metrics-mtls
-            mountPath: /etc/litellm/billing-mtls
-            readOnly: true
-
-  - it: still mounts the proxy config when enabled
-    template: deployment.yaml
-    set:
-      billingMetrics:
-        enabled: true
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].volumeMounts
-          content:
-            name: litellm-config
-            mountPath: /etc/litellm/config.yaml
-            subPath: config.yaml
-
-  # Only the proxy serves billable traffic. The migrations Job must never mount
-  # the client certificate, and it renders its own env and volumes, so nothing
-  # stops a future edit from wiring the billing include into it by mistake.
-  - it: does not touch the migrations job when enabled
-    template: migrations-job.yaml
-    set:
-      billingMetrics:
-        enabled: true
-    asserts:
-      - notContains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_BILLING_METRICS_ENDPOINT
-            value: https://telemetry.litellm.ai
-      - notExists:
-          path: spec.template.spec.containers[0].volumeMounts
-      - notExists:
-          path: spec.template.spec.volumes
-
-  - it: fails loudly when enabled with an emptied secretName
-    template: deployment.yaml
-    set:
-      billingMetrics:
-        enabled: true
-        secretName: ""
-    asserts:
-      - failedTemplate:
-          errorMessage: billingMetrics.secretName is required when billingMetrics.enabled is true (an existing Secret with tls.crt and tls.key)
-
-  - it: fails loudly when enabled without an endpoint
+  - it: requires the endpoint when enabled
     template: deployment.yaml
     set:
       billingMetrics:
@@ -295,3 +78,23 @@ tests:
     asserts:
       - failedTemplate:
           errorMessage: billingMetrics.endpoint is required when billingMetrics.enabled is true
+
+  - it: requires the token URL when enabled
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        tokenUrl: ""
+    asserts:
+      - failedTemplate:
+          errorMessage: billingMetrics.tokenUrl is required when billingMetrics.enabled is true
+
+  - it: requires the secret name when enabled
+    template: deployment.yaml
+    set:
+      billingMetrics:
+        enabled: true
+        secretName: ""
+    asserts:
+      - failedTemplate:
+          errorMessage: billingMetrics.secretName is required when billingMetrics.enabled is true

helm/litellm-helm/tests/collector_tests.yaml

@@ -191,7 +191,7 @@ tests:
       metricsServer.port: 9090
       billingMetrics.enabled: true
       billingMetrics.endpoint: https://metering.example.com
-      billingMetrics.secretName: billing-mtls
+      billingMetrics.secretName: billing-metrics
     asserts:
       - contains:
           path: spec.template.spec.containers[0].env
@@ -214,9 +214,9 @@ tests:
             name: LITELLM_BILLING_METRICS_ENDPOINT
           any: true
       - notContains:
-          path: spec.template.spec.containers[1].volumeMounts
+          path: spec.template.spec.containers[1].env
           content:
-            name: billing-metrics-mtls
+            name: LITELLM_BILLING_METRICS_CLIENT_SECRET
           any: true
 
   - it: should give the sidecar the same scratch mounts as the proxy on a read-only root

helm/litellm-helm/values.yaml

@@ -154,19 +154,13 @@ masterkeySecretName: ""
 # if set, use this secret key for the master key; otherwise, use the default key
 masterkeySecretKey: ""
 
-# Optional: enterprise billable-request metering. When enabled, the proxy counts
-# successful requests to inference, MCP, and A2A endpoints and pushes them to
-# LiteLLM's collector over mutual TLS. Requires an enterprise license.
-# The client certificate identifies the deployment, so it is mounted read-only
-# from an existing Secret and never passed through the environment.
+# Optional: enterprise billable-request metering. Requires an enterprise license.
 billingMetrics:
   enabled: false
-  endpoint: https://telemetry.litellm.ai   # collector to push the counter to
-  secretName: litellm-billing-metrics-mtls # existing Secret holding tls.crt and tls.key
-  # Only for private or test collectors whose server certificate is not on the
-  # public web PKI. The production collector needs no CA override.
-  caSecretName: ""                         # existing Secret holding ca.crt
-  exportIntervalMs: ""                     # push cadence; the proxy defaults to 60000
+  endpoint: https://telemetry.litellm.ai
+  tokenUrl: https://litellm-prod.auth.us-west-1.amazoncognito.com/oauth2/token
+  secretName: litellm-billing-metrics
+  exportIntervalMs: ""
 
 proxyConfigMap:
   # when true, creates a new configmap

helm/litellm/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: litellm
 description: LiteLLM componentized — gateway, UI backend, and UI as separate services
 type: application
-version: 0.1.0
+version: 0.1.1
 appVersion: "0.1.0"
 annotations:
   org.opencontainers.image.source: "https://github.com/BerriAI/litellm"

helm/litellm/README.md

@@ -22,6 +22,23 @@ backend:
   gatewayUrl: https://gateway.example.com
 ```
 
+## Enterprise billing metrics
+
+Billing metrics use Cognito bearer authentication. Create the Secret with
+`client-id` and `client-secret` keys, then enable billing metrics:
+
+```bash
+kubectl create secret generic litellm-billing-metrics --from-literal=client-id=... --from-literal=client-secret=...
+```
+
+```yaml
+billingMetrics:
+  enabled: true
+```
+
+The chart requires an HTTPS endpoint, an HTTPS token URL, and the Secret name
+when billing metrics are enabled
+
 ## Routes to another Service
 
 For a shared hostname, `ingress.extraPaths` can route to a Service in the same namespace using `serviceName` and `servicePort`. The port can be a number or a named Service port. Omit `service` when using these fields

helm/litellm/templates/_helpers.tpl

@@ -58,56 +58,27 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
 helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" }}
 {{- end -}}
 
-{{/*
-Enterprise billable-request metering. Wired into gateway and backend, not the
-migrations job. The gateway serves nearly all billable traffic, but the backend
-keeps the named-server MCP transport (/{mcp_server_name}/mcp), which writes a
-SpendLogs row, so metering only the gateway would silently drop that traffic.
-The client certificate identifies the deployment to LiteLLM's collector, so it is
-mounted read-only from an existing Secret rather than passed through the
-environment.
-*/}}
-{{- define "litellm.billingMetrics.certDir" -}}/etc/litellm/billing-mtls{{- end -}}
-{{- define "litellm.billingMetrics.caDir" -}}/etc/litellm/billing-mtls-ca{{- end -}}
-
 {{- define "litellm.billingMetricsEnv" -}}
 - name: LITELLM_BILLING_METRICS_ENDPOINT
   value: {{ required "billingMetrics.endpoint is required when billingMetrics.enabled is true" .Values.billingMetrics.endpoint | quote }}
-- name: LITELLM_BILLING_METRICS_CLIENT_CERT
-  value: {{ printf "%s/tls.crt" (include "litellm.billingMetrics.certDir" .) | quote }}
-- name: LITELLM_BILLING_METRICS_CLIENT_KEY
-  value: {{ printf "%s/tls.key" (include "litellm.billingMetrics.certDir" .) | quote }}
-{{- if .Values.billingMetrics.caSecretName }}
-- name: LITELLM_BILLING_METRICS_CA_CERT
-  value: {{ printf "%s/ca.crt" (include "litellm.billingMetrics.caDir" .) | quote }}
-{{- end }}
+- name: LITELLM_BILLING_METRICS_TOKEN_URL
+  value: {{ required "billingMetrics.tokenUrl is required when billingMetrics.enabled is true" .Values.billingMetrics.tokenUrl | quote }}
+- name: LITELLM_BILLING_METRICS_CLIENT_ID
+  valueFrom:
+    secretKeyRef:
+      name: {{ required "billingMetrics.secretName is required when billingMetrics.enabled is true" .Values.billingMetrics.secretName }}
+      key: client-id
+- name: LITELLM_BILLING_METRICS_CLIENT_SECRET
+  valueFrom:
+    secretKeyRef:
+      name: {{ required "billingMetrics.secretName is required when billingMetrics.enabled is true" .Values.billingMetrics.secretName }}
+      key: client-secret
 {{- with .Values.billingMetrics.exportIntervalMs }}
 - name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS
   value: {{ . | quote }}
 {{- end }}
 {{- end -}}
 
-{{- define "litellm.billingMetricsVolumes" -}}
-- name: billing-metrics-mtls
-  secret:
-    secretName: {{ required "billingMetrics.secretName is required when billingMetrics.enabled is true (an existing Secret with tls.crt and tls.key)" .Values.billingMetrics.secretName }}
-{{- if .Values.billingMetrics.caSecretName }}
-- name: billing-metrics-mtls-ca
-  secret:
-    secretName: {{ .Values.billingMetrics.caSecretName }}
-{{- end }}
-{{- end -}}
-
-{{- define "litellm.billingMetricsVolumeMounts" -}}
-- name: billing-metrics-mtls
-  mountPath: {{ include "litellm.billingMetrics.certDir" . }}
-  readOnly: true
-{{- if .Values.billingMetrics.caSecretName }}
-- name: billing-metrics-mtls-ca
-  mountPath: {{ include "litellm.billingMetrics.caDir" . }}
-  readOnly: true
-{{- end }}
-{{- end -}}
 
 {{/*
 Per-component selector labels — used in both Service selectors and Deployment matchLabels.

helm/litellm/templates/backend/deployment.yaml

@@ -79,16 +79,13 @@ spec:
             {{- include "litellm.billingMetricsEnv" . | nindent 12 }}
             {{- end }}
           {{- include "litellm.envFrom" .Values.backend | nindent 10 }}
-          {{- if or .Values.gateway.config.create .Values.backend.volumeMounts .Values.billingMetrics.enabled }}
+          {{- if or .Values.gateway.config.create .Values.backend.volumeMounts }}
           volumeMounts:
             {{- if .Values.gateway.config.create }}
             - name: gateway-config
               mountPath: /app/config/config.yaml
               subPath: config.yaml
             {{- end }}
-            {{- if .Values.billingMetrics.enabled }}
-            {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }}
-            {{- end }}
             {{- with .Values.backend.volumeMounts }}
             {{- toYaml . | nindent 12 }}
             {{- end }}
@@ -114,16 +111,13 @@ spec:
       {{- with .Values.backend.extraContainers }}
         {{- tpl (toYaml .) $ | nindent 8 }}
       {{- end }}
-      {{- if or .Values.gateway.config.create .Values.backend.volumes .Values.billingMetrics.enabled }}
+      {{- if or .Values.gateway.config.create .Values.backend.volumes }}
       volumes:
         {{- if .Values.gateway.config.create }}
         - name: gateway-config
           configMap:
             name: {{ include "litellm.gateway.fullname" . }}-config
         {{- end }}
-        {{- if .Values.billingMetrics.enabled }}
-        {{- include "litellm.billingMetricsVolumes" . | nindent 8 }}
-        {{- end }}
         {{- with .Values.backend.volumes }}
         {{- toYaml . | nindent 8 }}
         {{- end }}

helm/litellm/templates/gateway/deployment.yaml

@@ -86,7 +86,7 @@ spec:
             {{- include "litellm.gateway.collectorEnv" . | nindent 12 }}
             {{- end }}
           {{- include "litellm.envFrom" .Values.gateway | nindent 10 }}
-          {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled (include "litellm.gateway.collectorSocketDir" .) }}
+          {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts .Values.gateway.metricsServer.enabled (include "litellm.gateway.collectorSocketDir" .) }}
           volumeMounts:
             {{- if .Values.gateway.config.create }}
             - name: gateway-config
@@ -101,9 +101,6 @@ spec:
             - name: collector-socket
               mountPath: {{ include "litellm.gateway.collectorSocketDir" . }}
             {{- end }}
-            {{- if .Values.billingMetrics.enabled }}
-            {{- include "litellm.billingMetricsVolumeMounts" . | nindent 12 }}
-            {{- end }}
             {{- with .Values.gateway.volumeMounts }}
             {{- toYaml . | nindent 12 }}
             {{- end }}
@@ -206,7 +203,7 @@ spec:
       {{- with .Values.gateway.extraContainers }}
         {{- tpl (toYaml .) $ | nindent 8 }}
       {{- end }}
-      {{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.billingMetrics.enabled .Values.gateway.metricsServer.enabled (include "litellm.gateway.collectorSocketDir" .) }}
+      {{- if or .Values.gateway.config.create .Values.gateway.volumes .Values.gateway.metricsServer.enabled (include "litellm.gateway.collectorSocketDir" .) }}
       volumes:
         {{- if .Values.gateway.config.create }}
         - name: gateway-config
@@ -222,9 +219,6 @@ spec:
           emptyDir:
             sizeLimit: 1Mi
         {{- end }}
-        {{- if .Values.billingMetrics.enabled }}
-        {{- include "litellm.billingMetricsVolumes" . | nindent 8 }}
-        {{- end }}
         {{- with .Values.gateway.volumes }}
         {{- toYaml . | nindent 8 }}
         {{- end }}

helm/litellm/tests/billing_metrics_tests.yaml

@@ -1,39 +1,30 @@
-suite: test billingMetrics wiring on gateway and backend
+suite: test bearer-only billingMetrics wiring
 templates:
   - gateway/deployment.yaml
   - gateway/configmap.yaml
   - backend/deployment.yaml
+  - backend/configmap.yaml
   - migrations-job.yaml
 values:
   - ./values/required.yaml
 tests:
-  - it: is off by default, adding no env, volume, or mount
+  - it: is disabled by default
     template: gateway/deployment.yaml
     asserts:
-      - notContains:
-          path: spec.template.spec.volumes
-          content:
-            name: billing-metrics-mtls
-            secret:
-              secretName: billing-mtls
-      - equal:
-          path: spec.template.spec.containers[0].volumeMounts
-          value:
-            - name: gateway-config
-              mountPath: /app/config/config.yaml
-              subPath: config.yaml
       - notContains:
           path: spec.template.spec.containers[0].env
           content:
             name: LITELLM_BILLING_METRICS_ENDPOINT
-            value: https://telemetry.litellm.ai
+      - notContains:
+          path: spec.template.spec.volumes
+          content:
+            name: billing-metrics
 
-  - it: renders the endpoint and the mounted cert paths when enabled
+  - it: wires gateway URLs and bearer credentials from Secret keys
     template: gateway/deployment.yaml
     set:
       billingMetrics:
         enabled: true
-        secretName: billing-mtls
     asserts:
       - contains:
           path: spec.template.spec.containers[0].env
@@ -43,87 +34,68 @@ tests:
       - contains:
           path: spec.template.spec.containers[0].env
           content:
-            name: LITELLM_BILLING_METRICS_CLIENT_CERT
-            value: /etc/litellm/billing-mtls/tls.crt
+            name: LITELLM_BILLING_METRICS_TOKEN_URL
+            value: https://litellm-prod.auth.us-west-1.amazoncognito.com/oauth2/token
       - contains:
           path: spec.template.spec.containers[0].env
           content:
-            name: LITELLM_BILLING_METRICS_CLIENT_KEY
-            value: /etc/litellm/billing-mtls/tls.key
-
-  - it: mounts the cert secret read-only alongside the config volume
-    template: gateway/deployment.yaml
-    set:
-      billingMetrics:
-        enabled: true
-        secretName: billing-mtls
-    asserts:
+            name: LITELLM_BILLING_METRICS_CLIENT_ID
+            valueFrom:
+              secretKeyRef:
+                name: litellm-billing-metrics
+                key: client-id
       - contains:
-          path: spec.template.spec.volumes
+          path: spec.template.spec.containers[0].env
           content:
-            name: billing-metrics-mtls
-            secret:
-              secretName: billing-mtls
-      - contains:
-          path: spec.template.spec.containers[0].volumeMounts
-          content:
-            name: billing-metrics-mtls
-            mountPath: /etc/litellm/billing-mtls
-            readOnly: true
-
-  # The production collector presents a public web-PKI certificate, so the CA
-  # override must stay absent unless a private collector is configured.
-  - it: omits the CA env, volume, and mount when no caSecretName is set
-    template: gateway/deployment.yaml
-    set:
-      billingMetrics:
-        enabled: true
-        secretName: billing-mtls
-    asserts:
+            name: LITELLM_BILLING_METRICS_CLIENT_SECRET
+            valueFrom:
+              secretKeyRef:
+                name: litellm-billing-metrics
+                key: client-secret
       - notContains:
           path: spec.template.spec.volumes
           content:
-            name: billing-metrics-mtls-ca
-            secret:
-              secretName: billing-ca
+            name: billing-metrics
       - notContains:
-          path: spec.template.spec.containers[0].env
+          path: spec.template.spec.containers[0].volumeMounts
           content:
-            name: LITELLM_BILLING_METRICS_CA_CERT
-            value: /etc/litellm/billing-mtls-ca/ca.crt
+            name: billing-metrics
 
-  - it: mounts the CA secret when caSecretName is set
-    template: gateway/deployment.yaml
+  - it: wires billing metrics on the backend without volumes
+    template: backend/deployment.yaml
     set:
       billingMetrics:
         enabled: true
-        secretName: billing-mtls
-        caSecretName: billing-ca
     asserts:
       - contains:
           path: spec.template.spec.containers[0].env
           content:
-            name: LITELLM_BILLING_METRICS_CA_CERT
-            value: /etc/litellm/billing-mtls-ca/ca.crt
-      - contains:
+            name: LITELLM_BILLING_METRICS_CLIENT_SECRET
+            valueFrom:
+              secretKeyRef:
+                name: litellm-billing-metrics
+                key: client-secret
+      - notContains:
           path: spec.template.spec.volumes
           content:
-            name: billing-metrics-mtls-ca
-            secret:
-              secretName: billing-ca
-      - contains:
-          path: spec.template.spec.containers[0].volumeMounts
-          content:
-            name: billing-metrics-mtls-ca
-            mountPath: /etc/litellm/billing-mtls-ca
-            readOnly: true
+            name: billing-metrics
 
-  - it: passes the export interval through only when set
+  - it: does not add billing metrics to migrations
+    template: migrations-job.yaml
+    set:
+      billingMetrics:
+        enabled: true
+    asserts:
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: LITELLM_BILLING_METRICS_ENDPOINT
+
+  - it: renders an optional export interval
     template: gateway/deployment.yaml
     set:
       billingMetrics:
         enabled: true
-        secretName: billing-mtls
         exportIntervalMs: 5000
     asserts:
       - contains:
@@ -132,102 +104,27 @@ tests:
             name: LITELLM_BILLING_METRICS_EXPORT_INTERVAL_MS
             value: "5000"
 
-  - it: keeps user-supplied gateway volumes alongside the billing secret
+  - it: requires the endpoint when enabled
     template: gateway/deployment.yaml
     set:
       billingMetrics:
         enabled: true
-        secretName: billing-mtls
-      gateway.volumes:
-        - name: custom-callbacks
-          configMap:
-            name: my-callbacks
-      gateway.volumeMounts:
-        - name: custom-callbacks
-          mountPath: /app/callbacks
+        endpoint: ""
     asserts:
-      - contains:
-          path: spec.template.spec.volumes
-          content:
-            name: custom-callbacks
-            configMap:
-              name: my-callbacks
-      - contains:
-          path: spec.template.spec.volumes
-          content:
-            name: billing-metrics-mtls
-            secret:
-              secretName: billing-mtls
+      - failedTemplate:
+          errorMessage: billingMetrics.endpoint is required when billingMetrics.enabled is true
 
-  # The backend keeps the named-server MCP transport (/{mcp_server_name}/mcp),
-  # which writes a SpendLogs row, so it must meter too or that traffic is lost.
-  - it: meters the backend as well, since it serves the MCP transport
-    template: backend/deployment.yaml
-    set:
-      billingMetrics:
-        enabled: true
-        secretName: billing-mtls
-    asserts:
-      - contains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_BILLING_METRICS_ENDPOINT
-            value: https://telemetry.litellm.ai
-      - contains:
-          path: spec.template.spec.containers[0].volumeMounts
-          content:
-            name: billing-metrics-mtls
-            mountPath: /etc/litellm/billing-mtls
-            readOnly: true
-      - contains:
-          path: spec.template.spec.volumes
-          content:
-            name: billing-metrics-mtls
-            secret:
-              secretName: billing-mtls
-
-  - it: leaves the backend alone when metering is off
-    template: backend/deployment.yaml
-    asserts:
-      - notContains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_BILLING_METRICS_ENDPOINT
-            value: https://telemetry.litellm.ai
-
-  # The migrations job runs prisma and serves no traffic; it must never receive
-  # the client key.
-  - it: never mounts the billing cert on the migrations job
-    template: migrations-job.yaml
-    set:
-      billingMetrics:
-        enabled: true
-        secretName: billing-mtls
-    asserts:
-      - notContains:
-          path: spec.template.spec.containers[0].env
-          content:
-            name: LITELLM_BILLING_METRICS_ENDPOINT
-            value: https://telemetry.litellm.ai
-      - isNull:
-          path: spec.template.spec.volumes
-
-  # The conventional Secret name is the default, so enabling metering needs no
-  # secretName at all; the guard below only fires on an explicitly blanked one.
-  - it: uses the conventional secret name by default
+  - it: requires the token URL when enabled
     template: gateway/deployment.yaml
     set:
       billingMetrics:
         enabled: true
+        tokenUrl: ""
     asserts:
-      - contains:
-          path: spec.template.spec.volumes
-          content:
-            name: billing-metrics-mtls
-            secret:
-              secretName: litellm-billing-metrics-mtls
+      - failedTemplate:
+          errorMessage: billingMetrics.tokenUrl is required when billingMetrics.enabled is true
 
-  - it: fails loudly when the secretName is explicitly blanked
+  - it: requires the secret name when enabled
     template: gateway/deployment.yaml
     set:
       billingMetrics:
@@ -235,15 +132,4 @@ tests:
         secretName: ""
     asserts:
       - failedTemplate:
-          errorMessage: billingMetrics.secretName is required when billingMetrics.enabled is true (an existing Secret with tls.crt and tls.key)
-
-  - it: fails loudly when enabled without an endpoint
-    template: gateway/deployment.yaml
-    set:
-      billingMetrics:
-        enabled: true
-        endpoint: ""
-        secretName: billing-mtls
-    asserts:
-      - failedTemplate:
-          errorMessage: billingMetrics.endpoint is required when billingMetrics.enabled is true
+          errorMessage: billingMetrics.secretName is required when billingMetrics.enabled is true

helm/litellm/tests/migration_job_tests.yaml

@@ -91,7 +91,7 @@ tests:
             app.kubernetes.io/name: litellm
             app.kubernetes.io/instance: RELEASE-NAME
             app.kubernetes.io/managed-by: Helm
-            helm.sh/chart: litellm-0.1.0
+            helm.sh/chart: litellm-0.1.1
             app.kubernetes.io/component: migrations
 
   - it: renders pod-level and container-level securityContext in their own scopes

helm/litellm/values.yaml

@@ -180,23 +180,14 @@ masterKey:
   secretKey: master-key
 
 # Optional: enterprise billable-request metering. When enabled, the gateway and
-# backend count successful requests to inference, MCP, and A2A endpoints and push
-# them to LiteLLM's collector over mutual TLS. Both components serve billable
-# routes: the backend keeps the named-server MCP transport. Requires an
-# enterprise license. The client certificate identifies the deployment, so it is
-# mounted read-only from an existing Secret and never passed through the env.
+# backend count successful requests to inference, MCP, and A2A endpoints. Requires
+# an enterprise license.
 billingMetrics:
   enabled: false
-  endpoint: https://telemetry.litellm.ai              # collector to push the counter to
-  # An existing Secret holding the client certificate under tls.crt and its key
-  # under tls.key, usually created from the onboarding artifact. The default is
-  # the conventional name, so the common path is to create that Secret and set
-  # enabled: true. Override only if yours is named differently.
-  secretName: litellm-billing-metrics-mtls
-  # Only for private or test collectors whose server certificate is not on the
-  # public web PKI. The production collector needs no CA override.
-  caSecretName: ""                                    # existing Secret holding ca.crt
-  exportIntervalMs: ""                                # push cadence; the proxy defaults to 60000
+  endpoint: https://telemetry.litellm.ai
+  tokenUrl: https://litellm-prod.auth.us-west-1.amazoncognito.com/oauth2/token
+  secretName: litellm-billing-metrics
+  exportIntervalMs: ""
 
 # External Postgres connection. 
 database:

litellm-proxy-extras/litellm_proxy_extras/migrations/20261104000000_skill_zip_upload/migration.sql

@@ -0,0 +1,9 @@
+SET LOCAL lock_timeout = '10s';
+
+CREATE TABLE IF NOT EXISTS "LiteLLM_ClaudeCodePluginUpload" (
+    "plugin_name" TEXT NOT NULL,
+    "content" BYTEA NOT NULL,
+    CONSTRAINT "LiteLLM_ClaudeCodePluginUpload_pkey" PRIMARY KEY ("plugin_name"),
+    CONSTRAINT "LiteLLM_ClaudeCodePluginUpload_plugin_name_fkey" FOREIGN KEY ("plugin_name")
+        REFERENCES "LiteLLM_ClaudeCodePluginTable"("name") ON DELETE CASCADE ON UPDATE CASCADE
+);

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -1615,9 +1615,18 @@ model LiteLLM_ClaudeCodePluginTable {
   updated_at    DateTime? @default(now()) @updatedAt
   created_by    String?
 
+  uploaded_archive LiteLLM_ClaudeCodePluginUpload?
+
   @@map("LiteLLM_ClaudeCodePluginTable")
 }
 
+model LiteLLM_ClaudeCodePluginUpload {
+  plugin_name String @id
+  content     Bytes
+  plugin      LiteLLM_ClaudeCodePluginTable @relation(fields: [plugin_name], references: [name], onDelete: Cascade, onUpdate: Cascade)
+}
+
+
 // User/team-scoped memory store with a GLOBAL unique key.
 // `value` is a string (typically markdown/text meant for LLM context);
 // `metadata` is an optional JSON envelope for structured tags without schema changes.

litellm-rust/Cargo.lock

@@ -3389,15 +3389,19 @@ version = "0.1.0"
 dependencies = [
  "aws-config",
  "aws-credential-types",
+ "aws-runtime",
  "aws-sdk-sts",
  "aws-sigv4",
  "aws-smithy-runtime-api",
+ "aws-smithy-types",
  "aws-types",
  "bytes",
  "futures-util",
  "litellm-auth-types",
  "litellm-error",
  "litellm-http",
+ "litellm-secret-value",
+ "litellm-secrets",
  "moka",
  "reqwest",
  "rstest",
@@ -3406,6 +3410,7 @@ dependencies = [
  "sha2 0.10.9",
  "thiserror 2.0.19",
  "tokio",
+ "url",
  "veil",
 ]
 
@@ -3418,6 +3423,7 @@ dependencies = [
  "litellm-auth-types",
  "litellm-error",
  "litellm-secret-value",
+ "litellm-secrets",
  "moka",
  "rstest",
  "serde_json",
@@ -3431,6 +3437,7 @@ dependencies = [
 name = "litellm-auth-gcp"
 version = "0.1.0"
 dependencies = [
+ "axum",
  "base64 0.22.1",
  "gcp_auth",
  "google-cloud-auth",
@@ -3440,10 +3447,12 @@ dependencies = [
  "litellm-secret-value",
  "moka",
  "rstest",
+ "rustls 0.23.42",
  "serde",
  "serde_json",
  "sha2 0.10.9",
  "tempfile",
+ "time",
  "tokio",
  "veil",
 ]

litellm-rust/crates/auth-aws/Cargo.toml

@@ -11,14 +11,16 @@ litellm-error.workspace = true
 litellm-http.workspace = true
 reqwest.workspace = true
 
-moka = { workspace = true, features = ["sync"] }
+moka.workspace = true
+litellm-secret-value.workspace = true
 serde.workspace = true
 serde_json = { workspace = true, features = ["preserve_order"] }
 sha2.workspace = true
 thiserror.workspace = true
+url.workspace = true
 veil.workspace = true
 
-aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio"] }
+aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio", "credentials-process"] }
 aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"] }
 aws-sdk-sts = { version = "1.108.0", default-features = false, features = ["rustls", "rt-tokio"] }
 aws-sigv4 = "1.5.1"
@@ -26,6 +28,9 @@ aws-types = "1.4.0"
 aws-smithy-runtime-api = "1.13.0"
 
 [dev-dependencies]
+litellm-secrets.workspace = true
+aws-smithy-types = "1.6.4"
+aws-runtime = { version = "1.9.4", default-features = false }
 rstest.workspace = true
 litellm-http = { workspace = true, features = ["test-support"] }
 bytes.workspace = true

litellm-rust/crates/auth-aws/src/constants.rs

@@ -11,8 +11,8 @@ pub const AWS_SESSION_NAME: &str = AwsParams::SESSION_NAME.env[0];
 pub const AWS_PROFILE_NAME: &str = AwsParams::PROFILE_NAME.env[0];
 pub const AWS_ROLE_NAME: &str = AwsParams::ROLE_NAME.env[0];
 pub const AWS_WEB_IDENTITY_TOKEN: &str = AwsParams::WEB_IDENTITY_TOKEN.env[0];
-pub const AWS_ROLE_ARN: &str = "AWS_ROLE_ARN";
-pub const AWS_WEB_IDENTITY_TOKEN_FILE: &str = "AWS_WEB_IDENTITY_TOKEN_FILE";
+pub const AWS_ROLE_ARN: &str = AwsParams::ROLE_ARN_ENV;
+pub const AWS_WEB_IDENTITY_TOKEN_FILE: &str = AwsParams::WEB_IDENTITY_TOKEN_FILE_ENV;
 pub const AWS_STS_ENDPOINT: &str = AwsParams::STS_ENDPOINT.env[0];
 pub const AWS_EXTERNAL_ID: &str = AwsParams::EXTERNAL_ID.env[0];
 pub const AWS_BEARER_TOKEN_BEDROCK: &str = "AWS_BEARER_TOKEN_BEDROCK";

litellm-rust/crates/auth-aws/src/error.rs

@@ -2,6 +2,8 @@ use litellm_error::Source;
 
 #[derive(Clone, Debug, thiserror::Error, PartialEq, Eq)]
 pub enum Error {
+    #[error("AWS credential provider returned expired credentials")]
+    ExpiredCredentials,
     #[error("could not load AWS {flow} credentials")]
     Credentials {
         flow: CredentialFlow,

litellm-rust/crates/auth-aws/src/signer.rs

@@ -75,7 +75,8 @@ impl RequestSigner for SigV4Signer {
         )
         .map_err(|error| match error {
             Error::NonAsciiHeader { .. } => litellm_http::Error::unsignable_header(error),
-            Error::Credentials { .. }
+            Error::ExpiredCredentials
+            | Error::Credentials { .. }
             | Error::MissingWebIdentityCredentials
             | Error::WebIdentityExpiration(_)
             | Error::Signing { .. } => litellm_http::Error::signature(error),

litellm-rust/crates/auth-azure/Cargo.toml

@@ -8,17 +8,19 @@ repository.workspace = true
 [dependencies]
 litellm-secret-value.workspace = true
 litellm-auth-types.workspace = true
+litellm-error.workspace = true
 
 moka.workspace = true
 serde_json.workspace = true
 sha2.workspace = true
 strum.workspace = true
+tokio.workspace = true
 url.workspace = true
 
 azure_core = "1.0.0"
 azure_identity = { version = "1.0.0", features = ["tokio"] }
 
 [dev-dependencies]
-litellm-error.workspace = true
+litellm-secrets.workspace = true
 rstest.workspace = true
-tokio.workspace = true
+tokio = { workspace = true, features = ["test-util"] }

litellm-rust/crates/auth-azure/src/lib.rs

@@ -3,5 +3,5 @@ mod native;
 mod resolve;
 mod types;
 
-pub use resolve::{AzureAuthService, SECRET_NAMES};
+pub use resolve::{AzureAuthService, AzureIdentity, SECRET_NAMES};
 pub use types::{AzureAuthInputs, ConfigValue};

litellm-rust/crates/auth-azure/src/native.rs

@@ -59,6 +59,7 @@ pub(crate) enum NativeAzureRequest {
 pub(crate) struct ValidatedAzureRequest {
     request: NativeAzureRequest,
     credential_source: InputSource,
+    probe_imds: bool,
 }
 
 impl ValidatedAzureRequest {
@@ -68,9 +69,18 @@ impl ValidatedAzureRequest {
         Ok(Self {
             request,
             credential_source,
+            probe_imds: false,
         })
     }
 
+    pub(crate) fn with_imds_probe(self, enabled: bool) -> Self {
+        Self {
+            probe_imds: enabled
+                && matches!(self.request, NativeAzureRequest::ManagedIdentity { .. }),
+            ..self
+        }
+    }
+
     pub(crate) fn credential_source(&self) -> InputSource {
         self.credential_source
     }
@@ -87,9 +97,34 @@ impl ValidatedAzureRequest {
     }
 }
 
+pub(crate) enum CredentialAttempt {
+    Resolved(ResolvedCredential),
+    Unavailable(Error),
+    Failed(Error),
+}
+
+impl CredentialAttempt {
+    pub(crate) fn into_result(self) -> Result<ResolvedCredential, Error> {
+        match self {
+            Self::Resolved(credential) => Ok(credential),
+            Self::Unavailable(error) | Self::Failed(error) => Err(error),
+        }
+    }
+}
+
 pub(crate) struct NativeAzureTokenAcquirer {
     cache: AzureCredentialProviderCache,
     transport: Option<azure_core::http::Transport>,
+    imds_available: tokio::sync::Mutex<ImdsAvailability>,
+}
+
+enum ImdsAvailability {
+    Unknown,
+    Available,
+    Unavailable {
+        retry_at: tokio::time::Instant,
+        error: Error,
+    },
 }
 
 impl Default for NativeAzureTokenAcquirer {
@@ -103,6 +138,7 @@ impl NativeAzureTokenAcquirer {
         Self {
             cache: AzureCredentialProviderCache::new(cache_capacity),
             transport: None,
+            imds_available: tokio::sync::Mutex::new(ImdsAvailability::Unknown),
         }
     }
 
@@ -114,10 +150,51 @@ impl NativeAzureTokenAcquirer {
         Self {
             cache: AzureCredentialProviderCache::new(cache_capacity),
             transport: Some(transport),
+            imds_available: tokio::sync::Mutex::new(ImdsAvailability::Unknown),
         }
     }
 
-    pub(crate) async fn acquire(
+    pub(crate) async fn acquire(&self, request: ValidatedAzureRequest) -> CredentialAttempt {
+        let discover_imds = request.probe_imds;
+        if discover_imds && let Err(error) = self.discover_imds().await {
+            return CredentialAttempt::Unavailable(error);
+        }
+        match self.acquire_token(request).await {
+            Ok(credential) => CredentialAttempt::Resolved(credential),
+            Err(error) => {
+                let missing_identity = discover_imds && unassigned_identity(&error);
+                if missing_identity {
+                    CredentialAttempt::Unavailable(error)
+                } else {
+                    CredentialAttempt::Failed(error)
+                }
+            }
+        }
+    }
+
+    async fn discover_imds(&self) -> Result<(), Error> {
+        let mut availability = self.imds_available.lock().await;
+        match &*availability {
+            ImdsAvailability::Available => return Ok(()),
+            ImdsAvailability::Unavailable { retry_at, error }
+                if tokio::time::Instant::now() < *retry_at =>
+            {
+                return Err(error.clone());
+            }
+            ImdsAvailability::Unknown | ImdsAvailability::Unavailable { .. } => {}
+        }
+        let result = probe_imds(self.transport.clone().unwrap_or_default()).await;
+        *availability = match &result {
+            Ok(()) => ImdsAvailability::Available,
+            Err(error) => ImdsAvailability::Unavailable {
+                retry_at: tokio::time::Instant::now() + Duration::from_secs(30),
+                error: error.clone(),
+            },
+        };
+        result
+    }
+
+    async fn acquire_token(
         &self,
         request: ValidatedAzureRequest,
     ) -> Result<ResolvedCredential, Error> {
@@ -148,6 +225,54 @@ impl NativeAzureTokenAcquirer {
     }
 }
 
+fn unassigned_identity(error: &(dyn std::error::Error + 'static)) -> bool {
+    std::iter::successors(Some(error), |cause| cause.source()).any(|cause| {
+        cause
+            .downcast_ref::<azure_core::Error>()
+            .or_else(|| {
+                cause
+                    .downcast_ref::<litellm_error::Source>()?
+                    .downcast_ref::<azure_core::Error>()
+            })
+            .is_some_and(|error| {
+                error.http_status() == Some(azure_core::http::StatusCode::BadRequest)
+            })
+    })
+}
+
+async fn probe_imds(transport: azure_core::http::Transport) -> Result<(), Error> {
+    use azure_core::http::{Context, Method, Request, Url};
+
+    let endpoint =
+        Url::parse("http://169.254.169.254/metadata/identity/oauth2/token?api-version=2019-08-01")
+            .map_err(|error| {
+                Error::InvalidConfiguration(ErrorDetail::failed("Azure IMDS probe endpoint", error))
+            })?;
+    let mut request = Request::new(endpoint, Method::Get);
+    let response = tokio::time::timeout(
+        Duration::from_secs(1),
+        transport.send(&Context::new(), &mut request),
+    )
+    .await
+    .map_err(|error| {
+        Error::CredentialAcquisition(ErrorDetail::failed("Azure IMDS availability probe", error))
+    })?
+    .map_err(|error| {
+        Error::CredentialAcquisition(ErrorDetail::failed("Azure IMDS availability probe", error))
+    })?;
+    // Headerless IMDS discovery expects 400, checked 2026-11-01
+    // https://learn.microsoft.com/en-us/azure/virtual-machines/instance-metadata-service#errors-and-debugging
+    if response.status() != azure_core::http::StatusCode::BadRequest {
+        return Err(Error::CredentialAcquisition(
+            ErrorDetail::UnexpectedStatus {
+                operation: "Azure IMDS availability probe",
+                status: u16::from(response.status()),
+            },
+        ));
+    }
+    Ok(())
+}
+
 impl NativeAzureRequest {
     fn scope(&self) -> &str {
         match self {
@@ -493,6 +618,7 @@ impl ClientAssertion for StaticAssertion {
 #[cfg(test)]
 mod tests {
     use std::sync::{Arc, Mutex};
+    use std::time::Duration;
 
     use azure_core::http::headers::Headers;
     use azure_core::http::{AsyncRawResponse, HttpClient, Request, StatusCode, Transport};
@@ -569,6 +695,296 @@ mod tests {
         }
     }
 
+    #[derive(Debug)]
+    struct AvailableImdsClient {
+        probe_status: StatusCode,
+        probe_delay: Duration,
+        recover_after: Option<usize>,
+        response_status: Option<StatusCode>,
+        probes: std::sync::atomic::AtomicUsize,
+    }
+
+    impl Default for AvailableImdsClient {
+        fn default() -> Self {
+            Self {
+                probe_status: StatusCode::BadRequest,
+                probe_delay: Duration::ZERO,
+                recover_after: None,
+                response_status: None,
+                probes: std::sync::atomic::AtomicUsize::new(0),
+            }
+        }
+    }
+
+    impl HttpClient for AvailableImdsClient {
+        fn execute_request<'life0, 'life1, 'async_trait>(
+            &'life0 self,
+            request: &'life1 Request,
+        ) -> std::pin::Pin<
+            Box<dyn std::future::Future<Output = Result<AsyncRawResponse>> + Send + 'async_trait>,
+        >
+        where
+            'life0: 'async_trait,
+            'life1: 'async_trait,
+            Self: 'async_trait,
+        {
+            Box::pin(async move {
+                if request
+                    .headers()
+                    .get_optional_str(&azure_core::http::headers::HeaderName::from_static(
+                        "metadata",
+                    ))
+                    .is_none()
+                {
+                    let probe = self
+                        .probes
+                        .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+                    if self.recover_after.is_some_and(|failures| probe >= failures) {
+                        return Ok(AsyncRawResponse::from_bytes(
+                            StatusCode::BadRequest,
+                            Headers::new(),
+                            "metadata header required",
+                        ));
+                    }
+                    tokio::time::sleep(self.probe_delay).await;
+                    return Ok(AsyncRawResponse::from_bytes(
+                        self.probe_status,
+                        Headers::new(),
+                        "metadata header required",
+                    ));
+                }
+                if let Some(status) = self.response_status {
+                    return Ok(AsyncRawResponse::from_bytes(
+                        status,
+                        Headers::new(),
+                        "identity endpoint failure",
+                    ));
+                }
+                tokio::time::sleep(std::time::Duration::from_secs(2)).await;
+                let expires = std::time::SystemTime::now()
+                    .duration_since(std::time::UNIX_EPOCH)
+                    .unwrap()
+                    .as_secs()
+                    + 3600;
+                Ok(AsyncRawResponse::from_bytes(
+                    StatusCode::Ok,
+                    Headers::new(),
+                    serde_json::json!({
+                        "access_token": "managed-token", "expires_on": expires.to_string(),
+                        "token_type": "Bearer", "resource": "https://service.test",
+                    })
+                    .to_string(),
+                ))
+            })
+        }
+    }
+
+    #[rstest::rstest]
+    #[tokio::test(start_paused = true)]
+    async fn available_imds_retains_full_sdk_token_acquisition() {
+        let acquirer = NativeAzureTokenAcquirer::with_transport(
+            4,
+            Transport::new(Arc::new(AvailableImdsClient {
+                probe_status: StatusCode::BadRequest,
+                response_status: None,
+                ..Default::default()
+            })),
+        );
+        let request = ValidatedAzureRequest::new(NativeAzureRequest::ManagedIdentity {
+            client_id: None,
+            scope: deployment("https://service.test/.default".into()),
+            selection_source: InputSource::Deployment,
+        })
+        .unwrap()
+        .with_imds_probe(true);
+        let credential = acquirer.acquire(request).await.into_result().unwrap();
+        assert_eq!(credential.secret().expose(), "managed-token");
+    }
+
+    #[rstest::rstest]
+    #[case::another_metadata_service(StatusCode::NotFound)]
+    #[case::token_required(StatusCode::Unauthorized)]
+    #[case::proxy_rejection(StatusCode::Forbidden)]
+    #[case::unavailable(StatusCode::ServiceUnavailable)]
+    #[tokio::test(start_paused = true)]
+    async fn unrelated_metadata_responses_do_not_start_managed_identity_acquisition(
+        #[case] status: StatusCode,
+    ) {
+        let client = Arc::new(AvailableImdsClient {
+            probe_status: status,
+            ..Default::default()
+        });
+        let acquirer = NativeAzureTokenAcquirer::with_transport(4, Transport::new(client.clone()));
+        let request = ValidatedAzureRequest::new(NativeAzureRequest::ManagedIdentity {
+            client_id: None,
+            scope: deployment("https://service.test/.default".into()),
+            selection_source: InputSource::Deployment,
+        })
+        .unwrap()
+        .with_imds_probe(true);
+        assert!(matches!(
+            acquirer.acquire(request.clone()).await,
+            super::CredentialAttempt::Unavailable(_)
+        ));
+        assert!(matches!(
+            acquirer.acquire(request).await,
+            super::CredentialAttempt::Unavailable(_)
+        ));
+        assert_eq!(client.probes.load(std::sync::atomic::Ordering::SeqCst), 1);
+    }
+
+    #[rstest::rstest]
+    #[tokio::test(start_paused = true)]
+    async fn an_imds_timeout_is_not_repeated_on_the_next_request() {
+        let client = Arc::new(AvailableImdsClient {
+            probe_delay: Duration::from_secs(10),
+            ..Default::default()
+        });
+        let acquirer = NativeAzureTokenAcquirer::with_transport(4, Transport::new(client.clone()));
+        let request = ValidatedAzureRequest::new(NativeAzureRequest::ManagedIdentity {
+            client_id: None,
+            scope: deployment("https://service.test/.default".into()),
+            selection_source: InputSource::Deployment,
+        })
+        .unwrap()
+        .with_imds_probe(true);
+        assert!(matches!(
+            acquirer.acquire(request.clone()).await,
+            super::CredentialAttempt::Unavailable(_)
+        ));
+        let second = tokio::time::timeout(Duration::from_millis(10), acquirer.acquire(request))
+            .await
+            .unwrap();
+        assert!(matches!(second, super::CredentialAttempt::Unavailable(_)));
+        assert_eq!(client.probes.load(std::sync::atomic::Ordering::SeqCst), 1);
+    }
+
+    #[rstest::fixture]
+    fn imds_request() -> ValidatedAzureRequest {
+        ValidatedAzureRequest::new(NativeAzureRequest::ManagedIdentity {
+            client_id: None,
+            scope: deployment("https://service.test/.default".into()),
+            selection_source: InputSource::Deployment,
+        })
+        .unwrap()
+        .with_imds_probe(true)
+    }
+
+    #[rstest::rstest]
+    #[case::timeout(StatusCode::BadRequest, Duration::from_secs(10))]
+    #[case::service_unavailable(StatusCode::ServiceUnavailable, Duration::ZERO)]
+    #[tokio::test(start_paused = true)]
+    async fn failed_imds_discovery_recovers_without_restarting_the_acquirer(
+        imds_request: ValidatedAzureRequest,
+        #[case] status: StatusCode,
+        #[case] delay: Duration,
+    ) {
+        let client = Arc::new(AvailableImdsClient {
+            probe_status: status,
+            probe_delay: delay,
+            recover_after: Some(1),
+            ..Default::default()
+        });
+        let acquirer = NativeAzureTokenAcquirer::with_transport(4, Transport::new(client.clone()));
+        let (first, second) = tokio::join!(
+            acquirer.acquire(imds_request.clone()),
+            acquirer.acquire(imds_request.clone()),
+        );
+        assert!(matches!(first, super::CredentialAttempt::Unavailable(_)));
+        assert!(matches!(second, super::CredentialAttempt::Unavailable(_)));
+        assert_eq!(client.probes.load(std::sync::atomic::Ordering::SeqCst), 1);
+
+        tokio::time::advance(Duration::from_secs(30)).await;
+        let (first, second) = tokio::join!(
+            acquirer.acquire(imds_request.clone()),
+            acquirer.acquire(imds_request.clone()),
+        );
+        assert_eq!(
+            first.into_result().unwrap().secret().expose(),
+            "managed-token"
+        );
+        assert_eq!(
+            second.into_result().unwrap().secret().expose(),
+            "managed-token"
+        );
+        assert_eq!(client.probes.load(std::sync::atomic::Ordering::SeqCst), 2);
+
+        tokio::time::advance(Duration::from_secs(60)).await;
+        assert_eq!(
+            acquirer
+                .acquire(imds_request)
+                .await
+                .into_result()
+                .unwrap()
+                .secret()
+                .expose(),
+            "managed-token",
+        );
+        assert_eq!(client.probes.load(std::sync::atomic::Ordering::SeqCst), 2);
+    }
+
+    #[rstest::rstest]
+    #[tokio::test(start_paused = true)]
+    async fn a_persistent_imds_outage_reuses_each_failed_probe(
+        imds_request: ValidatedAzureRequest,
+    ) {
+        let client = Arc::new(AvailableImdsClient {
+            probe_delay: Duration::from_secs(10),
+            ..Default::default()
+        });
+        let acquirer = NativeAzureTokenAcquirer::with_transport(4, Transport::new(client.clone()));
+        assert!(matches!(
+            acquirer.acquire(imds_request.clone()).await,
+            super::CredentialAttempt::Unavailable(_),
+        ));
+        tokio::time::advance(Duration::from_secs(30)).await;
+        let (first, second) = tokio::join!(
+            acquirer.acquire(imds_request.clone()),
+            acquirer.acquire(imds_request),
+        );
+        assert!(matches!(first, super::CredentialAttempt::Unavailable(_)));
+        assert!(matches!(second, super::CredentialAttempt::Unavailable(_)));
+        assert_eq!(client.probes.load(std::sync::atomic::Ordering::SeqCst), 2);
+    }
+
+    #[rstest::rstest]
+    #[case::default_identity_not_assigned(true, StatusCode::BadRequest, true)]
+    #[case::explicit_identity_not_assigned(false, StatusCode::BadRequest, false)]
+    #[case::default_identity_forbidden(true, StatusCode::Forbidden, false)]
+    #[case::default_identity_service_failure(true, StatusCode::InternalServerError, false)]
+    #[tokio::test(start_paused = true)]
+    async fn only_unavailable_default_identities_allow_fallback(
+        #[case] discovery: bool,
+        #[case] status: StatusCode,
+        #[case] unavailable: bool,
+    ) {
+        let acquirer = NativeAzureTokenAcquirer::with_transport(
+            4,
+            Transport::new(Arc::new(AvailableImdsClient {
+                probe_status: StatusCode::BadRequest,
+                response_status: Some(status),
+                ..Default::default()
+            })),
+        );
+        let request = ValidatedAzureRequest::new(NativeAzureRequest::ManagedIdentity {
+            client_id: None,
+            scope: deployment("https://service.test/.default".into()),
+            selection_source: InputSource::Deployment,
+        })
+        .unwrap()
+        .with_imds_probe(discovery);
+        let outcome = acquirer.acquire(request).await;
+        let is_unavailable = matches!(outcome, super::CredentialAttempt::Unavailable(_));
+        let error = outcome.into_result().unwrap_err();
+        assert_eq!(
+            is_unavailable,
+            unavailable,
+            "{}",
+            litellm_error::Report::new(&error)
+        );
+    }
+
+    #[rstest::rstest]
     #[tokio::test]
     async fn client_secret_uses_sdk_protocol_and_reuses_cached_credential() {
         let transport = Arc::new(RecordingTokenClient::default());
@@ -582,8 +998,12 @@ mod tests {
             "https://login.test",
         );
 
-        let first = acquirer.acquire(request.clone()).await.unwrap();
-        let second = acquirer.acquire(request).await.unwrap();
+        let first = acquirer
+            .acquire(request.clone())
+            .await
+            .into_result()
+            .unwrap();
+        let second = acquirer.acquire(request).await.into_result().unwrap();
 
         assert_eq!(first.secret().expose(), "native-token");
         assert_eq!(second.secret().expose(), "native-token");
@@ -599,6 +1019,7 @@ mod tests {
         );
     }
 
+    #[rstest::rstest]
     #[tokio::test]
     async fn credential_provider_cache_isolates_every_client_secret_identity_field() {
         let transport = Arc::new(RecordingTokenClient::default());
@@ -645,10 +1066,10 @@ mod tests {
             ),
         ];
 
-        acquirer.acquire(base.clone()).await.unwrap();
-        acquirer.acquire(base).await.unwrap();
+        acquirer.acquire(base.clone()).await.into_result().unwrap();
+        acquirer.acquire(base).await.into_result().unwrap();
         for request in variants.into_iter().skip(1) {
-            acquirer.acquire(request).await.unwrap();
+            acquirer.acquire(request).await.into_result().unwrap();
         }
 
         assert_eq!(transport.requests.lock().unwrap().len(), 6);

litellm-rust/crates/auth-azure/src/resolve.rs

@@ -1,3 +1,4 @@
+use crate::native::CredentialAttempt;
 use litellm_auth_types::{
     CredentialFileRef, CredentialLookup, CredentialRef, InputSource, ResolvedCredential, Sourced,
     TokenProviderHandle,
@@ -12,25 +13,31 @@ use std::sync::Arc;
 use super::native::{NativeAzureRequest, NativeAzureTokenAcquirer, ValidatedAzureRequest};
 use super::types::{AzureAuthInputs, AzureCredentialType, ConfigValue, DEFAULT_AZURE_SCOPE};
 
-const AZURE_AD_TOKEN_ENV: &str = "AZURE_AD_TOKEN";
-const AZURE_TENANT_ID_ENV: &str = "AZURE_TENANT_ID";
-const AZURE_CLIENT_ID_ENV: &str = "AZURE_CLIENT_ID";
-const AZURE_CLIENT_SECRET_ENV: &str = "AZURE_CLIENT_SECRET";
-const AZURE_SCOPE_ENV: &str = "AZURE_SCOPE";
-const AZURE_AUTHORITY_HOST_ENV: &str = "AZURE_AUTHORITY_HOST";
-const AZURE_CREDENTIAL_ENV: &str = "AZURE_CREDENTIAL";
-const AZURE_FEDERATED_TOKEN_FILE_ENV: &str = "AZURE_FEDERATED_TOKEN_FILE";
+#[derive(strum::IntoStaticStr, strum::VariantArray, strum::VariantNames)]
+enum AzureSecretName {
+    #[strum(serialize = "AZURE_AD_TOKEN")]
+    AdToken,
+    #[strum(serialize = "AZURE_TENANT_ID")]
+    TenantId,
+    #[strum(serialize = "AZURE_CLIENT_ID")]
+    ClientId,
+    #[strum(serialize = "AZURE_CLIENT_SECRET")]
+    ClientSecret,
+    #[strum(serialize = "AZURE_SCOPE")]
+    Scope,
+    #[strum(serialize = "AZURE_AUTHORITY_HOST")]
+    AuthorityHost,
+    #[strum(serialize = "AZURE_CREDENTIAL")]
+    Credential,
+    #[strum(serialize = "AZURE_FEDERATED_TOKEN_FILE")]
+    FederatedTokenFile,
+    #[strum(serialize = "IDENTITY_ENDPOINT")]
+    IdentityEndpoint,
+    #[strum(serialize = "MSI_ENDPOINT")]
+    MsiEndpoint,
+}
 
-pub const SECRET_NAMES: &[&str] = &[
-    AZURE_AD_TOKEN_ENV,
-    AZURE_TENANT_ID_ENV,
-    AZURE_CLIENT_ID_ENV,
-    AZURE_CLIENT_SECRET_ENV,
-    AZURE_SCOPE_ENV,
-    AZURE_AUTHORITY_HOST_ENV,
-    AZURE_CREDENTIAL_ENV,
-    AZURE_FEDERATED_TOKEN_FILE_ENV,
-];
+pub const SECRET_NAMES: &[&str] = <AzureSecretName as strum::VariantNames>::VARIANTS;
 
 #[derive(Clone, Debug)]
 pub(crate) enum AzureCredentialPlan {
@@ -57,14 +64,14 @@ trait AzureTokenAcquirer: Send + Sync {
     fn acquire(
         &self,
         request: ValidatedAzureRequest,
-    ) -> Pin<Box<dyn Future<Output = Result<ResolvedCredential, Error>> + Send + '_>>;
+    ) -> Pin<Box<dyn Future<Output = CredentialAttempt> + Send + '_>>;
 }
 
 impl AzureTokenAcquirer for NativeAzureTokenAcquirer {
     fn acquire(
         &self,
         request: ValidatedAzureRequest,
-    ) -> Pin<Box<dyn Future<Output = Result<ResolvedCredential, Error>> + Send + '_>> {
+    ) -> Pin<Box<dyn Future<Output = CredentialAttempt> + Send + '_>> {
         Box::pin(NativeAzureTokenAcquirer::acquire(self, request))
     }
 }
@@ -83,12 +90,36 @@ impl AzureAuthService {
         Self { native }
     }
 
+    pub async fn identity_token(
+        &self,
+        identity: &AzureIdentity,
+        scope: &str,
+        env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
+    ) -> Result<Option<ResolvedCredential>, Error> {
+        let plan = identity_plan(identity, scope, env_lookup)?;
+        let inputs = AzureAuthInputs::default_credential_for_scope(scope);
+        Ok(self
+            .run_plan(plan, &inputs, env_lookup)
+            .await?
+            .map(Sourced::into_value))
+    }
+
     pub async fn get_azure_ad_token(
         &self,
         inputs: &AzureAuthInputs,
         env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
     ) -> Result<Option<Sourced<ResolvedCredential>>, Error> {
-        match select_auth_plan(inputs, env_lookup)? {
+        self.run_plan(select_auth_plan(inputs, env_lookup)?, inputs, env_lookup)
+            .await
+    }
+
+    async fn run_plan(
+        &self,
+        plan: AzureCredentialPlan,
+        inputs: &AzureAuthInputs,
+        env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
+    ) -> Result<Option<Sourced<ResolvedCredential>>, Error> {
+        match plan {
             AzureCredentialPlan::Supplied(credential) => Ok(Some(credential)),
             AzureCredentialPlan::Caller(caller) => {
                 let credential = caller.acquire().await?;
@@ -123,6 +154,7 @@ impl AzureAuthService {
                 self.native
                     .acquire(request)
                     .await
+                    .into_result()
                     .map(|credential| Sourced::new(credential, source))
                     .map(Some)
             }
@@ -131,6 +163,7 @@ impl AzureAuthService {
                 self.native
                     .acquire(request)
                     .await
+                    .into_result()
                     .map(|credential| Some(Sourced::new(credential, source)))
             }
             AzureCredentialPlan::Chain(requests) => {
@@ -138,8 +171,16 @@ impl AzureAuthService {
                 for request in requests {
                     let source = request.credential_source();
                     match self.native.acquire(request).await {
-                        Ok(credential) => return Ok(Some(Sourced::new(credential, source))),
-                        Err(error) => failures.push(error),
+                        CredentialAttempt::Resolved(credential) => {
+                            return Ok(Some(Sourced::new(credential, source)));
+                        }
+                        CredentialAttempt::Unavailable(error) => failures.push(error),
+                        CredentialAttempt::Failed(error) => {
+                            return Err(Error::CredentialChain {
+                                last: Box::new(error),
+                                earlier: failures,
+                            });
+                        }
                     }
                 }
                 match failures.pop() {
@@ -155,39 +196,161 @@ impl AzureAuthService {
     }
 }
 
+/// The identity azure-identity builds for one explicit credential class, without LiteLLM's provider settings
+/// (`AZURE_AD_TOKEN`, `AZURE_CREDENTIAL`, `AZURE_SCOPE`) that `get_azure_ad_token` also reads.
+#[derive(Clone, Debug)]
+pub enum AzureIdentity {
+    ClientSecret {
+        tenant_id: String,
+        client_id: String,
+        client_secret: SecretValue,
+    },
+    ManagedIdentity {
+        client_id: String,
+    },
+    Default,
+}
+
+fn identity_plan(
+    identity: &AzureIdentity,
+    scope: &str,
+    env_lookup: &dyn Fn(&str) -> Option<String>,
+) -> Result<AzureCredentialPlan, Error> {
+    let deployment = |value: &str| Sourced::new(value.to_owned(), InputSource::Deployment);
+    let environment = |name: &str| {
+        env_lookup(name)
+            .filter(|value| !value.trim().is_empty())
+            .map(|value| Sourced::new(value, InputSource::Environment))
+    };
+    let scope = deployment(scope);
+    let authority = environment(AzureSecretName::AuthorityHost.into());
+    match identity {
+        AzureIdentity::ClientSecret {
+            tenant_id,
+            client_id,
+            client_secret,
+        } => Ok(AzureCredentialPlan::Native(ValidatedAzureRequest::new(
+            NativeAzureRequest::ClientSecret {
+                tenant_id: deployment(tenant_id),
+                client_id: deployment(client_id),
+                client_secret: Sourced::new(client_secret.clone(), InputSource::Deployment),
+                scope,
+                authority,
+            },
+        )?)),
+        AzureIdentity::ManagedIdentity { client_id } => Ok(AzureCredentialPlan::Native(
+            ValidatedAzureRequest::new(NativeAzureRequest::ManagedIdentity {
+                client_id: Some(deployment(client_id)),
+                scope,
+                selection_source: InputSource::Deployment,
+            })?,
+        )),
+        AzureIdentity::Default => {
+            let tenant_id = environment(AzureSecretName::TenantId.into());
+            let client_id = environment(AzureSecretName::ClientId.into());
+            let from_environment = match (
+                tenant_id.clone(),
+                client_id.clone(),
+                environment(AzureSecretName::ClientSecret.into()),
+            ) {
+                (Some(tenant_id), Some(client_id), Some(client_secret)) => {
+                    Some(NativeAzureRequest::ClientSecret {
+                        tenant_id,
+                        client_id,
+                        client_secret: client_secret.map(SecretValue::new),
+                        scope: scope.clone(),
+                        authority: authority.clone(),
+                    })
+                }
+                _ => None,
+            };
+            let probe_imds = env_lookup(AzureSecretName::IdentityEndpoint.into()).is_none()
+                && env_lookup(AzureSecretName::MsiEndpoint.into()).is_none();
+            let AzureCredentialPlan::Chain(chain) = select_native_plan(
+                Some(Sourced::new(
+                    AzureCredentialType::DefaultAzureCredential,
+                    InputSource::Deployment,
+                )),
+                tenant_id,
+                client_id,
+                environment(AzureSecretName::FederatedTokenFile.into()),
+                scope,
+                authority,
+                InputSource::Environment,
+            )?
+            else {
+                unreachable!("DefaultAzureCredential always plans a chain")
+            };
+            let environment_request = from_environment
+                .map(ValidatedAzureRequest::new)
+                .transpose()?;
+            Ok(AzureCredentialPlan::Chain(
+                environment_request
+                    .into_iter()
+                    .chain(chain)
+                    .map(|request| request.with_imds_probe(probe_imds))
+                    .collect(),
+            ))
+        }
+    }
+}
+
 pub(crate) fn select_auth_plan(
     inputs: &AzureAuthInputs,
     env_lookup: &dyn Fn(&str) -> Option<String>,
 ) -> Result<AzureCredentialPlan, Error> {
-    let token = configured_secret(&inputs.azure_ad_token, AZURE_AD_TOKEN_ENV, env_lookup);
-    let tenant_id = configured_string(&inputs.tenant_id, AZURE_TENANT_ID_ENV, env_lookup);
-    let client_id = configured_string(&inputs.client_id, AZURE_CLIENT_ID_ENV, env_lookup);
-    let client_secret =
-        configured_secret(&inputs.client_secret, AZURE_CLIENT_SECRET_ENV, env_lookup);
-    let scope = configured_string(&inputs.azure_scope, AZURE_SCOPE_ENV, env_lookup)
-        .unwrap_or_else(|| Sourced::new(DEFAULT_AZURE_SCOPE.to_string(), InputSource::Environment));
-    let authority = configured_string(
-        &inputs.azure_authority_host,
-        AZURE_AUTHORITY_HOST_ENV,
+    let token = configured_secret(
+        &inputs.azure_ad_token,
+        AzureSecretName::AdToken.into(),
         env_lookup,
     );
-    let selector = configured_string(&inputs.azure_credential, AZURE_CREDENTIAL_ENV, env_lookup)
-        .map(|value| {
-            value
-                .value()
-                .parse::<AzureCredentialType>()
-                .map(|selector| Sourced::new(selector, value.source()))
-        })
-        .transpose()
-        .map_err(|error| {
-            Error::InvalidConfiguration(ErrorDetail::failed(
-                "Azure credential selector parsing",
-                error,
-            ))
-        })?;
+    let tenant_id = configured_string(
+        &inputs.tenant_id,
+        AzureSecretName::TenantId.into(),
+        env_lookup,
+    );
+    let client_id = configured_string(
+        &inputs.client_id,
+        AzureSecretName::ClientId.into(),
+        env_lookup,
+    );
+    let client_secret = configured_secret(
+        &inputs.client_secret,
+        AzureSecretName::ClientSecret.into(),
+        env_lookup,
+    );
+    let scope = configured_string(
+        &inputs.azure_scope,
+        AzureSecretName::Scope.into(),
+        env_lookup,
+    )
+    .unwrap_or_else(|| Sourced::new(DEFAULT_AZURE_SCOPE.to_string(), InputSource::Environment));
+    let authority = configured_string(
+        &inputs.azure_authority_host,
+        AzureSecretName::AuthorityHost.into(),
+        env_lookup,
+    );
+    let selector = configured_string(
+        &inputs.azure_credential,
+        AzureSecretName::Credential.into(),
+        env_lookup,
+    )
+    .map(|value| {
+        value
+            .value()
+            .parse::<AzureCredentialType>()
+            .map(|selector| Sourced::new(selector, value.source()))
+    })
+    .transpose()
+    .map_err(|error| {
+        Error::InvalidConfiguration(ErrorDetail::failed(
+            "Azure credential selector parsing",
+            error,
+        ))
+    })?;
     let federated_token_file = configured_string(
         &inputs.federated_token_file,
-        AZURE_FEDERATED_TOKEN_FILE_ENV,
+        AzureSecretName::FederatedTokenFile.into(),
         env_lookup,
     );
 
@@ -237,7 +400,12 @@ pub(crate) fn select_auth_plan(
         return Ok(AzureCredentialPlan::Missing);
     }
 
-    select_native_plan(
+    let probe_imds = selector
+        .as_ref()
+        .is_none_or(|selector| *selector.value() == AzureCredentialType::DefaultAzureCredential)
+        && env_lookup(AzureSecretName::IdentityEndpoint.into()).is_none()
+        && env_lookup(AzureSecretName::MsiEndpoint.into()).is_none();
+    match select_native_plan(
         selector,
         tenant_id,
         client_id,
@@ -245,7 +413,15 @@ pub(crate) fn select_auth_plan(
         scope,
         authority,
         inputs.enable_azure_ad_token_refresh.source(),
-    )
+    )? {
+        AzureCredentialPlan::Chain(requests) => Ok(AzureCredentialPlan::Chain(
+            requests
+                .into_iter()
+                .map(|request| request.with_imds_probe(probe_imds))
+                .collect(),
+        )),
+        plan => Ok(plan),
+    }
 }
 
 fn select_native_plan(
@@ -509,7 +685,10 @@ mod tests {
     #[derive(Debug)]
     struct FileResolver;
 
+    use crate::native::CredentialAttempt;
+
     struct ChainAcquirer {
+        terminal_on: Option<&'static str>,
         requests: Mutex<Vec<&'static str>>,
         succeed_on: Option<&'static str>,
     }
@@ -518,20 +697,25 @@ mod tests {
         fn acquire(
             &self,
             request: ValidatedAzureRequest,
-        ) -> std::pin::Pin<Box<dyn Future<Output = Result<ResolvedCredential, Error>> + Send + '_>>
-        {
+        ) -> std::pin::Pin<Box<dyn Future<Output = CredentialAttempt> + Send + '_>> {
             let kind = request.kind();
             self.requests.lock().unwrap().push(kind);
             Box::pin(async move {
                 if self.succeed_on == Some(kind) {
-                    Ok(ResolvedCredential::AccessToken {
+                    CredentialAttempt::Resolved(ResolvedCredential::AccessToken {
                         token: SecretValue::new("chain-token"),
                         expires_on: None,
                     })
                 } else {
-                    Err(Error::CredentialAcquisition(
-                        litellm_auth_types::ErrorDetail::Unresolved { subject: kind },
-                    ))
+                    let error =
+                        Error::CredentialAcquisition(litellm_auth_types::ErrorDetail::Unresolved {
+                            subject: kind,
+                        });
+                    if self.terminal_on == Some(kind) {
+                        CredentialAttempt::Failed(error)
+                    } else {
+                        CredentialAttempt::Unavailable(error)
+                    }
                 }
             })
         }
@@ -690,9 +874,210 @@ mod tests {
         assert_eq!(resolved, Some(SecretValue::new("rotated-assertion")));
     }
 
+    #[derive(Debug)]
+    struct UnavailableImdsClient;
+
+    impl azure_core::http::HttpClient for UnavailableImdsClient {
+        fn execute_request<'life0, 'life1, 'async_trait>(
+            &'life0 self,
+            _request: &'life1 azure_core::http::Request,
+        ) -> std::pin::Pin<
+            Box<
+                dyn Future<Output = azure_core::Result<azure_core::http::AsyncRawResponse>>
+                    + Send
+                    + 'async_trait,
+            >,
+        >
+        where
+            'life0: 'async_trait,
+            'life1: 'async_trait,
+            Self: 'async_trait,
+        {
+            Box::pin(std::future::pending())
+        }
+    }
+
+    struct OfflineChainAcquirer(super::NativeAzureTokenAcquirer);
+
+    impl AzureTokenAcquirer for OfflineChainAcquirer {
+        fn acquire(
+            &self,
+            request: ValidatedAzureRequest,
+        ) -> std::pin::Pin<Box<dyn Future<Output = CredentialAttempt> + Send + '_>> {
+            Box::pin(async move {
+                if request.kind() == "developer-tools" {
+                    return CredentialAttempt::Resolved(ResolvedCredential::AccessToken {
+                        token: SecretValue::new("cli-token"),
+                        expires_on: None,
+                    });
+                }
+                self.0.acquire(request).await
+            })
+        }
+    }
+
+    #[rstest::rstest]
+    #[case::default_off_azure("DefaultAzureCredential", None, true)]
+    #[case::explicit_managed_identity("ManagedIdentityCredential", None, false)]
+    #[case::app_service("DefaultAzureCredential", Some("http://app-service.test"), false)]
+    #[tokio::test(start_paused = true)]
+    async fn only_default_imds_discovery_is_bounded(
+        #[case] selector: &str,
+        #[case] identity_endpoint: Option<&str>,
+        #[case] expect_cli: bool,
+    ) {
+        let native = super::NativeAzureTokenAcquirer::with_transport(
+            4,
+            azure_core::http::Transport::new(Arc::new(UnavailableImdsClient)),
+        );
+        let service = AzureAuthService::with_acquirer(Arc::new(OfflineChainAcquirer(native)));
+        let inputs = AzureAuthInputs {
+            azure_credential: crate::types::ConfigValue::Value(Sourced::new(
+                selector.into(),
+                InputSource::Deployment,
+            )),
+            enable_azure_ad_token_refresh: Sourced::new(true, InputSource::Deployment),
+            ..Default::default()
+        };
+        let environment = |name: &str| {
+            (name == "IDENTITY_ENDPOINT")
+                .then_some(identity_endpoint)
+                .flatten()
+                .map(str::to_string)
+        };
+        let outcome = tokio::time::timeout(
+            std::time::Duration::from_secs(2),
+            service.get_azure_ad_token(&inputs, &environment),
+        )
+        .await;
+
+        assert_eq!(outcome.is_ok(), expect_cli);
+        if expect_cli {
+            assert_eq!(
+                outcome.unwrap().unwrap().unwrap().value().secret().expose(),
+                "cli-token"
+            );
+        }
+    }
+
+    #[rstest::rstest]
+    #[case::app_service(Some("IDENTITY_ENDPOINT"), false)]
+    #[case::legacy_app_service(Some("MSI_ENDPOINT"), false)]
+    #[case::off_azure(None, true)]
+    #[tokio::test(start_paused = true)]
+    async fn default_chain_uses_endpoints_from_the_secret_snapshot(
+        #[case] endpoint: Option<&str>,
+        #[case] expect_cli: bool,
+        #[values(false, true)] consumer_identity: bool,
+    ) {
+        use litellm_secrets::source::SecretSource;
+
+        struct EndpointSecrets<'a>(Option<&'a str>);
+
+        impl SecretSource for EndpointSecrets<'_> {
+            fn get_secret_str<'a>(
+                &'a self,
+                name: &'a str,
+            ) -> std::pin::Pin<
+                Box<
+                    dyn Future<Output = Result<Option<SecretValue>, litellm_secrets::Error>>
+                        + Send
+                        + 'a,
+                >,
+            > {
+                Box::pin(async move {
+                    Ok((self.0 == Some(name)).then(|| SecretValue::new("http://app-service.test")))
+                })
+            }
+        }
+
+        let snapshot = EndpointSecrets(endpoint)
+            .resolve(SECRET_NAMES)
+            .await
+            .unwrap();
+        let native = super::NativeAzureTokenAcquirer::with_transport(
+            4,
+            azure_core::http::Transport::new(Arc::new(UnavailableImdsClient)),
+        );
+        let service = AzureAuthService::with_acquirer(Arc::new(OfflineChainAcquirer(native)));
+        let lookup = |name: &str| snapshot.get(name);
+        let scope = "https://service.test/.default";
+        let outcome = tokio::time::timeout(std::time::Duration::from_secs(2), async {
+            if consumer_identity {
+                service
+                    .identity_token(&super::AzureIdentity::Default, scope, &lookup)
+                    .await
+            } else {
+                service
+                    .get_azure_ad_token(
+                        &AzureAuthInputs::default_credential_for_scope(scope),
+                        &lookup,
+                    )
+                    .await
+                    .map(|token| token.map(Sourced::into_value))
+            }
+        })
+        .await;
+
+        assert_eq!(outcome.is_ok(), expect_cli);
+        if expect_cli {
+            assert_eq!(
+                outcome.unwrap().unwrap().unwrap().secret().expose(),
+                "cli-token"
+            );
+        }
+    }
+
+    #[rstest::rstest]
+    #[case::litellm_provider_settings_are_ignored(
+        &[("AZURE_AD_TOKEN", "provider-token"), ("AZURE_CREDENTIAL", "ClientSecretCredential")],
+        &["managed-identity", "developer-tools"]
+    )]
+    #[case::environment_client_secret_comes_first(
+        &[("AZURE_TENANT_ID", "tenant"), ("AZURE_CLIENT_ID", "client"), ("AZURE_CLIENT_SECRET", "secret")],
+        &["client-secret", "managed-identity", "developer-tools"]
+    )]
+    #[case::aks_workload_identity(
+        &[("AZURE_TENANT_ID", "tenant"), ("AZURE_CLIENT_ID", "client"), ("AZURE_FEDERATED_TOKEN_FILE", "/var/run/token")],
+        &["workload-identity", "managed-identity", "developer-tools"]
+    )]
+    #[tokio::test]
+    async fn the_default_identity_follows_azure_identity_defaults(
+        #[case] environment: &[(&str, &str)],
+        #[case] expected: &[&str],
+    ) {
+        let acquirer = Arc::new(ChainAcquirer {
+            terminal_on: None,
+            requests: Mutex::new(Vec::new()),
+            succeed_on: Some("developer-tools"),
+        });
+        let service = AzureAuthService::with_acquirer(acquirer.clone());
+        let lookup = |name: &str| {
+            environment
+                .iter()
+                .find(|(key, _)| *key == name)
+                .map(|(_, value)| (*value).to_owned())
+        };
+
+        let credential = service
+            .identity_token(
+                &super::AzureIdentity::Default,
+                "https://redis.azure.com/.default",
+                &lookup,
+            )
+            .await
+            .unwrap()
+            .unwrap();
+
+        assert_eq!(credential.secret().expose(), "chain-token");
+        assert_eq!(*acquirer.requests.lock().unwrap(), expected);
+    }
+
+    #[rstest::rstest]
     #[tokio::test]
     async fn default_chain_uses_declared_order_and_stops_after_success() {
         let acquirer = Arc::new(ChainAcquirer {
+            terminal_on: None,
             requests: Mutex::new(Vec::new()),
             succeed_on: Some("developer-tools"),
         });
@@ -715,9 +1100,13 @@ mod tests {
         );
     }
 
+    #[rstest::rstest]
+    #[case::all_unavailable(false)]
+    #[case::terminal_failure_after_unavailable(true)]
     #[tokio::test]
-    async fn chain_reports_each_acquisition_failure() {
+    async fn chain_reports_each_acquisition_failure(#[case] terminal: bool) {
         let acquirer = Arc::new(ChainAcquirer {
+            terminal_on: terminal.then_some("developer-tools"),
             requests: Mutex::new(Vec::new()),
             succeed_on: None,
         });
@@ -744,6 +1133,27 @@ mod tests {
         );
     }
 
+    #[rstest::rstest]
+    #[tokio::test]
+    async fn a_configured_identity_failure_does_not_fall_back_to_a_developer_identity() {
+        let acquirer = Arc::new(ChainAcquirer {
+            terminal_on: Some("managed-identity"),
+            requests: Mutex::new(Vec::new()),
+            succeed_on: Some("developer-tools"),
+        });
+        let service = AzureAuthService::with_acquirer(acquirer.clone());
+        let inputs = AzureAuthInputs {
+            enable_azure_ad_token_refresh: Sourced::new(true, InputSource::Deployment),
+            ..Default::default()
+        };
+        let error = service
+            .get_azure_ad_token(&inputs, &|_| None)
+            .await
+            .unwrap_err();
+        assert_eq!(*acquirer.requests.lock().unwrap(), ["managed-identity"]);
+        assert!(matches!(error, Error::CredentialChain { earlier, .. } if earlier.is_empty()));
+    }
+
     #[derive(Debug)]
     struct CallerToken(&'static str);
 

litellm-rust/crates/auth-gcp/Cargo.toml

@@ -6,7 +6,7 @@ license.workspace = true
 repository.workspace = true
 
 [features]
-google-sdk = ["dep:google-cloud-auth", "dep:http"]
+google-sdk = ["dep:http"]
 
 [dependencies]
 litellm-secret-value.workspace = true
@@ -21,10 +21,14 @@ tokio.workspace = true
 veil.workspace = true
 
 gcp_auth = "0.12.7"
-google-cloud-auth = { workspace = true, optional = true }
+google-cloud-auth.workspace = true
 http = { workspace = true, optional = true }
 
 [dev-dependencies]
+time = { workspace = true, features = ["formatting"] }
+axum = { workspace = true, features = ["json"] }
 litellm-error.workspace = true
 rstest.workspace = true
+rustls.workspace = true
 tempfile.workspace = true
+tokio = { workspace = true, features = ["test-util"] }

litellm-rust/crates/auth-gcp/src/auth.rs

@@ -1,20 +1,11 @@
-use std::{future::Future, pin::Pin, sync::Arc};
+use std::sync::Arc;
 
-use gcp_auth::{CustomServiceAccount, TokenProvider};
-use litellm_auth_types::{CredentialPlacement, Error, ErrorDetail, http::apply_credential};
-use moka::future::Cache;
-use serde_json::Value;
-use sha2::{Digest, Sha256};
+use litellm_auth_types::{CredentialPlacement, Error, http::apply_credential};
 
 use crate::{
-    config::{
-        CredentialSource, VertexConfig, credential_source, get_vertex_ai_project,
-        is_base64_encoded_json, names_a_credentials_file, non_empty_env,
-    },
-    constants::{
-        CLOUD_PLATFORM_SCOPE, GOOGLE_OAUTH_TOKEN_ENDPOINT, VERTEX_AI_API_KEY_ENV,
-        VERTEXAI_API_KEY_ENV,
-    },
+    config::{VertexConfig, credential_source, get_vertex_ai_project, non_empty_env},
+    constants::{VERTEX_AI_API_KEY_ENV, VERTEXAI_API_KEY_ENV},
+    identity::{GcpAuth, GcpProviderLoader, GcpTokenSource},
 };
 
 pub struct VertexEnvironment {
@@ -22,28 +13,15 @@ pub struct VertexEnvironment {
     pub project_id: String,
 }
 
-struct VertexAccessToken {
-    token: String,
-    project_id: String,
-}
-
-#[derive(Clone)]
+#[derive(Clone, Default)]
 pub struct VertexAuth {
-    providers: Cache<CredentialCacheKey, Arc<dyn VertexTokenSource>>,
-    loader: Arc<dyn VertexProviderLoader>,
-}
-
-impl Default for VertexAuth {
-    fn default() -> Self {
-        Self::new(Arc::new(GcpProviderLoader))
-    }
+    identity: GcpAuth,
 }
 
 impl VertexAuth {
-    pub fn new(loader: Arc<dyn VertexProviderLoader>) -> Self {
+    pub fn new(loader: Arc<dyn GcpProviderLoader>) -> Self {
         Self {
-            providers: Cache::builder().max_capacity(64).build(),
-            loader,
+            identity: GcpAuth::new(loader),
         }
     }
 
@@ -52,7 +30,10 @@ impl VertexAuth {
         config: &VertexConfig,
         env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
     ) -> Result<String, Error> {
-        self.load_provider(config, env_lookup).await?.token().await
+        self.identity
+            .access_token(credential_source(config, env_lookup))
+            .await
+            .map(|token| token.expose().to_owned())
     }
 
     pub async fn validate_environment(
@@ -77,10 +58,18 @@ impl VertexAuth {
             (true, _) => None,
             (false, Some(token)) => Some(token),
             (false, None) => {
-                let access = self.get_access_token(config, env_lookup).await?;
+                let provider = self.load_provider(config, env_lookup).await?;
+                let (token, project_id) = match project_id {
+                    Some(project_id) => (provider.token().await?, project_id),
+                    None => tokio::try_join!(provider.token(), provider.project_id())?,
+                };
                 return Ok(VertexEnvironment {
-                    headers: apply_credential(headers, &access.token, CredentialPlacement::Bearer)?,
-                    project_id: project_id.unwrap_or(access.project_id),
+                    headers: apply_credential(
+                        headers,
+                        token.expose(),
+                        CredentialPlacement::Bearer,
+                    )?,
+                    project_id,
                 });
             }
         };
@@ -104,248 +93,171 @@ impl VertexAuth {
         })
     }
 
-    async fn get_access_token(
-        &self,
-        config: &VertexConfig,
-        env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
-    ) -> Result<VertexAccessToken, Error> {
-        let provider = self.load_provider(config, env_lookup).await?;
-        let (token, project_id) = tokio::try_join!(provider.token(), provider.project_id())?;
-        Ok(VertexAccessToken { token, project_id })
-    }
-
     async fn load_provider(
         &self,
         config: &VertexConfig,
         env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
-    ) -> Result<Arc<dyn VertexTokenSource>, Error> {
-        let source = credential_source(config, env_lookup);
-        let key = source.cache_key();
-        self.providers
-            .try_get_with(key, self.loader.load(source))
+    ) -> Result<Arc<dyn GcpTokenSource>, Error> {
+        self.identity
+            .load_provider(credential_source(config, env_lookup))
             .await
-            .map_err(|error| (*error).clone())
     }
 }
 
-pub trait VertexTokenSource: Send + Sync {
-    fn project_id(&self) -> VertexAuthFuture<'_, String>;
-    fn token(&self) -> VertexAuthFuture<'_, String>;
-}
-
-pub trait VertexProviderLoader: Send + Sync {
-    fn load(&self, source: CredentialSource) -> VertexAuthFuture<'_, Arc<dyn VertexTokenSource>>;
-}
-
-pub type VertexAuthFuture<'a, T> = Pin<Box<dyn Future<Output = Result<T, Error>> + Send + 'a>>;
-
-struct GcpTokenSource(Arc<dyn TokenProvider>);
-
-impl VertexTokenSource for GcpTokenSource {
-    fn project_id(&self) -> VertexAuthFuture<'_, String> {
-        Box::pin(async move {
-            self.0
-                .project_id()
-                .await
-                .map(|project| project.to_string())
-                .map_err(auth_acquisition_error)
-        })
-    }
-
-    fn token(&self) -> VertexAuthFuture<'_, String> {
-        Box::pin(async move {
-            self.0
-                .token(&[CLOUD_PLATFORM_SCOPE])
-                .await
-                .map(|token| token.as_str().to_string())
-                .map_err(auth_acquisition_error)
-        })
-    }
-}
-
-struct GcpProviderLoader;
-
-impl VertexProviderLoader for GcpProviderLoader {
-    fn load(&self, source: CredentialSource) -> VertexAuthFuture<'_, Arc<dyn VertexTokenSource>> {
-        Box::pin(async move {
-            let provider: Arc<dyn TokenProvider> = match source {
-                CredentialSource::Inline(configured) => Arc::new(
-                    CustomServiceAccount::from_json(validate_request_credentials(
-                        configured.expose(),
-                    )?)
-                    .map_err(auth_acquisition_error)?,
-                ),
-                CredentialSource::Trusted(configured) => {
-                    Arc::new(load_trusted_service_account(configured.expose())?)
-                }
-                CredentialSource::ApplicationCredentials(path) => {
-                    Arc::new(CustomServiceAccount::from_file(path).map_err(auth_acquisition_error)?)
-                }
-                CredentialSource::Adc => {
-                    gcp_auth::provider().await.map_err(auth_acquisition_error)?
-                }
-            };
-            Ok(Arc::new(GcpTokenSource(provider)) as Arc<dyn VertexTokenSource>)
-        })
-    }
-}
-
-fn validate_request_credentials(configured: &str) -> Result<&str, Error> {
-    let token_uri = serde_json::from_str::<Value>(configured)
-        .ok()
-        .and_then(|credentials| {
-            credentials
-                .get("token_uri")
-                .and_then(Value::as_str)
-                .map(str::to_string)
-        });
-    if token_uri.as_deref() != Some(GOOGLE_OAUTH_TOKEN_ENDPOINT) {
-        return Err(Error::InvalidConfiguration(ErrorDetail::InvalidType {
-            field: "request-controlled vertex_credentials token_uri".into(),
-            expected: "the canonical Google OAuth token endpoint",
-        }));
-    }
-    Ok(configured)
-}
-
-impl CredentialSource {
-    fn cache_key(&self) -> CredentialCacheKey {
-        match self {
-            Self::Inline(configured) => {
-                CredentialCacheKey::Inline(Sha256::digest(configured.expose()).into())
-            }
-            Self::Trusted(configured) => {
-                CredentialCacheKey::Trusted(Sha256::digest(configured.expose()).into())
-            }
-            Self::ApplicationCredentials(path) => {
-                CredentialCacheKey::ApplicationCredentials(path.clone())
-            }
-            Self::Adc => CredentialCacheKey::Adc,
-        }
-    }
-}
-
-#[derive(Clone, Debug, Hash, PartialEq, Eq)]
-enum CredentialCacheKey {
-    Inline([u8; 32]),
-    Trusted([u8; 32]),
-    ApplicationCredentials(String),
-    Adc,
-}
-
-fn load_trusted_service_account(configured: &str) -> Result<CustomServiceAccount, Error> {
-    if is_base64_encoded_json(configured) {
-        return Err(Error::InvalidConfiguration(
-            ErrorDetail::Base64EncodedJson {
-                field: "vertex_credentials",
-            },
-        ));
-    }
-    if names_a_credentials_file(configured) {
-        CustomServiceAccount::from_file(configured)
-    } else {
-        CustomServiceAccount::from_json(configured)
-    }
-    .map_err(auth_acquisition_error)
-}
-
-fn auth_acquisition_error(error: gcp_auth::Error) -> Error {
-    Error::CredentialAcquisition(litellm_auth_types::ErrorDetail::failed(
-        "Vertex AI credentials",
-        error,
-    ))
-}
-
 #[cfg(test)]
 mod tests {
+    use std::{
+        collections::{BTreeMap, BTreeSet},
+        sync::{
+            Arc,
+            atomic::{AtomicUsize, Ordering},
+        },
+    };
+
+    use crate::{
+        CredentialSource, GcpAuthFuture, GcpProviderLoader, GcpTokenSource, VertexAuth,
+        VertexConfig, get_vertex_ai_location, secret_names,
+    };
     use litellm_secret_value::SecretValue;
+    use rstest::rstest;
+    use serde_json::{Value, json};
 
-    use super::*;
-
-    #[rstest::rstest]
-    #[case::canonical_endpoint(r#"{"token_uri":"https://oauth2.googleapis.com/token"}"#, true)]
-    #[case::noncanonical_endpoint(r#"{"token_uri":"http://127.0.0.1/token"}"#, false)]
-    #[case::missing_endpoint("{}", false)]
-    fn request_credentials_require_canonical_token_endpoint(
-        #[case] credentials: &str,
-        #[case] accepted: bool,
-    ) {
-        assert_eq!(validate_request_credentials(credentials).is_ok(), accepted);
+    struct FakeProvider {
+        project: Option<&'static str>,
+        calls: Arc<AtomicUsize>,
     }
 
-    #[rstest::rstest]
-    #[case::missing_file(
-        "/nonexistent/vertex-key.json",
-        "failed to open application credentials file"
-    )]
-    #[case::malformed_json(
-        " {\"type\": \"service_account\",",
-        "failed to deserialize ApplicationCredentials"
-    )]
-    #[case::quoted_json(
-        "'{\"type\": \"service_account\"}'",
-        "failed to deserialize ApplicationCredentials"
-    )]
-    #[tokio::test]
-    async fn trusted_credentials_are_read_as_a_file_unless_they_are_json(
-        #[case] configured: &str,
-        #[case] expected: &str,
-    ) {
-        let Err(error) = GcpProviderLoader
-            .load(CredentialSource::Trusted(SecretValue::new(configured)))
-            .await
-        else {
-            panic!("{configured:?} should not load");
-        };
-        let report = litellm_error::Report::new(&error).to_string();
-        assert!(report.contains(expected), "{report}");
-    }
+    impl GcpTokenSource for FakeProvider {
+        fn project_id(&self) -> GcpAuthFuture<'_, String> {
+            self.calls.fetch_add(1, Ordering::SeqCst);
+            Box::pin(async move {
+                self.project.map(str::to_owned).ok_or(
+                    litellm_auth_types::Error::CredentialAcquisition(
+                        litellm_auth_types::ErrorDetail::Unresolved {
+                            subject: "credential project",
+                        },
+                    ),
+                )
+            })
+        }
 
-    #[rstest::rstest]
-    #[case::padded("eyJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCJ9")]
-    #[case::quoted_prefixed_unpadded("\"base64:eyJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCJ9\"")]
-    #[case::url_safe_wrapped("eyJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCIsICJr\nZXkiOiAiPz8_In0=")]
-    #[tokio::test]
-    async fn base64_encoded_credentials_are_named_as_base64(#[case] configured: &str) {
-        assert!(
-            trusted_load_error(configured)
-                .await
-                .contains("is base64-encoded JSON, not the JSON itself")
-        );
-    }
-
-    async fn trusted_load_error(configured: &str) -> String {
-        match GcpProviderLoader
-            .load(CredentialSource::Trusted(SecretValue::new(configured)))
-            .await
-        {
-            Ok(_) => panic!("{configured:?} should not load"),
-            Err(error) => litellm_error::Report::new(&error).to_string(),
+        fn token(&self) -> GcpAuthFuture<'_, SecretValue> {
+            self.calls.fetch_add(1, Ordering::SeqCst);
+            Box::pin(async { Ok(SecretValue::new("adc-token")) })
         }
     }
 
-    #[rstest::rstest]
-    #[tokio::test]
-    async fn a_relative_key_file_named_like_json_is_read_as_a_file() {
-        let key = tempfile::Builder::new()
-            .prefix("{vertex")
-            .suffix(".json")
-            .tempfile_in(".")
-            .unwrap();
-        std::fs::write(key.path(), r#"{"project_id":"file-project"}"#).unwrap();
-        let relative_name = key.path().file_name().unwrap().to_str().unwrap();
-        let absolute_path = std::fs::canonicalize(key.path()).unwrap();
-        assert_eq!(
-            trusted_load_error(relative_name).await,
-            trusted_load_error(absolute_path.to_str().unwrap()).await
-        );
+    struct FakeLoader {
+        loads: Arc<AtomicUsize>,
+        provider: Arc<dyn GcpTokenSource>,
     }
 
-    #[rstest::rstest]
-    fn inline_and_trusted_credentials_never_share_a_cache_entry() {
-        assert_ne!(
-            CredentialSource::Inline(SecretValue::new("same-value")).cache_key(),
-            CredentialSource::Trusted(SecretValue::new("same-value")).cache_key()
-        );
+    impl GcpProviderLoader for FakeLoader {
+        fn load(&self, _source: CredentialSource) -> GcpAuthFuture<'_, Arc<dyn GcpTokenSource>> {
+            let loads = self.loads.clone();
+            let provider = self.provider.clone();
+            Box::pin(async move {
+                loads.fetch_add(1, Ordering::SeqCst);
+                Ok(provider)
+            })
+        }
+    }
+
+    fn config(value: Value) -> VertexConfig {
+        VertexConfig::from_sourced_optional_params(value.as_object().unwrap(), &BTreeMap::new())
+            .unwrap()
+    }
+
+    fn auth(calls: Arc<AtomicUsize>, loads: Arc<AtomicUsize>) -> VertexAuth {
+        let provider: Arc<dyn GcpTokenSource> = Arc::new(FakeProvider {
+            calls,
+            project: Some("adc-project"),
+        });
+        VertexAuth::new(Arc::new(FakeLoader { loads, provider }))
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn secret_names_cover_environment_reads() {
+        let seen = Arc::new(std::sync::Mutex::new(BTreeSet::<String>::new()));
+        let recorded = seen.clone();
+        let env = |name: &str| {
+            recorded.lock().unwrap().insert(name.to_string());
+            None
+        };
+        let auth = auth(Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0)));
+        auth.validate_environment(Vec::new(), None, &VertexConfig::default(), &env)
+            .await
+            .unwrap();
+        get_vertex_ai_location(&VertexConfig::default(), &env);
+        let allowed = secret_names()
+            .iter()
+            .map(|name| (*name).to_owned())
+            .collect::<BTreeSet<_>>();
+        assert!(seen.lock().unwrap().is_subset(&allowed));
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn explicit_token_and_header_do_not_acquire_adc() {
+        let loads = Arc::new(AtomicUsize::new(0));
+        let auth = auth(Arc::new(AtomicUsize::new(0)), loads.clone());
+        let configured = config(json!({"vertex_project":"project-1"}));
+        let explicit = auth
+            .validate_environment(Vec::new(), Some("access-token"), &configured, &|_| None)
+            .await
+            .unwrap();
+        assert_eq!(explicit.headers[0].1, "Bearer access-token");
+        let existing = auth
+            .validate_environment(
+                vec![("authorization".into(), "Bearer existing".into())],
+                None,
+                &configured,
+                &|_| None,
+            )
+            .await
+            .unwrap();
+        assert_eq!(existing.headers[0].1, "Bearer existing");
+        assert_eq!(loads.load(Ordering::SeqCst), 0);
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn provider_is_reused_across_authentication_calls() {
+        let calls = Arc::new(AtomicUsize::new(0));
+        let loads = Arc::new(AtomicUsize::new(0));
+        let auth = auth(calls.clone(), loads.clone());
+        let config = VertexConfig::default();
+        let environment = |_: &str| None;
+        let (first, second) = tokio::try_join!(
+            auth.validate_environment(Vec::new(), None, &config, &environment),
+            auth.validate_environment(Vec::new(), None, &config, &environment),
+        )
+        .unwrap();
+        assert_eq!(first.project_id, "adc-project");
+        assert_eq!(first.headers[0].1, "Bearer adc-token");
+        assert_eq!(first.project_id, second.project_id);
+        assert_eq!(first.headers, second.headers);
+        assert_eq!(loads.load(Ordering::SeqCst), 1);
+        assert_eq!(calls.load(Ordering::SeqCst), 4);
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn an_explicit_project_does_not_require_project_metadata_in_the_credentials() {
+        let auth = VertexAuth::new(Arc::new(FakeLoader {
+            loads: Arc::new(AtomicUsize::new(0)),
+            provider: Arc::new(FakeProvider {
+                calls: Arc::new(AtomicUsize::new(0)),
+                project: None,
+            }),
+        }));
+        let configured = VertexConfig::new(None, Some("configured-project".into()), None);
+        let environment = auth
+            .validate_environment(Vec::new(), None, &configured, &|_| None)
+            .await
+            .unwrap();
+        assert_eq!(environment.project_id, "configured-project");
+        assert_eq!(environment.headers[0].1, "Bearer adc-token");
     }
 }

litellm-rust/crates/auth-gcp/src/config.rs

@@ -146,6 +146,20 @@ pub enum CredentialSource {
     Adc,
 }
 
+impl CredentialSource {
+    pub fn from_vertex_environment(env_lookup: &dyn Fn(&str) -> Option<String>) -> Self {
+        VertexParams::CREDENTIALS
+            .resolve(&|_| None, env_lookup)
+            .map(|configured| Self::Trusted(SecretValue::new(configured)))
+            .unwrap_or_else(|| Self::application_default(env_lookup))
+    }
+
+    pub fn application_default(env_lookup: &dyn Fn(&str) -> Option<String>) -> Self {
+        non_empty_env(env_lookup, GOOGLE_APPLICATION_CREDENTIALS_ENV)
+            .map_or(Self::Adc, Self::ApplicationCredentials)
+    }
+}
+
 pub(crate) fn credential_source(
     config: &VertexConfig,
     env_lookup: &dyn Fn(&str) -> Option<String>,
@@ -158,12 +172,7 @@ pub(crate) fn credential_source(
             }
         };
     }
-    if let Some(configured) = VertexParams::CREDENTIALS.resolve(&|_| None, env_lookup) {
-        return CredentialSource::Trusted(SecretValue::new(configured));
-    }
-    non_empty_env(env_lookup, GOOGLE_APPLICATION_CREDENTIALS_ENV)
-        .map(CredentialSource::ApplicationCredentials)
-        .unwrap_or(CredentialSource::Adc)
+    CredentialSource::from_vertex_environment(env_lookup)
 }
 
 fn optional_credentials(
@@ -319,6 +328,24 @@ mod tests {
         ));
     }
 
+    #[rstest::rstest]
+    fn application_default_ignores_vertex_credentials() {
+        let env = |name: &str| match name {
+            VERTEXAI_CREDENTIALS_ENV => Some("environment-json".to_string()),
+            GOOGLE_APPLICATION_CREDENTIALS_ENV => Some("adc.json".to_string()),
+            _ => None,
+        };
+        assert!(
+            matches!(CredentialSource::application_default(&env), CredentialSource::ApplicationCredentials(path) if path == "adc.json")
+        );
+        assert!(matches!(
+            CredentialSource::application_default(
+                &|name| (name == VERTEXAI_CREDENTIALS_ENV).then(|| "environment-json".into())
+            ),
+            CredentialSource::Adc
+        ));
+    }
+
     #[rstest::rstest]
     fn params_become_a_deployment_sourced_config() {
         let params = VertexParams {

litellm-rust/crates/auth-gcp/src/identity.rs

@@ -0,0 +1,445 @@
+use crate::{
+    config::{CredentialSource, is_base64_encoded_json, names_a_credentials_file},
+    constants::{CLOUD_PLATFORM_SCOPE, GOOGLE_OAUTH_TOKEN_ENDPOINT},
+};
+use gcp_auth::{CustomServiceAccount, TokenProvider};
+use litellm_auth_types::{Error, ErrorDetail};
+use litellm_secret_value::SecretValue;
+use moka::future::Cache;
+use serde_json::Value;
+use sha2::{Digest, Sha256};
+use std::{future::Future, pin::Pin, sync::Arc};
+
+#[derive(Clone)]
+pub struct GcpAuth {
+    providers: Cache<CredentialCacheKey, Arc<dyn GcpTokenSource>>,
+    loader: Arc<dyn GcpProviderLoader>,
+}
+
+impl Default for GcpAuth {
+    fn default() -> Self {
+        Self::new(Arc::new(NativeProviderLoader))
+    }
+}
+
+impl GcpAuth {
+    pub fn new(loader: Arc<dyn GcpProviderLoader>) -> Self {
+        Self {
+            providers: Cache::builder().max_capacity(64).build(),
+            loader,
+        }
+    }
+
+    pub async fn access_token(&self, source: CredentialSource) -> Result<SecretValue, Error> {
+        self.load_provider(source).await?.token().await
+    }
+
+    pub(crate) async fn load_provider(
+        &self,
+        source: CredentialSource,
+    ) -> Result<Arc<dyn GcpTokenSource>, Error> {
+        self.providers
+            .try_get_with(source.cache_key(), self.loader.load(source))
+            .await
+            .map_err(|error| (*error).clone())
+    }
+}
+
+pub trait GcpTokenSource: Send + Sync {
+    fn project_id(&self) -> GcpAuthFuture<'_, String>;
+    fn token(&self) -> GcpAuthFuture<'_, SecretValue>;
+}
+
+pub trait GcpProviderLoader: Send + Sync {
+    fn load(&self, source: CredentialSource) -> GcpAuthFuture<'_, Arc<dyn GcpTokenSource>>;
+}
+
+pub type GcpAuthFuture<'a, T> = Pin<Box<dyn Future<Output = Result<T, Error>> + Send + 'a>>;
+
+struct ServiceAccountTokenSource(Arc<dyn TokenProvider>);
+
+impl GcpTokenSource for ServiceAccountTokenSource {
+    fn project_id(&self) -> GcpAuthFuture<'_, String> {
+        Box::pin(async move {
+            self.0
+                .project_id()
+                .await
+                .map(|project| project.to_string())
+                .map_err(auth_acquisition_error)
+        })
+    }
+
+    fn token(&self) -> GcpAuthFuture<'_, SecretValue> {
+        Box::pin(async move {
+            self.0
+                .token(&[CLOUD_PLATFORM_SCOPE])
+                .await
+                .map(|token| SecretValue::new(token.as_str()))
+                .map_err(auth_acquisition_error)
+        })
+    }
+}
+
+struct SdkTokenSource {
+    credentials: google_cloud_auth::credentials::AccessTokenCredentials,
+    project_id: Option<String>,
+}
+
+impl GcpTokenSource for SdkTokenSource {
+    fn project_id(&self) -> GcpAuthFuture<'_, String> {
+        Box::pin(async move {
+            self.project_id
+                .clone()
+                .ok_or(Error::CredentialAcquisition(ErrorDetail::Unresolved {
+                    subject: "GCP resource project",
+                }))
+        })
+    }
+
+    fn token(&self) -> GcpAuthFuture<'_, SecretValue> {
+        Box::pin(async move {
+            self.credentials
+                .access_token()
+                .await
+                .map(|token| SecretValue::new(token.token))
+                .map_err(auth_acquisition_error)
+        })
+    }
+}
+
+async fn load_application_credentials(path: &str) -> Result<Arc<dyn GcpTokenSource>, Error> {
+    let contents = tokio::fs::read_to_string(path)
+        .await
+        .map_err(auth_acquisition_error)?;
+    load_credentials_json(&contents)
+}
+
+fn load_credentials_json(contents: &str) -> Result<Arc<dyn GcpTokenSource>, Error> {
+    let configured: Value = serde_json::from_str(contents).map_err(auth_acquisition_error)?;
+    let kind = configured.get("type").and_then(Value::as_str);
+    let project_id = configured
+        .get("project_id")
+        .and_then(Value::as_str)
+        .map(str::to_string);
+    let credentials = match kind {
+        Some("authorized_user") => {
+            google_cloud_auth::credentials::user_account::Builder::new(configured)
+                .with_scopes([CLOUD_PLATFORM_SCOPE])
+                .build_access_token_credentials()
+        }
+        Some("impersonated_service_account") => {
+            google_cloud_auth::credentials::impersonated::Builder::new(configured)
+                .with_scopes([CLOUD_PLATFORM_SCOPE])
+                .build_access_token_credentials()
+        }
+        Some("external_account") => {
+            google_cloud_auth::credentials::external_account::Builder::new(configured)
+                .with_scopes([CLOUD_PLATFORM_SCOPE])
+                .build_access_token_credentials()
+        }
+        Some("service_account") | None => {
+            let provider =
+                CustomServiceAccount::from_json(contents).map_err(auth_acquisition_error)?;
+            return Ok(Arc::new(ServiceAccountTokenSource(Arc::new(provider))));
+        }
+        Some(_) => {
+            return Err(Error::InvalidConfiguration(ErrorDetail::InvalidType {
+                field: "Google application credentials type".into(),
+                expected: "a supported Google credential type",
+            }));
+        }
+    }
+    .map_err(auth_acquisition_error)?;
+    Ok(Arc::new(SdkTokenSource {
+        credentials,
+        project_id,
+    }))
+}
+
+struct NativeProviderLoader;
+
+impl GcpProviderLoader for NativeProviderLoader {
+    fn load(&self, source: CredentialSource) -> GcpAuthFuture<'_, Arc<dyn GcpTokenSource>> {
+        Box::pin(async move {
+            let provider: Arc<dyn TokenProvider> = match source {
+                CredentialSource::Inline(configured) => Arc::new(
+                    CustomServiceAccount::from_json(validate_request_credentials(
+                        configured.expose(),
+                    )?)
+                    .map_err(auth_acquisition_error)?,
+                ),
+                CredentialSource::Trusted(configured) => {
+                    return load_trusted_credentials(configured.expose()).await;
+                }
+                CredentialSource::ApplicationCredentials(path) => {
+                    return load_application_credentials(&path).await;
+                }
+                CredentialSource::Adc => {
+                    gcp_auth::provider().await.map_err(auth_acquisition_error)?
+                }
+            };
+            Ok(Arc::new(ServiceAccountTokenSource(provider)) as Arc<dyn GcpTokenSource>)
+        })
+    }
+}
+
+fn validate_request_credentials(configured: &str) -> Result<&str, Error> {
+    let token_uri = serde_json::from_str::<Value>(configured)
+        .ok()
+        .and_then(|credentials| {
+            credentials
+                .get("token_uri")
+                .and_then(Value::as_str)
+                .map(str::to_string)
+        });
+    if token_uri.as_deref() != Some(GOOGLE_OAUTH_TOKEN_ENDPOINT) {
+        return Err(Error::InvalidConfiguration(ErrorDetail::InvalidType {
+            field: "request-controlled vertex_credentials token_uri".into(),
+            expected: "the canonical Google OAuth token endpoint",
+        }));
+    }
+    Ok(configured)
+}
+
+impl CredentialSource {
+    fn cache_key(&self) -> CredentialCacheKey {
+        match self {
+            Self::Inline(configured) => {
+                CredentialCacheKey::Inline(Sha256::digest(configured.expose()).into())
+            }
+            Self::Trusted(configured) => {
+                CredentialCacheKey::Trusted(Sha256::digest(configured.expose()).into())
+            }
+            Self::ApplicationCredentials(path) => {
+                CredentialCacheKey::ApplicationCredentials(path.clone())
+            }
+            Self::Adc => CredentialCacheKey::Adc,
+        }
+    }
+}
+
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
+enum CredentialCacheKey {
+    Inline([u8; 32]),
+    Trusted([u8; 32]),
+    ApplicationCredentials(String),
+    Adc,
+}
+
+async fn load_trusted_credentials(configured: &str) -> Result<Arc<dyn GcpTokenSource>, Error> {
+    if is_base64_encoded_json(configured) {
+        return Err(Error::InvalidConfiguration(
+            ErrorDetail::Base64EncodedJson {
+                field: "vertex_credentials",
+            },
+        ));
+    }
+    if names_a_credentials_file(configured) {
+        load_application_credentials(configured).await
+    } else {
+        load_credentials_json(configured)
+    }
+}
+
+fn auth_acquisition_error(error: impl std::error::Error + Send + Sync + 'static) -> Error {
+    Error::CredentialAcquisition(litellm_auth_types::ErrorDetail::failed(
+        "Google credentials",
+        error,
+    ))
+}
+
+#[cfg(test)]
+mod tests {
+    use litellm_secret_value::SecretValue;
+
+    use super::*;
+
+    #[rstest::rstest]
+    #[case::authorized_user("authorized_user", 2)]
+    #[case::external_account("external_account", 2)]
+    #[case::impersonated_service_account("impersonated_service_account", 4)]
+    #[tokio::test]
+    async fn explicit_application_credentials_refresh_expired_tokens(
+        #[case] kind: &str,
+        #[case] expected_requests: usize,
+        #[values(false, true)] inline: bool,
+    ) {
+        use axum::{Json, Router, routing::post};
+        use std::sync::atomic::{AtomicUsize, Ordering};
+
+        let _ = rustls::crypto::ring::default_provider().install_default();
+        let requests = Arc::new(AtomicUsize::new(0));
+        let recorded = requests.clone();
+        let impersonations = requests.clone();
+        let app = Router::new().route("/token", post(move || {
+            let recorded = recorded.clone();
+            async move {
+                Json(serde_json::json!({
+                    "access_token": format!("token-{}", recorded.fetch_add(1, Ordering::SeqCst)),
+                    "expires_in": 3600,
+                    "token_type": "Bearer",
+                    "issued_token_type": "urn:ietf:params:oauth:token-type:access_token",
+                }))
+            }
+        }));
+        let app = app.route("/v1/projects/-/serviceAccounts/test-principal:generateAccessToken", post(move || {
+            let recorded = impersonations.clone();
+            async move {
+                let expiration = (time::OffsetDateTime::now_utc() + time::Duration::hours(1)).format(&time::format_description::well_known::Rfc3339).unwrap();
+                Json(serde_json::json!({
+                    "accessToken": format!("impersonated-{}", recorded.fetch_add(1, Ordering::SeqCst)),
+                    "expireTime": expiration,
+                }))
+            }
+        }));
+        let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+        let address = listener.local_addr().unwrap();
+        let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
+        let file = tempfile::NamedTempFile::new().unwrap();
+        let subject_token = tempfile::NamedTempFile::new().unwrap();
+        std::fs::write(subject_token.path(), "subject-token").unwrap();
+        let configured = if kind == "external_account" {
+            serde_json::json!({
+                "type": "external_account", "audience": "//iam.googleapis.com/projects/test/locations/global/workloadIdentityPools/test/providers/test",
+                "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", "token_url": format!("http://{address}/token"),
+                "credential_source": { "file": subject_token.path().to_str().unwrap() }, "quota_project_id": "quota-project"
+            })
+        } else {
+            let source = serde_json::json!({
+                "type": "authorized_user", "client_id": "user-client", "client_secret": "user-secret",
+                "refresh_token": "user-refresh", "quota_project_id": "quota-project", "token_uri": format!("http://{address}/token")
+            });
+            if kind == "impersonated_service_account" {
+                serde_json::json!({
+                    "type": kind, "source_credentials": source, "quota_project_id": "quota-project",
+                    "service_account_impersonation_url": format!("http://{address}/v1/projects/-/serviceAccounts/test-principal:generateAccessToken")
+                })
+            } else {
+                source
+            }
+        };
+        std::fs::write(file.path(), configured.to_string()).unwrap();
+        let source = if inline {
+            CredentialSource::Trusted(SecretValue::new(configured.to_string()))
+        } else {
+            CredentialSource::ApplicationCredentials(file.path().to_str().unwrap().into())
+        };
+        let provider = NativeProviderLoader.load(source).await.unwrap();
+
+        assert!(matches!(
+            provider.project_id().await,
+            Err(Error::CredentialAcquisition(ErrorDetail::Unresolved {
+                subject: "GCP resource project"
+            }))
+        ));
+        let original = provider.token().await.unwrap();
+        tokio::time::pause();
+        tokio::time::advance(std::time::Duration::from_secs(3601)).await;
+        tokio::time::resume();
+        let refreshed = provider.token().await.unwrap();
+        assert_ne!(original, refreshed);
+        assert_eq!(requests.load(Ordering::SeqCst), expected_requests);
+        server.abort();
+    }
+
+    #[rstest::rstest]
+    #[case::canonical_endpoint(r#"{"token_uri":"https://oauth2.googleapis.com/token"}"#, true)]
+    #[case::noncanonical_endpoint(r#"{"token_uri":"http://127.0.0.1/token"}"#, false)]
+    #[case::missing_endpoint("{}", false)]
+    fn request_credentials_require_canonical_token_endpoint(
+        #[case] credentials: &str,
+        #[case] accepted: bool,
+    ) {
+        assert_eq!(validate_request_credentials(credentials).is_ok(), accepted);
+    }
+
+    #[rstest::rstest]
+    #[case::missing_file("/nonexistent/vertex-key.json", "Google credentials")]
+    #[case::malformed_json(" {\"type\": \"service_account\",", "Google credentials")]
+    #[case::quoted_json("'{\"type\": \"service_account\"}'", "Google credentials")]
+    #[tokio::test]
+    async fn trusted_credentials_are_read_as_a_file_unless_they_are_json(
+        #[case] configured: &str,
+        #[case] expected: &str,
+    ) {
+        let Err(error) = NativeProviderLoader
+            .load(CredentialSource::Trusted(SecretValue::new(configured)))
+            .await
+        else {
+            panic!("{configured:?} should not load");
+        };
+        let report = litellm_error::Report::new(&error).to_string();
+        assert!(report.contains(expected), "{report}");
+    }
+
+    #[rstest::rstest]
+    #[case::padded("eyJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCJ9")]
+    #[case::quoted_prefixed_unpadded("\"base64:eyJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCJ9\"")]
+    #[case::url_safe_wrapped("eyJ0eXBlIjogInNlcnZpY2VfYWNjb3VudCIsICJr\nZXkiOiAiPz8_In0=")]
+    #[tokio::test]
+    async fn base64_encoded_credentials_are_named_as_base64(#[case] configured: &str) {
+        assert!(
+            trusted_load_error(configured)
+                .await
+                .contains("is base64-encoded JSON, not the JSON itself")
+        );
+    }
+
+    async fn trusted_load_error(configured: &str) -> String {
+        match NativeProviderLoader
+            .load(CredentialSource::Trusted(SecretValue::new(configured)))
+            .await
+        {
+            Ok(_) => panic!("{configured:?} should not load"),
+            Err(error) => litellm_error::Report::new(&error).to_string(),
+        }
+    }
+
+    #[rstest::rstest]
+    #[tokio::test]
+    async fn a_relative_key_file_named_like_json_is_read_as_a_file() {
+        let key = tempfile::Builder::new()
+            .prefix("{vertex")
+            .suffix(".json")
+            .tempfile_in(".")
+            .unwrap();
+        std::fs::write(key.path(), r#"{"project_id":"file-project"}"#).unwrap();
+        let relative_name = key.path().file_name().unwrap().to_str().unwrap();
+        let absolute_path = std::fs::canonicalize(key.path()).unwrap();
+        assert_eq!(
+            trusted_load_error(relative_name).await,
+            trusted_load_error(absolute_path.to_str().unwrap()).await
+        );
+    }
+
+    #[rstest::rstest]
+    #[tokio::test]
+    async fn unknown_application_credential_types_are_rejected_without_service_account_fallback() {
+        let file = tempfile::NamedTempFile::new().unwrap();
+        std::fs::write(
+            file.path(),
+            r#"{"type":"unsupported","private_key":"credential-value"}"#,
+        )
+        .unwrap();
+        let result = GcpAuth::default()
+            .access_token(CredentialSource::ApplicationCredentials(
+                file.path().to_str().unwrap().into(),
+            ))
+            .await;
+        let error = result.unwrap_err();
+        assert!(matches!(error, Error::InvalidConfiguration(_)));
+        assert!(
+            !litellm_error::Report::new(&error)
+                .to_string()
+                .contains("credential-value")
+        );
+    }
+
+    #[rstest::rstest]
+    fn inline_and_trusted_credentials_never_share_a_cache_entry() {
+        assert_ne!(
+            CredentialSource::Inline(SecretValue::new("same-value")).cache_key(),
+            CredentialSource::Trusted(SecretValue::new("same-value")).cache_key()
+        );
+    }
+}

litellm-rust/crates/auth-gcp/src/lib.rs

@@ -1,16 +1,16 @@
 mod auth;
 mod config;
 pub mod constants;
+mod identity;
 #[cfg(feature = "google-sdk")]
 mod sdk;
 
-pub use auth::{
-    VertexAuth, VertexAuthFuture, VertexEnvironment, VertexProviderLoader, VertexTokenSource,
-};
+pub use auth::{VertexAuth, VertexEnvironment};
 pub use config::{
     CredentialSource, VertexConfig, get_vertex_ai_location, get_vertex_ai_project,
     get_vertex_ai_project_from_credentials,
 };
 pub use constants::secret_names;
+pub use identity::{GcpAuth, GcpAuthFuture, GcpProviderLoader, GcpTokenSource};
 #[cfg(feature = "google-sdk")]
 pub use sdk::GoogleCredentials;

litellm-rust/crates/auth-gcp/tests/vertex_auth.rs

@@ -1,120 +0,0 @@
-use std::{
-    collections::{BTreeMap, BTreeSet},
-    sync::{
-        Arc,
-        atomic::{AtomicUsize, Ordering},
-    },
-};
-
-use litellm_auth_gcp::{
-    CredentialSource, VertexAuth, VertexAuthFuture, VertexConfig, VertexProviderLoader,
-    VertexTokenSource, get_vertex_ai_location, secret_names,
-};
-use rstest::rstest;
-use serde_json::{Value, json};
-
-struct FakeProvider {
-    calls: Arc<AtomicUsize>,
-}
-
-impl VertexTokenSource for FakeProvider {
-    fn project_id(&self) -> VertexAuthFuture<'_, String> {
-        self.calls.fetch_add(1, Ordering::SeqCst);
-        Box::pin(async { Ok("adc-project".into()) })
-    }
-
-    fn token(&self) -> VertexAuthFuture<'_, String> {
-        self.calls.fetch_add(1, Ordering::SeqCst);
-        Box::pin(async { Ok("adc-token".into()) })
-    }
-}
-
-struct FakeLoader {
-    loads: Arc<AtomicUsize>,
-    provider: Arc<dyn VertexTokenSource>,
-}
-
-impl VertexProviderLoader for FakeLoader {
-    fn load(&self, _source: CredentialSource) -> VertexAuthFuture<'_, Arc<dyn VertexTokenSource>> {
-        let loads = self.loads.clone();
-        let provider = self.provider.clone();
-        Box::pin(async move {
-            loads.fetch_add(1, Ordering::SeqCst);
-            Ok(provider)
-        })
-    }
-}
-
-fn config(value: Value) -> VertexConfig {
-    VertexConfig::from_sourced_optional_params(value.as_object().unwrap(), &BTreeMap::new())
-        .unwrap()
-}
-
-fn auth(calls: Arc<AtomicUsize>, loads: Arc<AtomicUsize>) -> VertexAuth {
-    let provider: Arc<dyn VertexTokenSource> = Arc::new(FakeProvider { calls });
-    VertexAuth::new(Arc::new(FakeLoader { loads, provider }))
-}
-
-#[rstest]
-#[tokio::test]
-async fn secret_names_cover_environment_reads() {
-    let seen = Arc::new(std::sync::Mutex::new(BTreeSet::<String>::new()));
-    let recorded = seen.clone();
-    let env = |name: &str| {
-        recorded.lock().unwrap().insert(name.to_string());
-        None
-    };
-    let auth = auth(Arc::new(AtomicUsize::new(0)), Arc::new(AtomicUsize::new(0)));
-    auth.validate_environment(Vec::new(), None, &VertexConfig::default(), &env)
-        .await
-        .unwrap();
-    get_vertex_ai_location(&VertexConfig::default(), &env);
-    assert!(
-        seen.lock()
-            .unwrap()
-            .iter()
-            .all(|name| secret_names().contains(&name.as_str()))
-    );
-}
-
-#[rstest]
-#[tokio::test]
-async fn explicit_token_and_header_do_not_acquire_adc() {
-    let loads = Arc::new(AtomicUsize::new(0));
-    let auth = auth(Arc::new(AtomicUsize::new(0)), loads.clone());
-    let configured = config(json!({"vertex_project":"project-1"}));
-    let explicit = auth
-        .validate_environment(Vec::new(), Some("access-token"), &configured, &|_| None)
-        .await
-        .unwrap();
-    assert_eq!(explicit.headers[0].1, "Bearer access-token");
-    let existing = auth
-        .validate_environment(
-            vec![("authorization".into(), "Bearer existing".into())],
-            None,
-            &configured,
-            &|_| None,
-        )
-        .await
-        .unwrap();
-    assert_eq!(existing.headers[0].1, "Bearer existing");
-    assert_eq!(loads.load(Ordering::SeqCst), 0);
-}
-
-#[rstest]
-#[tokio::test]
-async fn provider_is_reused_across_authentication_calls() {
-    let calls = Arc::new(AtomicUsize::new(0));
-    let loads = Arc::new(AtomicUsize::new(0));
-    let auth = auth(calls.clone(), loads.clone());
-    for _ in 0..2 {
-        let environment = auth
-            .validate_environment(Vec::new(), None, &VertexConfig::default(), &|_| None)
-            .await
-            .unwrap();
-        assert_eq!(environment.project_id, "adc-project");
-        assert_eq!(environment.headers[0].1, "Bearer adc-token");
-    }
-    assert_eq!(loads.load(Ordering::SeqCst), 1);
-    assert_eq!(calls.load(Ordering::SeqCst), 4);
-}

litellm-rust/crates/auth-types/src/error.rs

@@ -92,6 +92,11 @@ impl Error {
 
 #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
 pub enum ErrorDetail {
+    #[error("{operation} returned HTTP {status}")]
+    UnexpectedStatus {
+        operation: &'static str,
+        status: u16,
+    },
     #[error("{field} must be {expected}")]
     InvalidType {
         field: String,

litellm-rust/crates/auth-types/src/params/aws.rs

@@ -53,6 +53,9 @@ pub struct AwsParams {
 }
 
 impl AwsParams {
+    pub const ROLE_ARN_ENV: &str = "AWS_ROLE_ARN";
+    pub const WEB_IDENTITY_TOKEN_FILE_ENV: &str = "AWS_WEB_IDENTITY_TOKEN_FILE";
+
     pub const ACCESS_KEY_ID: ParamSpec = spec(
         "access key id",
         ParamKind::Credential,
@@ -149,6 +152,10 @@ impl AwsParams {
             AwsParams::SPECS
                 .iter()
                 .flat_map(|spec| spec.env.iter().copied())
+                .chain([
+                    AwsParams::ROLE_ARN_ENV,
+                    AwsParams::WEB_IDENTITY_TOKEN_FILE_ENV,
+                ])
                 .fold(Vec::new(), |mut names, name| {
                     if !names.contains(&name) {
                         names.push(name);

litellm-rust/crates/auth-types/tests/aws_params.rs

@@ -3,21 +3,13 @@ use rstest::rstest;
 use serde_json::{Map, Value};
 
 #[rstest]
-fn fields_name_every_param_once_and_in_declaration_order() {
+fn every_named_param_roundtrips_without_loss() {
     let filled: Map<String, Value> = AwsParams::fields()
         .map(|name| (name.to_string(), Value::from(format!("value-of-{name}"))))
         .collect();
     let typed = AwsParams::from_optional_params(&filled);
     let serialized = serde_json::to_value(&typed).unwrap();
-    assert_eq!(
-        serialized
-            .as_object()
-            .unwrap()
-            .keys()
-            .map(String::as_str)
-            .collect::<Vec<_>>(),
-        AwsParams::fields().collect::<Vec<_>>()
-    );
+    assert_eq!(filled.len(), AwsParams::fields().count());
     assert_eq!(serialized, Value::Object(filled.clone()));
     assert_eq!(
         serde_json::from_value::<AwsParams>(Value::Object(filled)).unwrap(),

litellm-rust/crates/cache-gcs/src/cache.rs

@@ -8,7 +8,7 @@ use litellm_cache::{
 use litellm_http::Client;
 use percent_encoding::{AsciiSet, NON_ALPHANUMERIC, percent_encode};
 
-use crate::{GcpTokenSource, TokenSource};
+use crate::{GcsTokenSource, TokenSource};
 
 pub const DEFAULT_ENDPOINT: &str = "https://storage.googleapis.com";
 
@@ -62,7 +62,7 @@ pub struct GcsCache<S: CacheCodec> {
 
 impl<S: CacheCodec> GcsCache<S> {
     pub fn new(config: GcsConfig, client: Client, codec: S) -> Self {
-        let token = Arc::new(GcpTokenSource::new(config.path_service_account.clone()));
+        let token = Arc::new(GcsTokenSource::new(config.path_service_account.clone()));
         Self::with_token_source(config, client, codec, token)
     }
 

litellm-rust/crates/cache-gcs/src/lib.rs

@@ -2,4 +2,4 @@ mod cache;
 mod token;
 
 pub use cache::{DEFAULT_ENDPOINT, GcsCache, GcsConfig, key_prefix};
-pub use token::{GcpTokenSource, StaticTokenSource, TokenSource};
+pub use token::{GcsTokenSource, StaticTokenSource, TokenSource};

litellm-rust/crates/cache-gcs/src/token.rs

@@ -1,7 +1,6 @@
 use std::{future::Future, pin::Pin};
 
-use litellm_auth_gcp::{VertexAuth, VertexConfig};
-use litellm_auth_types::{InputSource, Sourced};
+use litellm_auth_gcp::{CredentialSource, GcpAuth};
 use litellm_secret_value::SecretValue;
 
 use litellm_cache::Error;
@@ -10,28 +9,30 @@ pub trait TokenSource: Send + Sync + 'static {
     fn bearer_token(&self) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + '_>>;
 }
 
-pub struct GcpTokenSource {
-    auth: VertexAuth,
-    config: VertexConfig,
+pub struct GcsTokenSource {
+    auth: GcpAuth,
+    credentials: Option<String>,
 }
 
-impl GcpTokenSource {
+impl GcsTokenSource {
     pub fn new(path_service_account: Option<String>) -> Self {
-        let credentials = path_service_account
-            .map(|path| Sourced::new(SecretValue::new(path), InputSource::Deployment));
         Self {
-            auth: VertexAuth::default(),
-            config: VertexConfig::new(credentials, None, None),
+            auth: GcpAuth::default(),
+            credentials: path_service_account.filter(|path| !path.trim().is_empty()),
         }
     }
 }
 
-impl TokenSource for GcpTokenSource {
+impl TokenSource for GcsTokenSource {
     fn bearer_token(&self) -> Pin<Box<dyn Future<Output = Result<String, Error>> + Send + '_>> {
         Box::pin(async move {
             self.auth
-                .access_token(&self.config, &|name| std::env::var(name).ok())
+                .access_token(match &self.credentials {
+                    Some(path) => CredentialSource::Trusted(SecretValue::new(path.clone())),
+                    None => CredentialSource::application_default(&|name| std::env::var(name).ok()),
+                })
                 .await
+                .map(|token| token.expose().to_owned())
                 .map_err(Error::unavailable)
         })
     }

litellm-rust/crates/cache-s3/src/auth.rs

@@ -36,9 +36,12 @@ impl ProvideCredentials for S3Credentials {
                 self.config.secret_access_key.clone(),
             ) {
                 return Ok(AwsCredentials::new(
-                    access_key_id,
-                    secret_access_key,
-                    self.config.session_token.clone(),
+                    access_key_id.expose(),
+                    secret_access_key.expose(),
+                    self.config
+                        .session_token
+                        .as_ref()
+                        .map(|value| value.expose().to_owned()),
                     None,
                     "litellm-s3-cache",
                 ));

litellm-rust/crates/cache-s3/tests/auth.rs

@@ -5,9 +5,9 @@ use rstest::rstest;
 
 fn explicit(session_token: Option<&str>) -> AwsAuthConfig {
     AwsAuthConfig {
-        access_key_id: Some("key".to_string()),
-        secret_access_key: Some("secret".to_string()),
-        session_token: session_token.map(str::to_string),
+        access_key_id: Some("key".into()),
+        secret_access_key: Some("secret".into()),
+        session_token: session_token.map(Into::into),
         region_name: Some("us-east-1".to_string()),
         ..Default::default()
     }

litellm-rust/crates/cache-s3/tests/support/mod.rs

@@ -23,8 +23,8 @@ pub fn config(endpoint: &str) -> S3CacheConfig {
             url: endpoint.to_string(),
         }),
         auth: AwsAuthConfig {
-            access_key_id: Some("key".to_string()),
-            secret_access_key: Some("secret".to_string()),
+            access_key_id: Some("key".into()),
+            secret_access_key: Some("secret".into()),
             region_name: Some("us-east-1".to_string()),
             ..Default::default()
         },

litellm-rust/crates/inference-ocr/tests/resources.rs

@@ -7,7 +7,7 @@ use std::sync::{
 
 use litellm_auth::AuthServices;
 use litellm_auth_gcp::{
-    CredentialSource, VertexAuth, VertexAuthFuture, VertexProviderLoader, VertexTokenSource,
+    CredentialSource, GcpAuthFuture, GcpProviderLoader, GcpTokenSource, VertexAuth,
 };
 use litellm_http::{HttpSettings, Resolution};
 use litellm_inference::resources::CoreResources;
@@ -20,28 +20,28 @@ use support::{ReceivedRequest, json_response, upstream};
 
 struct TokenSource(String);
 
-impl VertexTokenSource for TokenSource {
-    fn project_id(&self) -> VertexAuthFuture<'_, String> {
+impl GcpTokenSource for TokenSource {
+    fn project_id(&self) -> GcpAuthFuture<'_, String> {
         Box::pin(async { Ok(self.0.clone()) })
     }
 
-    fn token(&self) -> VertexAuthFuture<'_, String> {
-        Box::pin(async { Ok(self.0.clone()) })
+    fn token(&self) -> GcpAuthFuture<'_, litellm_secret_value::SecretValue> {
+        Box::pin(async { Ok(litellm_secret_value::SecretValue::new(self.0.clone())) })
     }
 }
 
 #[derive(Default)]
 struct Loader(AtomicUsize);
 
-impl VertexProviderLoader for Loader {
-    fn load(&self, source: CredentialSource) -> VertexAuthFuture<'_, Arc<dyn VertexTokenSource>> {
+impl GcpProviderLoader for Loader {
+    fn load(&self, source: CredentialSource) -> GcpAuthFuture<'_, Arc<dyn GcpTokenSource>> {
         Box::pin(async move {
             self.0.fetch_add(1, Ordering::SeqCst);
             let identity = match source {
                 CredentialSource::Trusted(secret) => secret.expose().to_string(),
                 other => panic!("unexpected credential source: {other:?}"),
             };
-            Ok(Arc::new(TokenSource(identity)) as Arc<dyn VertexTokenSource>)
+            Ok(Arc::new(TokenSource(identity)) as Arc<dyn GcpTokenSource>)
         })
     }
 }

litellm-rust/crates/secret-value/src/secret_value.rs

@@ -16,6 +16,18 @@ impl SecretValue {
     }
 }
 
+impl From<String> for SecretValue {
+    fn from(value: String) -> Self {
+        Self::new(value)
+    }
+}
+
+impl From<&str> for SecretValue {
+    fn from(value: &str) -> Self {
+        Self::new(value)
+    }
+}
+
 impl PartialEq for SecretValue {
     fn eq(&self, other: &Self) -> bool {
         subtle::ConstantTimeEq::ct_eq(self.0.as_bytes(), other.0.as_bytes()).into()

litellm-rust/crates/secrets-aws/src/auth.rs

@@ -39,30 +39,15 @@ impl Credentials {
         Self {
             auth,
             config: AwsAuthConfig {
-                access_key_id: context
-                    .access_key_id
-                    .as_ref()
-                    .map(|value| value.expose().to_owned()),
-                secret_access_key: context
-                    .secret_access_key
-                    .as_ref()
-                    .map(|value| value.expose().to_owned()),
-                session_token: context
-                    .session_token
-                    .as_ref()
-                    .map(|value| value.expose().to_owned()),
+                access_key_id: context.access_key_id.as_ref().cloned(),
+                secret_access_key: context.secret_access_key.as_ref().cloned(),
+                session_token: context.session_token.as_ref().cloned(),
                 region_name: region(settings, environment.as_ref()).ok(),
                 role_name: settings.aws_role_name.clone(),
                 session_name: settings.aws_session_name.clone(),
-                external_id: settings
-                    .aws_external_id
-                    .as_ref()
-                    .map(|v| v.expose().to_owned()),
+                external_id: settings.aws_external_id.as_ref().cloned(),
                 profile_name: settings.aws_profile_name.clone(),
-                web_identity_token: settings
-                    .aws_web_identity_token
-                    .as_ref()
-                    .map(|v| v.expose().to_owned()),
+                web_identity_token: settings.aws_web_identity_token.as_ref().cloned(),
                 sts_endpoint: settings.aws_sts_endpoint.clone(),
             },
             environment,

litellm/__init__.py

@@ -1796,6 +1796,9 @@ if TYPE_CHECKING:
     from .llms.bespoke.decisions.transformation import (
         BespokeDecisionsConfig as BespokeDecisionsConfig,
     )
+    from .llms.empiriolabs.decisions.transformation import (
+        EmpirioLabsDecisionsConfig as EmpirioLabsDecisionsConfig,
+    )
     from .llms.eu_jev.decisions.transformation import (
         EuJevDecisionsConfig as EuJevDecisionsConfig,
     )

litellm/_lazy_imports_registry.py

@@ -176,6 +176,7 @@ LLM_CONFIG_NAMES: Final = (
     "OllamaDecisionsConfig",
     "LayaDecisionsConfig",
     "BespokeDecisionsConfig",
+    "EmpirioLabsDecisionsConfig",
     "EuJevDecisionsConfig",
     "OpenAIDecisionsConfig",
     "AzureDecisionsConfig",
@@ -759,6 +760,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = {
     "OllamaDecisionsConfig": (".llms.ollama.decisions.transformation", "OllamaDecisionsConfig"),
     "LayaDecisionsConfig": (".llms.laya.decisions.transformation", "LayaDecisionsConfig"),
     "BespokeDecisionsConfig": (".llms.bespoke.decisions.transformation", "BespokeDecisionsConfig"),
+    "EmpirioLabsDecisionsConfig": (".llms.empiriolabs.decisions.transformation", "EmpirioLabsDecisionsConfig"),
     "EuJevDecisionsConfig": (".llms.eu_jev.decisions.transformation", "EuJevDecisionsConfig"),
     "OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"),
     "AzureDecisionsConfig": (".llms.azure.decisions.transformation", "AzureDecisionsConfig"),

litellm/accounting/__init__.py

@@ -1,7 +1,7 @@
 """Public accounting contracts. Importing these values does not enable accounting."""
 
 from litellm.accounting.measurements import measurements_for
-from litellm.accounting.protocols import BudgetEnforcer, CostAggregator, CostCompiler, UsageMeter
+from litellm.accounting.protocols import CostAggregator
 from litellm.accounting.types import (
     AdmissionDecision,
     AggregateQuery,
@@ -33,9 +33,7 @@ __all__ = [
     "AdmissionDecision",
     "AggregateQuery",
     "AggregateRow",
-    "BudgetEnforcer",
     "CostAggregator",
-    "CostCompiler",
     "CostEvent",
     "DeliveryState",
     "DeliveryTarget",
@@ -56,7 +54,6 @@ __all__ = [
     "PricingSnapshot",
     "Rate",
     "Unpriced",
-    "UsageMeter",
     "measurements_for",
     "validate_aggregate_rows",
 ]

litellm/accounting/projection.py

@@ -0,0 +1,304 @@
+from collections.abc import Iterator, Mapping
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from itertools import chain
+from types import MappingProxyType
+from typing import Final
+
+from pydantic import TypeAdapter, ValidationError
+
+from litellm.constants import (
+    BILLED_RETRY_ATTEMPT_METADATA_KEY,
+    INTERNAL_CALL_ORIGIN_METADATA_KEY,
+    MODEL_INSIGHTS_DEFAULT_TASK,
+    MODEL_INSIGHTS_TASK_TAG_PREFIX,
+)
+from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import GuardrailCostByUnitEntry
+from litellm.proxy.db.daily_spend_bulk_upsert import DAILY_SPEND_TABLES, daily_spend_entity_ids
+from litellm.proxy.db.model_insights_tasks import load_model_insight_tasks
+from litellm.proxy.db.model_usage_rollup import date_from_start_time
+from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING
+from litellm.proxy.spend_tracking.compression_savings import extract_compression_saved_tokens
+from litellm.proxy.spend_tracking.savings import extract_cache_creation_tokens, extract_cache_read_tokens
+from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN
+
+_FIELDS: Final = TypeAdapter(dict[str, object])
+_STRINGS: Final = TypeAdapter(tuple[str, ...])
+_ROWS: Final = TypeAdapter(tuple[dict[str, object], ...])
+_TASK_TYPES: Final = frozenset(load_model_insight_tasks())
+
+
+@dataclass(frozen=True, slots=True)
+class Contribution:
+    table: str
+    key: tuple[tuple[str, str], ...]
+    increments: tuple[tuple[str, int | float], ...]
+    values: tuple[tuple[str, str | None], ...] = ()
+
+
+def _fields(value: object) -> Mapping[str, object]:
+    if isinstance(value, str):
+        return _FIELDS.validate_json(value)
+    return _FIELDS.validate_python(value) if isinstance(value, Mapping) else MappingProxyType({})
+
+
+def _text(value: object) -> str:
+    return "" if value is None else str(value)
+
+
+def _number(value: object) -> int | float:
+    return value if isinstance(value, (int, float)) else 0
+
+
+def _tags(row: Mapping[str, object]) -> tuple[str, ...]:
+    value: Final = row.get("request_tags")
+    if value is None:
+        return ()
+    return _STRINGS.validate_json(value) if isinstance(value, str) else _STRINGS.validate_python(value)
+
+
+def _date(row: Mapping[str, object]) -> str | None:
+    start: Final = row.get("startTime")
+    return date_from_start_time(start) if isinstance(start, (datetime, str)) else None
+
+
+def _spend(table: str, key: tuple[tuple[str, str], ...], spend: float) -> Contribution:
+    return Contribution(table, key, (("spend", spend),))
+
+
+def _entity_contributions(row: Mapping[str, object], metadata: Mapping[str, object]) -> tuple[Contribution, ...]:
+    spend: Final = _number(row.get("spend"))
+    entities: Final = _fields(metadata.get("accounting_entities"))
+    identities: Final = (
+        ("LiteLLM_VerificationToken", "token", entities.get("api_key")),
+        ("LiteLLM_UserTable", "user_id", entities.get("user")),
+        ("LiteLLM_UserTable", "user_id", entities.get("proxy_budget_name")),
+        ("LiteLLM_TeamTable", "team_id", entities.get("team_id")),
+        ("LiteLLM_OrganizationTable", "organization_id", entities.get("organization_id")),
+        ("LiteLLM_EndUserTable", "user_id", entities.get("end_user")),
+        ("LiteLLM_AgentsTable", "agent_id", entities.get("agent_id")),
+        ("LiteLLM_ProjectTable", "project_id", entities.get("project_id")),
+    )
+    groups: Final = _STRINGS.validate_python(metadata.get("accounting_model_access_groups", ()))
+    return (
+        *(
+            _spend(table, ((column, identity),), spend)
+            for table, column, identity in identities
+            if isinstance(identity, str)
+        ),
+        *(_spend("LiteLLM_TagTable", (("tag_name", tag),), spend) for tag in _tags(row) if tag),
+        *(_spend("LiteLLM_ModelAccessGroupBudgetTable", (("access_group_name", group),), spend) for group in groups),
+        *_membership_contributions(entities, spend),
+    )
+
+
+def _membership_contributions(row: Mapping[str, object], spend: float) -> tuple[Contribution, ...]:
+    user: Final = row.get("user")
+    if not isinstance(user, str):
+        return ()
+    owners: Final = (
+        ("LiteLLM_TeamMembership", "team_id", row.get("team_id")),
+        ("LiteLLM_OrganizationMembership", "organization_id", row.get("organization_id")),
+    )
+    return tuple(
+        _spend(table, ((column, identity), ("user_id", user)), spend)
+        for table, column, identity in owners
+        if isinstance(identity, str)
+    )
+
+
+def _daily_counters(row: Mapping[str, object], metadata: Mapping[str, object]) -> tuple[tuple[str, int | float], ...]:
+    usage: Final = _fields(metadata.get("usage_object"))
+    savings: Final = _fields(metadata.get("accounting_savings"))
+    internal: Final = bool(metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY))
+    caller: Final = not internal and metadata.get(BILLED_RETRY_ATTEMPT_METADATA_KEY) is not True
+    success: Final = metadata.get("status") != "failure"
+    duration: Final = row.get("request_duration_ms")
+    timed: Final = not internal and success and isinstance(duration, int) and duration >= 0
+    return (
+        ("spend", _number(row.get("spend"))),
+        ("prompt_tokens", _number(row.get("prompt_tokens"))),
+        ("completion_tokens", _number(row.get("completion_tokens"))),
+        ("api_requests", int(caller)),
+        ("successful_requests", int(caller and success)),
+        ("failed_requests", int(caller and not success)),
+        ("cache_read_input_tokens", extract_cache_read_tokens(usage)),
+        ("cache_creation_input_tokens", extract_cache_creation_tokens(usage)),
+        ("compression_saved_tokens", extract_compression_saved_tokens(metadata)),
+        ("compression_savings_spend", _number(savings.get("compression"))),
+        ("prompt_caching_savings_spend", _number(savings.get("prompt_caching"))),
+        ("gateway_injected_caching_savings_spend", _number(savings.get("gateway_injected_caching"))),
+        ("autorouter_savings_spend", 0.0 if internal else _number(savings.get("autorouter"))),
+        ("total_response_time_ms", _number(duration) if timed else 0),
+        ("timed_requests", int(timed)),
+    )
+
+
+def _daily_contributions(row: Mapping[str, object], metadata: Mapping[str, object]) -> tuple[Contribution, ...]:
+    date: Final = _date(row)
+    if date is None or "api_key" not in row:
+        return ()
+    if "mcp_namespaced_tool_name" not in row and (
+        "model" not in row or "custom_llm_provider" not in row or "model_group" not in row
+    ):
+        return ()
+    counters: Final = _daily_counters(row, metadata)
+    endpoint: Final = ROUTE_ENDPOINT_MAPPING.get(_text(row.get("call_type")))
+    shared_key: Final = (
+        ("date", date),
+        ("api_key", _text(row.get("api_key"))),
+        ("model", _text(row.get("model"))),
+        ("custom_llm_provider", _text(row.get("custom_llm_provider"))),
+        ("mcp_namespaced_tool_name", _text(row.get("mcp_namespaced_tool_name"))),
+        ("endpoint", _text(endpoint)),
+    )
+    # The legacy daily organization writer uses the charged org argument, not
+    # SpendLogs' normalized empty-string identity for a request with no org.
+    daily_row: Final = {**row, "organization_id": _fields(metadata.get("accounting_entities")).get("organization_id")}
+    return tuple(_daily_rows(daily_row, shared_key, counters))
+
+
+def _daily_rows(
+    row: Mapping[str, object],
+    shared_key: tuple[tuple[str, str], ...],
+    counters: tuple[tuple[str, int | float], ...],
+) -> Iterator[Contribution]:
+    for entity, table in DAILY_SPEND_TABLES.items():
+        for identity in daily_spend_entity_ids(row, entity):
+            yield Contribution(
+                table.name,
+                ((table.entity_id_column, _text(identity)), *shared_key),
+                counters,
+                (
+                    ("model_group", None if row.get("model_group") is None else _text(row.get("model_group"))),
+                    *((("request_id", _text(row.get("request_id"))),) if table.carries_request_id else ()),
+                ),
+            )
+
+
+def _window_contributions(row: Mapping[str, object], metadata: Mapping[str, object]) -> tuple[Contribution, ...]:
+    windows: Final = _ROWS.validate_python(metadata.get("accounting_windows", ()))
+    return tuple(
+        Contribution(
+            "LiteLLM_BudgetWindowSpend",
+            tuple((column, _text(window[column])) for column in ("entity_type", "entity_id", "window_duration")),
+            (("spend", _number(window.get("spend", row.get("spend")))),),
+            (("window_start", _text(window["window_start"])), ("started_at", _text(row.get("startTime")))),
+        )
+        for window in windows
+    )
+
+
+def project(row: Mapping[str, object]) -> tuple[Contribution, ...]:
+    metadata: Final = _fields(row.get("metadata"))
+    return (
+        *_entity_contributions(row, metadata),
+        *_daily_contributions(row, metadata),
+        *_window_contributions(row, metadata),
+        *_model_contributions(row, metadata),
+        *_guardrail_contributions(row, metadata),
+    )
+
+
+def _model_contributions(row: Mapping[str, object], metadata: Mapping[str, object]) -> tuple[Contribution, ...]:
+    date: Final = _date(row)
+    origin: Final = metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)
+    if date is None or (origin and origin != BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN):
+        return ()
+    model: Final = _text(row.get("model")) or "unknown"
+    caller: Final = not origin
+    success: Final = row.get("status") == "success"
+    return (
+        Contribution(
+            "LiteLLM_DailyModelUsage",
+            (
+                ("date", date),
+                ("model_group", _text(row.get("model_group")) or model),
+                ("model", model),
+                ("custom_llm_provider", _text(row.get("custom_llm_provider")) or "unknown"),
+                ("task_type", _model_task(row)),
+            ),
+            (
+                ("spend", _number(row.get("spend"))),
+                ("prompt_tokens", _number(row.get("prompt_tokens"))),
+                ("completion_tokens", _number(row.get("completion_tokens"))),
+                ("request_count", int(caller)),
+                ("successful_requests", int(caller and success)),
+                ("failed_requests", int(caller and not success)),
+            ),
+        ),
+    )
+
+
+def _utc_date(row: Mapping[str, object]) -> str | None:
+    start: Final = row.get("startTime")
+    try:
+        parsed: Final = datetime.fromisoformat(start.replace("Z", "+00:00")) if isinstance(start, str) else start
+    except ValueError:
+        return None
+    if not isinstance(parsed, datetime):
+        return None
+    return (
+        (parsed.replace(tzinfo=timezone.utc) if parsed.tzinfo is None else parsed)
+        .astimezone(timezone.utc)
+        .date()
+        .isoformat()
+    )
+
+
+def _guardrail_contributions(row: Mapping[str, object], metadata: Mapping[str, object]) -> tuple[Contribution, ...]:
+    date: Final = _utc_date(row)
+    info: Final = metadata.get("guardrail_information") or metadata.get("standard_logging_guardrail_information")
+    if not row.get("request_id") or date is None or not isinstance(info, list):
+        return ()
+    return tuple(chain.from_iterable(_guardrail_entry(row, date, entry) for entry in _ROWS.validate_python(info)))
+
+
+def _guardrail_entry(row: Mapping[str, object], date: str, entry: Mapping[str, object]) -> tuple[Contribution, ...]:
+    identity: Final = _text(entry.get("guardrail_id") or entry.get("guardrail_name"))
+    usage: Final = entry.get("guardrail_usage")
+    if not identity or not isinstance(usage, Mapping):
+        return ()
+    costs: Final = _guardrail_unit_costs(entry)
+    return tuple(
+        _guardrail_unit(row, (identity, date, unit), units, None if costs is None else costs.get(unit))
+        for unit, units in _fields(usage).items()
+        if isinstance(units, int) and not isinstance(units, bool) and units > 0
+    )
+
+
+def _guardrail_unit(
+    row: Mapping[str, object], identity: tuple[str, str, str], units: int, cost: float | None
+) -> Contribution:
+    guardrail_id, date, unit = identity
+    return Contribution(
+        "LiteLLM_DailyGuardrailUsageUnits",
+        (
+            ("guardrail_id", guardrail_id),
+            ("date", date),
+            ("team_id", _text(row.get("team_id"))),
+            ("api_key", _text(row.get("api_key"))),
+            ("usage_unit", unit),
+        ),
+        (("units", units), ("cost", cost or 0.0), ("untracked_units", units if cost is None else 0)),
+    )
+
+
+def _model_task(row: Mapping[str, object]) -> str:
+    return next(
+        (
+            task
+            for tag in _tags(row)
+            if tag.startswith(MODEL_INSIGHTS_TASK_TAG_PREFIX)
+            and (task := tag.removeprefix(MODEL_INSIGHTS_TASK_TAG_PREFIX)) in _TASK_TYPES
+        ),
+        MODEL_INSIGHTS_DEFAULT_TASK,
+    )
+
+
+def _guardrail_unit_costs(entry: Mapping[str, object]) -> Mapping[str, float | None] | None:
+    try:
+        cost: Final = GuardrailCostByUnitEntry.model_validate(entry)
+    except ValidationError:
+        return None
+    return None if cost.guardrail_cost_in_spend is False else cost.guardrail_cost_by_unit

litellm/accounting/protocols.py

@@ -1,58 +1,14 @@
-"""Contracts only. Implementations own persistence, pricing, projections and live budgets.
-
-The event row's ``payload`` is a private envelope versioned by the row's ``schema_version``.
-Usage envelopes retain the public event and request-scoped pricing context: quantities with units,
-model/provider, requested and billed tier/region when known, and per-request/deployment overrides.
-These inputs, including the effective cost-map rates, must survive request teardown.
-Cost envelopes retain the public CostEvent, including its required finalized pricing snapshot.
-T5/T6 own envelope serialization; this public API does not introduce a second persisted schema.
-"""
+from __future__ import annotations
 
+from collections.abc import Mapping
 from typing import Protocol
 
-from litellm.accounting.types import (
-    AdmissionDecision,
-    AggregateQuery,
-    AggregateRow,
-    CostEvent,
-    Dimension,
-    InboundUsageEvent,
-    Measurement,
-    Metric,
-    OutboundUsageEvent,
-    Unpriced,
-)
-
-
-class UsageMeter(Protocol):
-    async def record(self, event: InboundUsageEvent | OutboundUsageEvent) -> None:
-        """Acknowledge commit, no-op identical IDs and reject conflicting content.
-
-        Persist the event and its request-scoped pricing context before acknowledging.
-        HTTP acknowledgement is a separate policy. Unknown usage remains None.
-        """
-        ...
-
-
-class CostCompiler(Protocol):
-    async def compile(self, event: OutboundUsageEvent) -> CostEvent | Unpriced:
-        """Reuse the committed charge or finalize one charge and its delivery intent.
-
-        Resolve captured request-scoped context and rates by event identity.
-        Persist the resolved inputs in the required snapshot. Replay
-        reuses the committed result and never consumes free allowance again.
-        Missing pricing is Unpriced, never an inferred zero.
-        """
-        ...
+from litellm.accounting.types import AggregateQuery, AggregateRow
 
 
 class CostAggregator(Protocol):
-    async def apply(self, measurements: tuple[Measurement, ...]) -> None:
-        """Apply one source event's complete contribution set per target/version.
-
-        Each target commits all contributions and its receipt atomically. A receipt
-        for the first metric must not suppress the rest. Usage need not wait for cost.
-        """
+    async def apply(self, rows: tuple[Mapping[str, object], ...]) -> None:
+        """Apply stored request contributions atomically with their delivery receipts."""
         ...
 
     async def query(self, query: AggregateQuery) -> tuple[AggregateRow, ...]:
@@ -63,23 +19,3 @@ class CostAggregator(Protocol):
         combinations. Different groups may overlap, including non-additive tags.
         """
         ...
-
-
-class BudgetEnforcer(Protocol):
-    async def reserve(
-        self, reservation_id: str, dimensions: tuple[Dimension, ...], estimated: tuple[Metric, ...]
-    ) -> AdmissionDecision:
-        """Apply existing admission policies and estimates; duplicate IDs add no hold."""
-        ...
-
-    async def settle(self, reservation_id: str | None, cost_event: CostEvent) -> None:
-        """Settle the committed charge with atomic deduplication and hold mutation.
-
-        Preserve scope/window attribution and retry unfinished settlement after
-        restart. SQL projection receipts do not provide live-counter atomicity.
-        """
-        ...
-
-    async def release(self, reservation_id: str) -> None:
-        """Idempotently release unused exposure at the request boundary, preserving charges."""
-        ...

litellm/accounting/snapshot.py

@@ -0,0 +1,97 @@
+from collections.abc import Callable, Mapping, Sequence
+from datetime import datetime
+from typing import TYPE_CHECKING, Final
+
+from pydantic import TypeAdapter
+
+from litellm.litellm_core_utils.safe_json_dumps import safe_json_structure
+from litellm.proxy._types import SpendLogsPayload
+from litellm.proxy.spend_tracking.compression_savings import extract_compression_saved_tokens
+from litellm.proxy.spend_tracking.savings import compute_savings_spend, marks_gateway_injection
+
+if TYPE_CHECKING:
+    from litellm.router import Router
+
+_FIELDS: Final = TypeAdapter(dict[str, object])
+
+
+def _mapping(value: object) -> Mapping[str, object] | None:
+    return _FIELDS.validate_python(value) if isinstance(value, Mapping) else None
+
+
+_METADATA_FIELDS: Final = frozenset(
+    {
+        "usage_object",
+        "status",
+        "guardrail_information",
+        "standard_logging_guardrail_information",
+        "compression_savings",
+        "internal_call_origin",
+        "billed_retry_attempt",
+    }
+)
+
+
+_ACCOUNTING_FIELDS: Final = frozenset(
+    {
+        "request_id",
+        "call_type",
+        "api_key",
+        "spend",
+        "prompt_tokens",
+        "completion_tokens",
+        "startTime",
+        "model",
+        "model_group",
+        "custom_llm_provider",
+        "user",
+        "team_id",
+        "organization_id",
+        "end_user",
+        "agent_id",
+        "request_tags",
+        "status",
+        "request_duration_ms",
+        "mcp_namespaced_tool_name",
+    }
+)
+
+
+def build_accounting_payload(
+    payload: SpendLogsPayload,
+    *,
+    entities: Mapping[str, str | None],
+    model_access_groups: Sequence[str],
+    windows: Sequence[Mapping[str, object]],
+    router: "Callable[[], Router | None]",
+) -> Mapping[str, object]:
+    metadata: Final = _FIELDS.validate_json(payload["metadata"])
+    savings: Final = compute_savings_spend(
+        model=payload["model"],
+        custom_llm_provider=payload["custom_llm_provider"],
+        compression_saved_tokens=extract_compression_saved_tokens(metadata),
+        gateway_injected_cache=marks_gateway_injection(metadata, payload["model_id"]),
+        routing_decision=_mapping(metadata.get("routing_decision")),
+        model_id=payload["model_id"],
+        llm_router=router,
+        usage_object=_mapping(metadata.get("usage_object")),
+        cost_breakdown=_mapping(metadata.get("cost_breakdown")),
+        recorded_autorouter_savings=metadata.get("autorouter_savings"),
+        recorded_autorouter_savings_estimate=_mapping(metadata.get("autorouter_savings_estimate")),
+        billed_at=payload["endTime"],
+    )
+    accounting_metadata: Final = {
+        **{key: value for key, value in metadata.items() if key in _METADATA_FIELDS},
+        "accounting_entities": entities,
+        "accounting_savings": savings._asdict(),
+        "accounting_model_access_groups": model_access_groups,
+        "accounting_windows": windows,
+    }
+    return {
+        **{
+            key: value.isoformat() if isinstance(value, datetime) else value
+            for key, value in payload.items()
+            if key in _ACCOUNTING_FIELDS
+        },
+        "metadata": _FIELDS.validate_python(safe_json_structure(accounting_metadata, non_finite_as_null=True)),
+    }

litellm/anthropic_beta_headers_manager.py

@@ -198,7 +198,6 @@ def _load_beta_headers_config() -> dict:
         if _BETA_HEADERS_CONFIG is not None:
             return _BETA_HEADERS_CONFIG
 
-        # Get the URL from environment or use default
         from litellm import anthropic_beta_headers_url
 
         _BETA_HEADERS_CONFIG = get_beta_headers_config(url=anthropic_beta_headers_url)

litellm/batches/batch_utils.py

@@ -10,6 +10,7 @@ from pydantic import JsonValue, TypeAdapter
 
 import litellm
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils import batch_utils as core_batch_utils
 from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS
 from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details
 from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo
@@ -38,8 +39,8 @@ class BatchCostUsageResult:
     completion_cost: float = 0.0
 
 
-_COMPLETED_BATCH_STATUSES: Final = frozenset({"completed", "complete"})
 _DECODED_BATCH_LINE: Final = TypeAdapter(object)
+batch_cost_is_final: Final = core_batch_utils.batch_cost_is_final
 
 
 def _uses_native_vertex_output(
@@ -54,26 +55,6 @@ def _uses_native_vertex_output(
     return first_row is not None and is_native_vertex_batch_output_row(first_row)
 
 
-_TERMINAL_BATCH_STATUSES: Final = _COMPLETED_BATCH_STATUSES | frozenset({"failed", "cancelled", "expired"})
-
-
-def batch_cost_is_final(batch: Batch) -> bool:
-    """Whether this retrieve of the batch is the one to account its cost from.
-
-    A batch still in flight has nothing to price, and a "completed" batch can report
-    no output_file_id for a moment before the output populates; pricing either records
-    $0 under the batch's single spend row and pins it there. Final means a completed
-    batch whose output file has arrived or whose counts prove no line succeeded, or
-    any other terminal status (failed, cancelled, expired).
-    """
-    if batch.status not in _TERMINAL_BATCH_STATUSES:
-        return False
-    if batch.status not in _COMPLETED_BATCH_STATUSES or batch.output_file_id is not None:
-        return True
-    request_counts: Final = batch.request_counts
-    return request_counts is not None and request_counts.total > 0 and request_counts.completed == 0
-
-
 async def calculate_batch_cost_and_usage(
     file_content_dictionary: Sequence[dict[str, object]],
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"],

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -1180,11 +1180,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             include_namespace=True,
             replay_openai_history=provider == "openai",
         )
-        # OpenAI's Responses API rejects an empty input. For a system-only
-        # request, carry the system message as a system-role input item instead
-        # of instructions, mirroring how non-string system content is already
-        # handled in convert_chat_completion_messages_to_responses_api.
-        is_system_only_request: Final = not converted_input_items and converted_instructions is not None
+        # Keep the system text in input instead of instructions when the Responses API needs it there.
+        # It rejects an empty input, and json_object requires the word "json" in input, ignoring instructions.
+        typed_optional_params: Final = cast("Mapping[str, object]", optional_params)  # cast-ok: untyped dict
+        system_goes_in_input: Final = converted_instructions is not None and (
+            not converted_input_items or _payload_type(typed_optional_params.get("response_format")) == "json_object"
+        )
         target_input_items: Final = (
             _without_audio_input_parts(converted_input_items)
             if _drops_audio_input(model, litellm_params)
@@ -1196,12 +1197,13 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                     "type": "message",
                     "role": "system",
                     "content": [{"type": "input_text", "text": converted_instructions}],
-                }
+                },
+                *target_input_items,
             ]
-            if is_system_only_request
+            if system_goes_in_input
             else target_input_items
         )
-        instructions: Final = None if is_system_only_request else converted_instructions
+        instructions: Final = None if system_goes_in_input else converted_instructions
 
         # Build responses API request using the reverse transformation logic
         responses_api_request: Final = ResponsesAPIOptionalRequestParams()

litellm/cost_calculator.py

@@ -50,6 +50,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
     get_batch_cost_rates,
     get_billable_input_tokens,
     get_cost_per_unit,
+    get_fedramp_uplift_multiplier,
     get_modality_token_cost_key,
     get_regional_uplift_multiplier,
     get_service_tier_cost_key,
@@ -615,11 +616,17 @@ def cost_per_token(
         )
         if per_second is not None:
             return per_second
+        if custom_llm_provider == "anthropic":
+            return anthropic_cost_per_token(
+                model=model, usage=usage_block, service_tier=service_tier, model_info=requested_model_info_override
+            )
         return generic_cost_per_token(
             model=model,
             usage=usage_block,
             custom_llm_provider=custom_llm_provider or "",
             service_tier=service_tier,
+            data_residency=data_residency,
+            vertex_location=vertex_location,
             model_info=requested_model_info_override,
         )
 
@@ -3311,7 +3318,9 @@ def batch_cost_calculator(
     text_tokens: Final = usage.completion_tokens - image_tokens
     total_completion_cost = text_tokens * text_rate + image_tokens * image_rate
 
-    uplift: Final = get_regional_uplift_multiplier(model_info, data_residency)
+    uplift: Final = get_regional_uplift_multiplier(model_info, data_residency) * get_fedramp_uplift_multiplier(
+        model_info, data_residency
+    )
     if uplift != 1.0:
         total_prompt_cost *= uplift
         total_completion_cost *= uplift
@@ -3320,15 +3329,35 @@ def batch_cost_calculator(
     return total_prompt_cost * vertex_uplift, total_completion_cost * vertex_uplift
 
 
+def _standard_rate(model_info: ModelInfo, key: str, prompt_tokens: int) -> float | None:
+    return get_cost_per_unit(model_info, get_modality_token_cost_key(model_info, key, prompt_tokens), None)
+
+
+def _cached_audio_batch_fallback(
+    model_info: ModelInfo, batch_cache_read_rate: float | None, cache_read_rate: float | None, prompt_tokens: int
+) -> float:
+    if batch_cache_read_rate is not None:
+        return batch_cache_read_rate
+    standard_audio_rate: Final = _standard_rate(model_info, "cache_read_input_audio_token_cost", prompt_tokens)
+    if standard_audio_rate is not None:
+        return standard_audio_rate
+    return 0.0 if cache_read_rate is None else cache_read_rate
+
+
 def _batch_prompt_cost(
     usage: Usage,
     model_info: ModelInfo,
     input_rate: float,
-    cache_read_rate: float | None,
+    batch_cache_read_rate: float | None,
     cache_creation_rate: float | None,
     cache_creation_rates_by_ttl: Mapping[str, float],
 ) -> float:
     details: Final = parse_prompt_tokens_details(usage)
+    cache_read_rate: Final = (
+        _standard_rate(model_info, "cache_read_input_token_cost", usage.prompt_tokens)
+        if batch_cache_read_rate is None
+        else batch_cache_read_rate
+    )
     cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0
     cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cache_read_rate is not None else 0
     bills_cache_writes: Final = cache_creation_rate is not None or bool(cache_creation_rates_by_ttl)
@@ -3349,7 +3378,11 @@ def _batch_prompt_cost(
         + video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate)
         + (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0)
         + cached_audio_tokens
-        * _batch_rate(model_info, "cache_read_input_audio_token_cost_batches", cache_read_rate or 0.0)
+        * _batch_rate(
+            model_info,
+            "cache_read_input_audio_token_cost_batches",
+            _cached_audio_batch_fallback(model_info, batch_cache_read_rate, cache_read_rate, usage.prompt_tokens),
+        )
         + (
             calculate_cache_writing_cost(
                 written_tokens,

litellm/images/main.py

@@ -67,6 +67,10 @@ from litellm.utils import (
 )
 
 _PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints)
+_OPTIONAL_OBJECT_DICT_ADAPTER: Final = TypeAdapter[dict[str, object] | None](dict[str, object] | None)
+_OPTIONAL_STRING_DICT_ADAPTER: Final = TypeAdapter[dict[str, str] | None](dict[str, str] | None)
+_OBJECT_DICT_ADAPTER: Final = TypeAdapter[dict[str, object]](dict[str, object])
+_STRING_DICT_ADAPTER: Final = TypeAdapter[dict[str, str]](dict[str, str])
 _PROVIDERS_RESOLVING_THEIR_OWN_IMAGE_BASE: Final = (
     LlmProviders.DASHSCOPE,
     LlmProviders.QWENCLOUD,
@@ -93,8 +97,8 @@ def _chatgpt_image_params(kwargs: Mapping[str, object]) -> GenericLiteLLMParams:
     from litellm.responses.utils import ResponsesAPIRequestUtils
 
     headers: Final = ResponsesAPIRequestUtils.merge_client_forwarded_headers(
-        TypeAdapter[dict[str, object] | None](dict[str, object] | None).validate_python(kwargs.get("extra_headers")),
-        TypeAdapter[dict[str, str] | None](dict[str, str] | None).validate_python(kwargs.get("headers")),
+        _OPTIONAL_OBJECT_DICT_ADAPTER.validate_python(kwargs.get("extra_headers")),
+        _OPTIONAL_STRING_DICT_ADAPTER.validate_python(kwargs.get("headers")),
     )
     return GenericLiteLLMParams.model_validate({**kwargs, "extra_headers": headers})
 
@@ -167,7 +171,7 @@ async def aimage_generation(*args, **kwargs) -> ImageResponse:
             custom_llm_provider=provider_hints.get("custom_llm_provider"),
             api_base=provider_hints.get("api_base"),
             litellm_params=(
-                _chatgpt_image_params(TypeAdapter(dict[str, object]).validate_python(kwargs))
+                _chatgpt_image_params(_OBJECT_DICT_ADAPTER.validate_python(kwargs))
                 if provider_hints.get("custom_llm_provider") == "chatgpt"
                 or (isinstance(model, str) and model.startswith("chatgpt/"))
                 else None
@@ -299,7 +303,7 @@ def image_generation(
             headers.update(extra_headers)
         model_response: ImageResponse = litellm.utils.ImageResponse()
         chatgpt_params: Final = (
-            _chatgpt_image_params(TypeAdapter(dict[str, object]).validate_python({**kwargs, "api_key": api_key}))
+            _chatgpt_image_params(_OBJECT_DICT_ADAPTER.validate_python({**kwargs, "api_key": api_key}))
             if custom_llm_provider == "chatgpt" or (model is not None and model.startswith("chatgpt/"))
             else None
         )
@@ -582,7 +586,7 @@ def image_generation(
                         _chatgpt_image_headers(
                             model,
                             chatgpt_params.model_copy(update={"api_key": dynamic_api_key or api_key}),
-                            TypeAdapter(dict[str, str]).validate_python(headers),
+                            _STRING_DICT_ADAPTER.validate_python(headers),
                         )
                     )
                     if chatgpt_params is not None

litellm/integrations/otel/model/spans.py

@@ -341,6 +341,7 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset(
         "LiteLLM_AccountingDelivery",
         "LiteLLM_AccountingEvent",
         "LiteLLM_ClaudeCodePluginTable",
+        "LiteLLM_ClaudeCodePluginUpload",
         "LiteLLM_MemoryTable",
         "LiteLLM_AdaptiveRouterState",
         "LiteLLM_AdaptiveRouterSession",

litellm/integrations/prometheus.py

@@ -54,6 +54,7 @@ from litellm.litellm_core_utils.get_llm_provider_logic import is_registered_cust
 from litellm.litellm_core_utils.service_tier_utils import (
     get_service_tier_from_standard_logging_payload,
 )
+from litellm.llms.azure.common_utils import caused_by_entra_rejection
 from litellm.llms.github_copilot.per_user_auth import github_copilot_user_session_from
 from litellm.llms.openai_like.json_loader import JSONProviderRegistry
 from litellm.models.end_user import LiteLLM_EndUserTable
@@ -448,7 +449,7 @@ def _is_deployment_failure(exception: openai.APIError, logged_call: Mapping[str,
     return not (
         isinstance(exception, CallerCredentialAuthenticationError)
         or bool(logged_call.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL))
-        or logged_call.get("api_call_start_time") is None
+        or (logged_call.get("api_call_start_time") is None and not caused_by_entra_rejection(exception))
         or _used_a_caller_credential(logged_call)
     )
 

litellm/litellm_core_utils/audio_utils/transcription_streaming.py

@@ -177,7 +177,7 @@ class _TranscriptionEventCollector:
 
 
 def _billed_usage(response: TranscriptionResponse) -> Usage:
-    usage: Final[object] = getattr(response, "usage", None)
+    usage: Final = response.usage
     token_usage: Final = (
         TranscriptionUsageObjectTransformation.transform_transcription_usage_object(usage)
         if isinstance(usage, TranscriptionUsageTokensObject)

litellm/litellm_core_utils/batch_utils.py

@@ -3,21 +3,32 @@ import json
 from collections.abc import Callable, Generator, Iterable, Mapping, Sequence
 from contextlib import contextmanager
 from contextvars import ContextVar
+from dataclasses import dataclass
 from os import PathLike
-from typing import Final, TypeVar
+from typing import Final, Literal, TypeVar
 
 from openai.types import Batch
 from pydantic import TypeAdapter, ValidationError
 
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.core_helpers import finite_cost
+from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_information_cost
 from litellm.types.llms.openai import FileTypes, OpenAIFilesPurpose
 from litellm.types.utils import CallTypes, LlmProviders
+from litellm.types.videos.main import VideoObject
+from litellm.types.videos.utils import POLLED_VIDEO_STATUS_KWARG, canonical_video_id, is_video_status_call_type
 
 _DECODED_JSONL_ROW: Final = TypeAdapter(object)
 _DEPLOYMENT_FIELD: Final = TypeAdapter(Mapping[str, object])
+_PAYLOAD_FIELDS: Final = TypeAdapter(Mapping[str, object])
 _ENDPOINT_PATHS: Final = TypeAdapter(tuple[str, ...])
 
+_COMPLETED_BATCH_STATUSES: Final = frozenset({"completed", "complete"})
+_TERMINAL_BATCH_STATUSES: Final = _COMPLETED_BATCH_STATUSES | frozenset({"failed", "cancelled", "expired"})
 BATCH_ENDPOINT: Final = "/v1/batch"
+POLLED_BATCH_ID_KWARG: Final = "polled_batch_id"
+POLLED_BATCH_COST_IS_FINAL_KWARG: Final = "polled_batch_cost_is_final"
+BATCH_COST_REQUEST_ID_SUFFIX: Final = "_batch_cost"
 _AZURE_PROVIDERS: Final = frozenset({LlmProviders.AZURE.value, LlmProviders.AZURE_AI.value})
 
 _batch_create_routing: Final[ContextVar[bool]] = ContextVar("batch_create_routing", default=False)
@@ -289,12 +300,151 @@ def is_batch_retrieve_call_type(call_type: object) -> bool:
     return isinstance(call_type, str) and call_type in BATCH_RETRIEVE_CALL_TYPES
 
 
-def batch_charged_once_marker_key(spend_key: str, batch_id: str) -> str:
-    return f"{spend_key}:batch:{batch_id}"
+@dataclass(frozen=True, slots=True)
+class ChargeOnceJob:
+    kind: Literal["batch", "video"]
+    job_id: str
+    repeat_poll_cost: float = 0.0
 
 
-def batch_id_to_charge_once(call_type: object, response_obj: object, response_cost: float) -> str | None:
-    """A finished batch reports its whole cost on every poll, so its id is charged once per counter."""
-    if response_cost <= 0 or not is_batch_retrieve_call_type(call_type):
+def job_charged_once_marker_key(spend_key: str, job: ChargeOnceJob) -> str:
+    return f"{spend_key}:{job.kind}:{job.job_id}"
+
+
+def _guardrail_cost_of_poll(standard_logging_payload: object) -> float:
+    guardrail_information: Final = (
+        _PAYLOAD_FIELDS.validate_python(standard_logging_payload).get("guardrail_information")
+        if isinstance(standard_logging_payload, Mapping)
+        else None
+    )
+    return guardrail_information_cost(guardrail_information)
+
+
+def charged_poll_cost(kwargs: Mapping[str, object]) -> object:
+    """The poll's whole cost as the cost callback charges it, guardrails included, which the logging kwargs' own
+    response_cost leaves out."""
+    standard_logging_payload: Final = kwargs.get("standard_logging_object")
+    logged_cost: Final = (
+        _PAYLOAD_FIELDS.validate_python(standard_logging_payload).get("response_cost")
+        if isinstance(standard_logging_payload, Mapping)
+        else None
+    )
+    return kwargs.get("response_cost") if logged_cost is None else logged_cost
+
+
+def job_cost_of_poll(response_cost: object, standard_logging_payload: object) -> float:
+    """What the polled job itself cost, since a guardrail on the poll charges every poll, running or finished."""
+    return (finite_cost(response_cost) or 0.0) - _guardrail_cost_of_poll(standard_logging_payload)
+
+
+def repeat_poll_cost(call_type: object, response_cost: object, standard_logging_payload: object) -> float:
+    """What a poll still costs once an earlier poll charged its job: a video poll's guardrails, which every video
+    poll has always paid, and nothing for a batch poll, whose repeats have always been free."""
+    if not is_video_status_call_type(call_type):
+        return 0.0
+    return max(0.0, min(finite_cost(response_cost) or 0.0, _guardrail_cost_of_poll(standard_logging_payload)))
+
+
+def batch_cost_is_final(batch: Batch) -> bool:
+    """Whether this retrieve of the batch is the one to account its cost from.
+
+    A batch still in flight has nothing to price, and a "completed" batch can report
+    no output_file_id for a moment before the output populates; pricing either records
+    $0 under the batch's single spend row and pins it there. Final means a completed
+    batch whose output file has arrived or whose counts prove no line succeeded, or
+    any other terminal status (failed, cancelled, expired).
+    """
+    if batch.status not in _TERMINAL_BATCH_STATUSES:
+        return False
+    if batch.status not in _COMPLETED_BATCH_STATUSES or batch.output_file_id is not None:
+        return True
+    request_counts: Final = batch.request_counts
+    return request_counts is not None and request_counts.total > 0 and request_counts.completed == 0
+
+
+@dataclass(frozen=True, slots=True)
+class PolledBatch:
+    id: str
+    cost_is_final: bool
+
+
+def _polled_batch(kwargs: Mapping[str, object], response_obj: object) -> PolledBatch | None:
+    """The retrieved batch, read from the logging kwargs when message redaction replaced the logged response."""
+    if not is_batch_retrieve_call_type(kwargs.get("call_type")):
         return None
-    return response_obj.id if isinstance(response_obj, Batch) else None
+    if isinstance(response_obj, Batch):
+        return PolledBatch(id=response_obj.id, cost_is_final=batch_cost_is_final(response_obj))
+    stamped_id: Final = kwargs.get(POLLED_BATCH_ID_KWARG)
+    stamped_cost_is_final: Final = kwargs.get(POLLED_BATCH_COST_IS_FINAL_KWARG)
+    if not isinstance(stamped_id, str) or not stamped_id or not isinstance(stamped_cost_is_final, bool):
+        return None
+    return PolledBatch(id=stamped_id, cost_is_final=stamped_cost_is_final)
+
+
+def defers_batch_cost(kwargs: Mapping[str, object], response_obj: object) -> bool:
+    """A batch retrieve before the batch's cost is final charges nothing, guardrails included, since the retrieve
+    that settles the batch charges its whole cost once."""
+    polled: Final = _polled_batch(kwargs, response_obj)
+    return polled is not None and not polled.cost_is_final
+
+
+def pays_only_its_guardrail(call_type: object, kwargs: Mapping[str, object], response_cost: object) -> bool:
+    """A video poll whose whole cost is its guardrails', charged like any other request."""
+    if not is_video_status_call_type(call_type):
+        return False
+    return (
+        job_cost_of_poll(response_cost, kwargs.get("standard_logging_object"))
+        <= 0
+        < (finite_cost(response_cost) or 0.0)
+    )
+
+
+def _requested_video_id(kwargs: Mapping[str, object]) -> str | None:
+    litellm_params: Final = kwargs.get("litellm_params")
+    requested: Final = (
+        _PAYLOAD_FIELDS.validate_python(litellm_params).get("video_id") if isinstance(litellm_params, Mapping) else None
+    )
+    return requested if isinstance(requested, str) and requested else None
+
+
+def charged_video_id(
+    call_type: object, kwargs: Mapping[str, object], fallback_video_id: str | None, response_cost: object
+) -> str | None:
+    """The id a finished video's own cost is charged once under.
+
+    That is the id the client polled, which names the deployment that made the video, since a
+    custom provider's own id can repeat across deployments. The status comes from the logging
+    kwargs because message redaction replaces the logged video object.
+    """
+    if not is_video_status_call_type(call_type) or kwargs.get(POLLED_VIDEO_STATUS_KWARG) != "completed":
+        return None
+    if job_cost_of_poll(response_cost, kwargs.get("standard_logging_object")) <= 0:
+        return None
+    video_id: Final = _requested_video_id(kwargs) or fallback_video_id
+    return None if video_id is None else canonical_video_id(video_id)
+
+
+def _charged_batch(kwargs: Mapping[str, object], response_obj: object, response_cost: float) -> ChargeOnceJob | None:
+    polled: Final = _polled_batch(kwargs, response_obj)
+    if polled is None or not polled.cost_is_final or response_cost <= 0:
+        return None
+    return ChargeOnceJob(kind="batch", job_id=polled.id)
+
+
+def job_to_charge_once(
+    kwargs: Mapping[str, object], response_obj: object, response_cost: float
+) -> ChargeOnceJob | None:
+    """A finished batch or video reports its whole cost on every poll, so it is charged once per counter."""
+    call_type: Final = kwargs.get("call_type")
+    if not is_video_status_call_type(call_type):
+        return _charged_batch(kwargs, response_obj, response_cost)
+    video_id: Final = charged_video_id(
+        call_type, kwargs, response_obj.id if isinstance(response_obj, VideoObject) else None, response_cost
+    )
+    if video_id is None:
+        return None
+    return ChargeOnceJob(
+        kind="video",
+        job_id=video_id,
+        repeat_poll_cost=repeat_poll_cost(call_type, response_cost, kwargs.get("standard_logging_object")),
+    )

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -13,6 +13,7 @@ from litellm.litellm_core_utils.fallback_generalizations import (
     match_routing_generalization,
 )
 from litellm.llms.openai_like.json_loader import JSONProviderRegistry
+from litellm.llms.opencode.common_utils import is_opencode_api_base
 from litellm.secret_managers.main import get_secret, get_secret_str
 
 from ..types.router import GenericLiteLLMParams, LiteLLM_Params
@@ -1008,6 +1009,4 @@ def _get_openai_compatible_provider_info(
 
 
 def get_provider_affinity_header(api_base: str | None) -> str | None:
-    from litellm.llms.opencode.transformation import OpenCodeProviderAdapter
-
-    return OpenCodeProviderAdapter().get_provider_affinity_header(api_base)
+    return "x-opencode-session" if is_opencode_api_base(api_base) else None

litellm/litellm_core_utils/get_model_cost_map.py

@@ -320,9 +320,7 @@ def _classify_fetch_content(content: bytes, url: str, etag: str | None) -> _Fetc
         return ModelCostMapReloadUnavailable(reason=f"invalid JSON from {url}: {e}")
     if not isinstance(parsed, dict):
         return ModelCostMapReloadUnavailable(reason=f"expected a JSON object from {url}, got {type(parsed).__name__}")
-    model_cost_map: Final = cast(  # cast-ok: JSON object keys are strings by definition
-        dict[str, object], parsed
-    )
+    model_cost_map: Final = _BUNDLED_CATALOG_ADAPTER.validate_python(parsed)
     return ModelCostMapReloaded(model_cost_map=model_cost_map, revision=git_blob_id(content), etag=etag)
 
 

litellm/litellm_core_utils/get_provider_specific_headers.py

@@ -12,10 +12,18 @@ _PROVIDER_HEADERS: Final[TypeAdapter[ProviderSpecificHeader | Sequence[ProviderS
 
 class ProviderSpecificHeaderUtils:
     @staticmethod
-    def forwarded_authorization(provider_specific_header: object, custom_llm_provider: str | None) -> str | None:
-        headers: Final = ProviderSpecificHeaderUtils.get_provider_specific_headers(
+    def validated_provider_specific_headers(
+        provider_specific_header: object, custom_llm_provider: str | None
+    ) -> Mapping[str, str]:
+        return ProviderSpecificHeaderUtils.get_provider_specific_headers(
             _PROVIDER_HEADERS.validate_python(provider_specific_header), custom_llm_provider
         )
+
+    @staticmethod
+    def forwarded_authorization(provider_specific_header: object, custom_llm_provider: str | None) -> str | None:
+        headers: Final = ProviderSpecificHeaderUtils.validated_provider_specific_headers(
+            provider_specific_header, custom_llm_provider
+        )
         return next((value for key, value in headers.items() if key.lower() == "authorization"), None)
 
     @staticmethod

litellm/litellm_core_utils/litellm_logging.py

@@ -19,7 +19,7 @@ from datetime import datetime as dt_object
 from enum import Enum
 from functools import lru_cache
 from types import MappingProxyType, TracebackType
-from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NamedTuple, TypeVar, cast
+from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, TypeVar, cast
 
 from httpx import Response, Timeout
 from openai import AsyncStream as OpenAIAsyncStream
@@ -38,7 +38,7 @@ from litellm._logging import (
     verbose_logger,
 )
 from litellm._uuid import uuid
-from litellm.batches.batch_utils import batch_cost_is_final, handle_completed_batch
+from litellm.batches.batch_utils import handle_completed_batch
 from litellm.caching.caching import DualCache
 from litellm.caching.caching_handler import LLMCachingHandler
 from litellm.caching.redis_batch import flush_post_call_redis_batches
@@ -79,6 +79,11 @@ from litellm.integrations.mlflow import MlflowLogger
 from litellm.integrations.sqs import SQSLogger
 from litellm.litellm_core_utils.add_retry_fallback_headers import get_hidden_params_dict
 from litellm.litellm_core_utils.agentic_followup_failure import ServedRound
+from litellm.litellm_core_utils.batch_utils import (
+    POLLED_BATCH_COST_IS_FINAL_KWARG,
+    POLLED_BATCH_ID_KWARG,
+    batch_cost_is_final,
+)
 from litellm.litellm_core_utils.classifier_logging import (
     classifier_audit_fields,
     classifier_input_snapshot,
@@ -220,6 +225,7 @@ from litellm.types.utils import (
     batch_chat_usage,
 )
 from litellm.types.videos.main import VideoObject
+from litellm.types.videos.utils import POLLED_VIDEO_STATUS_KWARG
 from litellm.utils import (
     get_base_model_from_metadata,
     print_verbose,
@@ -283,8 +289,6 @@ if TYPE_CHECKING:
     from litellm.llms.base_llm.passthrough.transformation import PassthroughStreamCollector
     from litellm.proxy.hooks.autorouter_baseline_cache import BaselineCacheContext, CapturedBaselineObservation
     from litellm.proxy.hooks.search_free_tier import SearchAllowanceAllocation
-    from litellm.proxy.spend_tracking.usage_capture import CapturedUsage
-    from litellm.proxy.spend_tracking.usage_meter import UsageCaptureError
 try:
     from litellm_enterprise.enterprise_callbacks.callback_controls import (
         EnterpriseCallbackControls,
@@ -662,54 +666,7 @@ class UsageObservation(Enum):
     ESTIMATED = "estimated"
 
 
-@dataclass(frozen=True, slots=True)
-class UsageCaptureObservation:
-    usage_json: str | None
-    provenance: UsageObservation
-    response_cost: float | None = None
-
-    @classmethod
-    def from_usage(
-        cls, usage: Usage | None, provenance: UsageObservation, response_cost: float | None = None
-    ) -> "UsageCaptureObservation | UsageCaptureError":
-        from litellm.proxy.spend_tracking.usage_meter import UsageCaptureError
-
-        try:
-            serialized: Final = (
-                None if usage is None or provenance is UsageObservation.UNKNOWN else usage.model_dump_json()
-            )
-        except Exception as error:  # noqa: BLE001  # observation failures must not interrupt legacy retry handling
-            return UsageCaptureError(str(error))
-        else:
-            return cls(serialized, provenance, response_cost)
-
-
 class Logging(LiteLLMLoggingBaseClass):
-    usage_capture_callback: ClassVar[
-        Callable[
-            ["Logging", object, datetime.datetime, datetime.datetime, bool], "CapturedUsage | UsageCaptureError | None"
-        ]
-        | None
-    ] = None
-
-    def capture_usage_before_logging(
-        self,
-        response: object,
-        start_time: datetime.datetime | None = None,
-        end_time: datetime.datetime | None = None,
-        *,
-        freeze_only: bool = False,
-    ) -> None:
-        capture: Final = type(self).usage_capture_callback
-        if capture is not None and (not self.stream or self._is_assembled_stream_success(response)):
-            capture(
-                self,
-                response,
-                start_time or self.start_time,
-                end_time or datetime.datetime.now(datetime.timezone.utc),
-                freeze_only,
-            )
-
     global \
         supabaseClient, \
         promptLayerLogger, \
@@ -949,7 +906,6 @@ class Logging(LiteLLMLoggingBaseClass):
         self._stash_owed_on_success: bool = False
         self._stash_billed_by_continuation: bool = False
         self._failed_attempts_owed_on_success: tuple[ServedRound, ...] = ()
-        self._failed_attempt_ids_owed_on_success: tuple[str, ...] = ()
         self.failed_attempts_billed_on_success: tuple[ServedRound, ...] = ()
         self.shadow_eval_request_snapshot: GuardrailRequestSnapshot | None = None
 
@@ -2582,11 +2538,6 @@ class Logging(LiteLLMLoggingBaseClass):
             )
             return
 
-        self.capture_usage_before_logging(
-            TypeAdapter(object).validate_python(result),
-            TypeAdapter[datetime.datetime | None](datetime.datetime | None).validate_python(start_time),
-            TypeAdapter[datetime.datetime | None](datetime.datetime | None).validate_python(end_time),
-        )
         await self.async_success_handler(
             result,
             start_time=start_time,
@@ -2698,20 +2649,12 @@ class Logging(LiteLLMLoggingBaseClass):
         self.finalized_cost = finalized
         self.model_call_details["response_cost"] = finalized.amount
 
-    def record_usage_capture(self, captured: "CapturedUsage") -> None:
-        self.model_call_details["_accounting_capture"] = captured
-
-    def record_usage_pricing(self, pricing: object) -> None:
-        self.model_call_details["_accounting_pricing_snapshot"] = pricing
-
     def record_partial_usage_for_failure(
         self, usage: Usage, response_cost: float, *, observation: UsageObservation = UsageObservation.KNOWN
     ) -> None:
         """Stash what an interrupted stream already consumed so the failure log bills it instead of zero."""
         self.model_call_details["combined_usage_object"] = usage
         self.model_call_details["response_cost"] = response_cost
-        if type(self).usage_capture_callback is not None:
-            self.model_call_details["_accounting_usage_observed"] = observation
 
     def record_failure_after_earlier_rounds(
         self, earlier: BilledRoundsSpend, failed_usage: Usage, failed_cost: float
@@ -2795,12 +2738,6 @@ class Logging(LiteLLMLoggingBaseClass):
         owed_on_success: Final = self._stash_owed_on_success
         self._stash_billed_by_continuation = False
         self._stash_owed_on_success = False
-        captured_attempt_ids: Final = (
-            self.capture_failed_attempt_usage(usage)
-            if isinstance(usage, Usage) and type(self).usage_capture_callback is not None
-            else ()
-        )
-        self.model_call_details.pop("_accounting_capture", None)
         self.model_call_details.pop("_accounting_pricing_snapshot", None)
         self.model_call_details.pop("_accounting_usage_observed", None)
         self.model_call_details.pop("combined_usage_object", None)
@@ -2816,60 +2753,10 @@ class Logging(LiteLLMLoggingBaseClass):
         if not owed_on_success or billed_by_continuation:
             return
         self._failed_attempts_owed_on_success = (*self._failed_attempts_owed_on_success, attempt)
-        self._failed_attempt_ids_owed_on_success = (*self._failed_attempt_ids_owed_on_success, *captured_attempt_ids)
 
     def mark_stash_billed_by_continuation(self) -> None:
         self._stash_billed_by_continuation = True
 
-    def capture_failed_attempt_usage(self, usage: Usage) -> tuple[str, ...]:
-        from litellm.constants import BILLED_RETRY_ATTEMPT_METADATA_KEY
-        from litellm.proxy.spend_tracking.usage_capture import CapturedUsage
-
-        capture: Final = type(self).usage_capture_callback
-        if capture is None:
-            return ()
-        attempt: Final = self.attempt_scoped_copy()
-        attempt.prepared_cost = None
-        attempt.finalized_cost = None
-        identity: Final = (
-            f"{self.litellm_call_id}-attempt-{len(self.model_call_details.get('failed_attempts_usage', ()))}"
-        )
-        params: Final = _LITELLM_PARAMS.validate_python(self.model_call_details.get("litellm_params") or {})
-        metadata: Final = StandardLoggingPayloadSetup.merge_litellm_metadata(params)
-        observed: Final[object] = self.model_call_details.get("_accounting_usage_observed")
-        cost: Final[object] = self.model_call_details.get("response_cost")
-        attempt.model_call_details = {
-            **attempt.model_call_details,
-            "litellm_call_id": identity,
-            "parent_request_id": self.litellm_call_id,
-            "_accounting_capture": None,
-            "_accounting_prior_captures": (),
-            "_accounting_usage_snapshot": UsageCaptureObservation.from_usage(
-                usage,
-                observed if isinstance(observed, UsageObservation) else UsageObservation.KNOWN,
-                float(cost) if isinstance(cost, (int, float)) else None,
-            ),
-            "litellm_params": {
-                **params,
-                "metadata": {**metadata, BILLED_RETRY_ATTEMPT_METADATA_KEY: True, "status": "failure"},
-                "litellm_metadata": {**metadata, BILLED_RETRY_ATTEMPT_METADATA_KEY: True, "status": "failure"},
-            },
-        }
-        captured: Final = capture(
-            attempt,
-            {"id": identity, "usage": usage.model_dump(), "status": "failed"},
-            self.start_time,
-            datetime.datetime.now(datetime.timezone.utc),
-            False,
-        )
-        if not isinstance(captured, CapturedUsage):
-            return ()
-        self.model_call_details["_accounting_prior_captures"] = (
-            *self.model_call_details.get("_accounting_prior_captures", ()),
-            captured,
-        )
-        return tuple(write.event.event_id for write in captured.writes)
-
     def record_continued_partial(self, usage: Usage, response_cost: float, response_id: str) -> None:
         """Bill what a cut stream already consumed on this continuation's success log, under the id the caller saw."""
         self.model_call_details["continued_partial_usage"] = usage
@@ -2909,8 +2796,6 @@ class Logging(LiteLLMLoggingBaseClass):
         self.model_call_details.pop("failed_attempts_usage", None)
         self.failed_attempts_billed_on_success = failed_attempts
         self.model_call_details["failed_attempts_billed_on_success"] = failed_attempts
-        if self._failed_attempt_ids_owed_on_success:
-            self.model_call_details["_accounting_included_attempt_ids"] = self._failed_attempt_ids_owed_on_success
         self.model_call_details["combined_usage_object"] = BaseTokenUsageProcessor.combine_usage_objects(
             usage_objects=[own_usage, *(usage for usage, _ in failed_attempts)]
         )
@@ -3073,45 +2958,6 @@ class Logging(LiteLLMLoggingBaseClass):
             return self._handle_a2a_response_logging(result=result)
         return result
 
-    def usage_capture_response(self, result: object) -> object:
-        if self.stream and not self._is_assembled_stream_success(result):
-            return None
-        copied: Final = result.model_copy(deep=True) if isinstance(result, BaseModel) else result
-        normalized: Final = self.normalize_logging_result(self.normalize_endpoint_logging_result(copied))
-        if self.model_call_details.get("call_type") == CallTypes.arealtime.value and isinstance(result, list):
-            frames: Final = TypeAdapter(tuple[dict[str, object], ...]).validate_python(result)
-            measured: Final = any(
-                isinstance(_LITELLM_PARAMS.validate_python(frame.get("response") or {}).get("usage"), Mapping)
-                for frame in frames
-                if frame.get("type") == "response.done"
-            )
-            if not measured:
-                metered: Final = self.model_call_details.get(REALTIME_METERED_USAGE_KEY)
-                return {
-                    "id": self.litellm_call_id,
-                    "usage": {"metered_usage": TypeAdapter(object).dump_python(metered, mode="json")}
-                    if metered is not None
-                    else None,
-                }
-        assembled: Final = self._get_assembled_streaming_response(
-            normalized, self.start_time, self.start_time, True, []
-        )
-        return normalized if assembled is None else assembled
-
-    def usage_capture_pricing(self, result: object) -> Mapping[str, JsonValue]:
-        from litellm.utils import count_characters
-
-        return TypeAdapter(dict[str, JsonValue]).validate_python(
-            {
-                "custom_pricing": self.custom_pricing_for(result),
-                "custom_cost_per_token": self.custom_cost_per_token,
-                "custom_cost_per_second": self.custom_cost_per_second,
-                "prompt_characters": count_characters(self._prompt_for_cost_calculation())
-                if self.model_call_details.get("call_type") in (CallTypes.speech.value, CallTypes.aspeech.value)
-                else None,
-            }
-        )
-
     def normalize_logging_result(self, result: object) -> object:
         """
         Some endpoints return a different type of result than what is expected by the logging system.
@@ -3389,21 +3235,12 @@ class Logging(LiteLLMLoggingBaseClass):
         set_hidden_params(served, {**(get_hidden_params(translated) or {}), **(get_hidden_params(response) or {})})
         return served
 
-    def _with_continued_response_id(
+    def _with_response_id(
         self,
         response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse,
+        response_id: str | None,
     ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse:
-        response_id: Final = self.model_call_details.get("continued_response_id")
-        if not isinstance(response_id, str) or getattr(response, "id", None) == response_id:
-            return response
-        return response.model_copy(update={"id": response_id})
-
-    def _with_client_facing_response_id(
-        self,
-        response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse,
-    ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse:
-        response_id: Final = self.client_facing_response_id
-        if response_id is None or getattr(response, "id", None) == response_id:
+        if response_id is None or response.id == response_id:
             return response
         return response.model_copy(update={"id": response_id})
 
@@ -3411,8 +3248,13 @@ class Logging(LiteLLMLoggingBaseClass):
         self,
         response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse,
     ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse:
-        return self._with_client_facing_response_id(
-            self._with_continued_response_id(self._with_client_facing_stream_model(response))
+        continued_response_id: Final = self.model_call_details.get("continued_response_id")
+        return self._with_response_id(
+            self._with_response_id(
+                self._with_client_facing_stream_model(response),
+                continued_response_id if isinstance(continued_response_id, str) else None,
+            ),
+            self.client_facing_response_id,
         )
 
     def _success_handler_helper_fn(
@@ -3472,6 +3314,11 @@ class Logging(LiteLLMLoggingBaseClass):
             result = self.normalize_endpoint_logging_result(result)
             if self.stream is not True:
                 result = self._as_served_response(result)  # rebind-ok: bridged chat calls log the served response
+            if isinstance(result, VideoObject):
+                self.model_call_details[POLLED_VIDEO_STATUS_KWARG] = result.status
+            if isinstance(result, LiteLLMBatch):
+                self.model_call_details[POLLED_BATCH_ID_KWARG] = result.id
+                self.model_call_details[POLLED_BATCH_COST_IS_FINAL_KWARG] = batch_cost_is_final(result)
 
             logging_result: Final = self.normalize_logging_result(result=result)
             if cache_hit is None and _served_from_cache_at_no_charge(logging_result):
@@ -3830,7 +3677,6 @@ class Logging(LiteLLMLoggingBaseClass):
                     self._as_served_chat_response(complete_streaming_response)
                 )
                 self.model_call_details["complete_streaming_response"] = logged_streaming_response
-                self.capture_usage_before_logging(logged_streaming_response, start_time, end_time)
                 ## STANDARDIZED LOGGING PAYLOAD
                 if not for_async_request:
                     self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
@@ -4356,8 +4202,6 @@ class Logging(LiteLLMLoggingBaseClass):
                     cost_for_built_in_tools_cost_usd_dollar=0.0,
                 )
 
-        self.capture_usage_before_logging(result, start_time, end_time)
-
         self.truncated_messages_for_logging = await truncate_base64_in_messages_async(
             StandardLoggingPayloadSetup.append_system_prompt_messages(
                 kwargs=self.model_call_details, messages=self.model_call_details.get("messages")
@@ -6499,7 +6343,7 @@ def _init_custom_logger_compatible_class(
     logging_integration: str,
     internal_usage_cache: DualCache | None,
     llm_router: object,
-    custom_logger_init_args: dict[str, object] | None = {},
+    custom_logger_init_args: dict[str, object] | None = None,
     raise_import_errors: bool = False,
 ) -> CustomLogger | None:
     """
@@ -7299,6 +7143,7 @@ class StandardLoggingPayloadSetup:
         init_response_obj: object,
         api_base: str | None = None,
         router_model_id: str | None = None,
+        custom_model_info: ModelInfo | None = None,
     ) -> StandardLoggingModelInformation:
         model_cost_name: Final = select_model_name_for_cost_calc(
             model=base_model if custom_pricing else None,
@@ -7317,6 +7162,8 @@ class StandardLoggingPayloadSetup:
             if router_model_id is not None
             else model_cost_name
         )
+        if model_cost_name is not None and custom_model_info is not None:
+            return StandardLoggingModelInformation(model_map_key=model_cost_name, model_map_value=custom_model_info)
         if model_cost_name is None:
             model_cost_information = StandardLoggingModelInformation(model_map_key="", model_map_value=None)
         else:
@@ -8067,6 +7914,7 @@ def get_standard_logging_object_payload(
             init_response_obj=init_response_obj,
             api_base=litellm_params.get("api_base"),
             router_model_id=logging_obj.get_router_model_id(),
+            custom_model_info=logging_obj.custom_model_info if custom_pricing else None,
         )
 
         error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload(

litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py

@@ -224,7 +224,20 @@ class StandardBuiltInToolCostTracking:
             web_search_options=standard_built_in_tools_params.get("web_search_options", None),
             model_info=model_info,
         )
-        return per_call_cost * StandardBuiltInToolCostTracking._count_web_search_calls(response_object)
+        return per_call_cost * StandardBuiltInToolCostTracking._billable_web_search_calls(
+            resolved_usage, response_object
+        )
+
+    @staticmethod
+    def _billable_web_search_calls(usage: Usage | None, response_object: object) -> int:
+        """
+        Providers without their own calculator but whose usage carries server_tool_use.web_search_requests
+        (OpenAI-compatible providers that report a per-tool count) are billed that many calls on the per-call path.
+        """
+        reported: Final = None if usage is None else get_web_search_requests_from_usage(usage)
+        if isinstance(reported, int) and not isinstance(reported, bool) and reported > 0:
+            return reported
+        return StandardBuiltInToolCostTracking._count_web_search_calls(response_object)
 
     @staticmethod
     def _count_web_search_calls(response_object: object) -> int:

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -1441,6 +1441,29 @@ def get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str |
 _get_regional_uplift_multiplier = get_regional_uplift_multiplier
 
 
+def get_fedramp_uplift_multiplier(model_info: ModelInfo, data_residency: str | None) -> float:
+    """
+    Resolve the per-model uplift multiplier for the OpenAI FedRAMP endpoint.
+
+    OpenAI applies a flat percentage uplift (e.g. +10%) on all token costs for
+    requests served from gov.api.openai.com. The multiplier is stored on the
+    model entry as ``fedramp_uplift_multiplier`` (e.g. 1.10).
+
+    Returns 1.0 (no uplift) unless ``data_residency`` is ``"fedramp"`` and the
+    model has a multiplier configured.
+    """
+    if data_residency is None or data_residency.lower() != DataResidency.FEDRAMP.value:
+        return 1.0
+    multiplier: Final = model_info.get("fedramp_uplift_multiplier")
+    if multiplier is None:
+        return 1.0
+    try:
+        return float(multiplier)
+    except (TypeError, ValueError):
+        verbose_logger.exception("Invalid fedramp_uplift_multiplier for model; defaulting to 1.0")
+        return 1.0
+
+
 def get_vertex_regional_endpoint_uplift(model_info: ModelInfo, vertex_location: str | None) -> float:
     """
     Resolve the per-model uplift multiplier for Vertex AI non-global (regional and
@@ -1550,8 +1573,8 @@ def generic_cost_per_token(
     Input:
         - model: str, the model name without provider prefix
         - usage: LiteLLM Usage block, containing anthropic caching information
-        - data_residency: optional OpenAI data-residency region (e.g. "eu", "us"),
-          used to apply the per-model regional-processing uplift multiplier.
+        - data_residency: optional OpenAI data-residency region (e.g. "eu", "us") or
+          "fedramp", used to apply the per-model regional-processing or FedRAMP uplift multiplier.
         - vertex_location: optional Vertex AI location the request was served from
           (e.g. "us-east5", "global"), used to apply the per-model
           regional-endpoint uplift multiplier when non-global.
@@ -1728,10 +1751,9 @@ def generic_cost_per_token(
         )
         completion_cost += float(video_tokens) * _output_cost_per_video_token
 
-    ## REGIONAL DATA-RESIDENCY UPLIFT
-    # Applied as a flat multiplier across all token costs for the request
-    # when the upstream is a regionalized OpenAI host (eu./us.api.openai.com).
-    uplift: Final = get_regional_uplift_multiplier(resolved_model_info, data_residency)
+    uplift: Final = get_regional_uplift_multiplier(resolved_model_info, data_residency) * get_fedramp_uplift_multiplier(
+        resolved_model_info, data_residency
+    )
     if uplift != 1.0:
         prompt_cost *= uplift
         completion_cost *= uplift
@@ -1874,6 +1896,7 @@ def _cost_map_billed_rates(
     )
     multiplier: Final = (
         get_regional_uplift_multiplier(model_info, data_residency)
+        * get_fedramp_uplift_multiplier(model_info, data_residency)
         * get_vertex_regional_endpoint_uplift(model_info, vertex_location)
         * get_provider_specific_geo_multiplier(model_info=model_info, usage=usage)
         * get_provider_specific_speed_multiplier(model_info=model_info, usage=usage)
@@ -2059,8 +2082,10 @@ def calculate_prompt_caching_savings(
             {ttl: rate - prompt_base_cost for ttl, rate in base_costs.cache_creation_rates_by_ttl.items()}
         ),
     )
-    uplift: Final = get_regional_uplift_multiplier(model_info, data_residency) * get_vertex_regional_endpoint_uplift(
-        model_info, vertex_location
+    uplift: Final = (
+        get_regional_uplift_multiplier(model_info, data_residency)
+        * get_fedramp_uplift_multiplier(model_info, data_residency)
+        * get_vertex_regional_endpoint_uplift(model_info, vertex_location)
     )
     return (read_discount - write_premium) * uplift
 

litellm/litellm_core_utils/realtime_streaming.py

@@ -836,8 +836,6 @@ class RealTimeStreaming:
             if self.session_tools or self.tool_calls:
                 self.logging_obj.model_call_details["realtime_tools"] = self.session_tools
                 self.logging_obj.model_call_details["realtime_tool_calls"] = self.tool_calls
-            if LiteLLMLogging.usage_capture_callback is not None:
-                self.logging_obj.capture_usage_before_logging(self.messages)
             # Route through the bounded logging worker (per-coroutine timeout +
             # concurrency cap) instead of a bare create_task, so a slow callback
             # can't leave suspended tasks pinning each call's response in memory.

litellm/litellm_core_utils/redact_messages.py

@@ -272,15 +272,31 @@ def _redacted_provider_specific_fields(fields: Mapping[str, object]) -> Mapping[
     return {
         **fields,
         **({"reasoning_content": REDACTED_BY_LITELLM} if fields.get("reasoning_content") is not None else {}),
+        **({"reasoning": REDACTED_BY_LITELLM} if fields.get("reasoning") is not None else {}),
+        **({"reasoning_text": REDACTED_BY_LITELLM} if fields.get("reasoning_text") is not None else {}),
         **{
             key: None
             for key in (
                 "thinking_blocks",
                 "reasoningContentBlocks",
+                "reasoningContent",
+                "annotations",
                 "citations",
+                "citation",
+                "citationsContent",
                 "compaction_blocks",
                 "compaction_start",
                 "compaction_delta",
+                "reasoning_details",
+                "web_search_calls",
+                "web_search_results",
+                "tool_results",
+                "code_interpreter_results",
+                "search_results",
+                "rerank_results",
+                "executed_tools",
+                "custom_outputs",
+                "tool_search_items",
                 "token_ids",
             )
             if key in fields

litellm/litellm_core_utils/streaming_handler.py

@@ -2826,8 +2826,6 @@ class CustomStreamWrapper:
         return None if is_model_response_stream_empty(model_response=usage_free_chunk) else usage_free_chunk
 
     def _accumulate_streamed_delta(self, delta: object) -> None:
-        # Shared by the sync, async, and non-aiohttp iteration sites so answer
-        # text and the disqualifying-content latch stay in step across all three.
         get: Final = getattr(delta, "get", None)
         content: Final = get("content") if callable(get) else None
         self.response_uptil_now += content if isinstance(content, str) else ""

litellm/litellm_core_utils/tool_fields.py

@@ -10,7 +10,7 @@ from typing_extensions import ReadOnly, TypedDict
 import litellm
 from litellm._logging import verbose_logger
 from litellm.exceptions import UnsupportedParamsError
-from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params
+from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, normalize_drop_params
 
 ROUTES_WITH_TOOLS: Final = frozenset(
     {
@@ -58,6 +58,19 @@ _TOOL_ENTRY: Final = TypeAdapter(_ToolEntry)
 _WEB_SEARCH_OPTIONS: Final = TypeAdapter(dict[str, object] | None, config=ConfigDict(strict=True))
 
 
+def declared_function_name(tool: object) -> str | None:
+    fields: Final = as_str_mapping(tool)
+    if fields is None:
+        return None
+    function: Final = as_str_mapping(fields.get("function"))
+    name: Final = (fields if function is None else function).get("name")
+    return name if isinstance(name, str) and name else None
+
+
+def qualified_member_name(namespace: str, name: str) -> str:
+    return f"{namespace}__{name}"
+
+
 @dataclass(frozen=True, slots=True)
 class DropPolicy:
     drop_params: bool

litellm/litellm_core_utils/url_utils.py

@@ -102,6 +102,14 @@ class SSRFError(ValueError):
     """Raised when a URL targets a blocked network."""
 
 
+def is_https_url_with_host(url: str) -> bool:
+    try:
+        parsed_url: Final = urlsplit(url)
+        return parsed_url.scheme == "https" and bool(parsed_url.hostname)
+    except ValueError:
+        return False
+
+
 def encode_url_path_segment(value: object, *, field_name: str = "path parameter") -> str:
     """Percent-encode one user-controlled URL path segment.
 

litellm/llms/anthropic/chat/guardrail_translation/handler.py

@@ -781,12 +781,17 @@ class AnthropicMessagesHandler(BaseTranslation):
             for event in AnthropicMessagesHandler._parse_sse_data_line(line)
         )
 
-    def _translate_to_openai(self, data: dict) -> ChatCompletionRequest:
+    def _translate_to_openai(
+        self,
+        data: dict,  # mutable-ok: the request body the adapter reads
+        *,
+        keep_cache_control: bool = False,
+    ) -> ChatCompletionRequest:
         """Translate Anthropic request to OpenAI chat completion format."""
         (
             chat_completion_compatible_request,
             _tool_name_mapping,
-        ) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
+        ) = LiteLLMAnthropicMessagesAdapter(keep_cache_control=keep_cache_control).translate_anthropic_to_openai(
             anthropic_message_request=cast(AnthropicMessagesRequest, scannable_tool_result_request(data)),
             preserve_midturn_system=True,
         )
@@ -813,9 +818,13 @@ class AnthropicMessagesHandler(BaseTranslation):
         data: dict,
         guardrail_to_apply: "CustomGuardrail",
         litellm_logging_obj: "LiteLLMLoggingObj | None" = None,
+        *,
+        request_data: dict[str, object] | None = None,  # mutable-ok: the request dict guardrails receive
+        keep_cache_control: bool = False,
     ) -> Mapping[str, object]:
         """
-        Process input messages by applying guardrails to text content.
+        Process input messages by applying guardrails to text content. ``request_data`` is what the guardrail sees as
+        the request when ``data`` is only its body, e.g. a provider pass-through route that nests the body
         """
         messages: Final = data.get("messages")
         if messages is None:
@@ -827,8 +836,17 @@ class AnthropicMessagesHandler(BaseTranslation):
 
         # The top-level prompt is translated on its own below so it can be hoisted in front of
         # any mid-turn system entries and scanned first, aligned with that structured position.
-        translation_source: Final = {key: value for key, value in data.items() if key != "system"}
-        chat_completion_compatible_request: Final = self._translate_to_openai(translation_source)
+        body_model: Final = data.get("model")
+        request_model: Final = (request_data or {}).get("model")
+        model: Final[str] = (
+            body_model
+            if isinstance(body_model, str) and body_model
+            else (request_model if isinstance(request_model, str) else "")
+        )
+        translation_source: Final = {**{key: value for key, value in data.items() if key != "system"}, "model": model}
+        chat_completion_compatible_request: Final = self._translate_to_openai(
+            translation_source, keep_cache_control=keep_cache_control
+        )
 
         full_structured_messages: Final = cast(
             list[AllMessageValues],
@@ -837,7 +855,9 @@ class AnthropicMessagesHandler(BaseTranslation):
         has_midturn_system_message: Final = any(
             str(message.get("role") or "").lower() == "system" for message in full_structured_messages
         )
-        hoisted_system_message: Final = None if skip_system else self._hoisted_top_level_system_message(data)
+        hoisted_system_message: Final = (
+            None if skip_system else self._hoisted_top_level_system_message(data, model, keep_cache_control)
+        )
         if hoisted_system_message is not None:
             full_structured_messages.insert(0, hoisted_system_message)
         # skip_system already excluded the trusted top-level prompt (it is simply not hoisted);
@@ -899,7 +919,6 @@ class AnthropicMessagesHandler(BaseTranslation):
             if structured_messages:
                 inputs["structured_messages"] = structured_messages
             # Include model information if available
-            model: Final = data.get("model")
             if model:
                 inputs["model"] = model
             original_tool_names: Final = _original_tool_names(
@@ -907,7 +926,7 @@ class AnthropicMessagesHandler(BaseTranslation):
             )
             guardrailed_inputs: Final = await guardrail_to_apply.apply_guardrail(
                 inputs=inputs,
-                request_data=data,
+                request_data=data if request_data is None else request_data,
                 input_type="request",
                 logging_obj=litellm_logging_obj,
             )
@@ -955,6 +974,7 @@ class AnthropicMessagesHandler(BaseTranslation):
                     hoisted_system_message=hoisted_system_message,
                     preserve_system_messages=has_midturn_system_message,
                     original_tool_names=original_tool_names.used,
+                    model=model,
                 )
             else:
                 if guardrailed_texts and len(guardrailed_texts) != len(scanned):
@@ -977,17 +997,20 @@ class AnthropicMessagesHandler(BaseTranslation):
 
         return data
 
-    def _hoisted_top_level_system_message(self, data: Mapping[str, object]) -> AllMessageValues | None:
+    def _hoisted_top_level_system_message(
+        self, data: Mapping[str, object], model: str, keep_cache_control: bool
+    ) -> AllMessageValues | None:
         """Return the system message produced by translating the top-level prompt."""
         system: Final = data.get("system")
         if not system:
             return None
         probe: Final = self._translate_to_openai(
             {
-                "model": data.get("model") or "",
+                "model": model,
                 "messages": [],
                 "system": system,
-            }
+            },
+            keep_cache_control=keep_cache_control,
         )
         hoisted: Final = probe.get("messages") or []
         return hoisted[0] if hoisted else None
@@ -1121,6 +1144,7 @@ class AnthropicMessagesHandler(BaseTranslation):
         preserve_system_messages: bool = False,
         *,
         original_tool_names: Mapping[str, str],
+        model: str,
     ) -> None:
         """Write a guardrail's structured-message rewrite back without losing corrections."""
         from litellm.litellm_core_utils.prompt_templates.factory import (
@@ -1129,7 +1153,6 @@ class AnthropicMessagesHandler(BaseTranslation):
         )
 
         _is_system: Final = AnthropicMessagesHandler._is_system
-        model: Final = str(data.get("model") or "")
         converted: Final[list] = []  # mutable-ok: API message payload
 
         def _convert_run(run: list) -> None:  # mutable-ok: API message payload

litellm/llms/anthropic/messages/adapters/transformation.py

@@ -380,8 +380,8 @@ _CLAUDE_OVER_REASONING_EFFORT_PROVIDERS: Final = frozenset({LlmProviders.KYMA.va
 
 
 class LiteLLMAnthropicMessagesAdapter:
-    def __init__(self):
-        pass
+    def __init__(self, *, keep_cache_control: bool = False) -> None:
+        self.keep_cache_control: Final = keep_cache_control
 
     ### FOR [BETA] `/v1/messages` endpoint support
 
@@ -437,7 +437,7 @@ class LiteLLMAnthropicMessagesAdapter:
         cache_control: Final = (
             source.get("cache_control") if isinstance(source, dict) else getattr(source, "cache_control", None)
         )
-        if cache_control and model and self.target_consumes_cache_control(model):
+        if cache_control and (self.keep_cache_control or (model and self.target_consumes_cache_control(model))):
             # TypedDict objects support dict operations at runtime
             # Use type ignore consistent with codebase pattern (see anthropic/chat/transformation.py:432)
             if isinstance(target, dict):

litellm/llms/anthropic/messages/caller_model_checks.py

@@ -243,8 +243,7 @@ async def caller_within_model_budget(
         return "allowed"
 
     model_max_budget: Final = getattr(user_api_key_auth, "model_max_budget", None)
-    token: Final = getattr(user_api_key_auth, "token", None)
-    if isinstance(model_max_budget, dict) and model_max_budget and token is not None:
+    if isinstance(model_max_budget, dict) and model_max_budget and user_api_key_auth.token is not None:
         try:
             await model_max_budget_limiter.is_key_within_model_budget(
                 user_api_key_dict=user_api_key_auth,
@@ -300,10 +299,8 @@ async def caller_within_model_budget(
             )
             return "denied_for_now"
 
-    end_user_model_max_budget: Final[dict[str, object] | None] = getattr(
-        user_api_key_auth, "end_user_model_max_budget", None
-    )
-    end_user_id: Final[str | None] = getattr(user_api_key_auth, "end_user_id", None)
+    end_user_model_max_budget: Final[dict[str, object] | None] = user_api_key_auth.end_user_model_max_budget
+    end_user_id: Final[str | None] = user_api_key_auth.end_user_id
     if isinstance(end_user_model_max_budget, dict) and end_user_model_max_budget and end_user_id is not None:
         try:
             await model_max_budget_limiter.is_end_user_within_model_budget(

litellm/llms/aws_polly/text_to_speech/transformation.py

@@ -178,7 +178,6 @@ class AWSPollyTextToSpeechConfig(BaseTextToSpeechConfig, BaseAWSLLM):
         else:
             mapped_params["output_format"] = self.DEFAULT_OUTPUT_FORMAT
 
-        # model arrives without the "aws_polly/" prefix, e.g. "standard"
         engine: Final = self._extract_engine_from_model(model)
         mapped_params["engine"] = engine
 

litellm/llms/azure/common_utils.py

@@ -1,15 +1,16 @@
 import asyncio
 import hashlib
+import itertools
 import json
 import os
 import re
-from collections.abc import Awaitable, Callable, Mapping
+from collections.abc import Awaitable, Callable, Iterator, Mapping
 from contextlib import closing
 from types import MappingProxyType
-from typing import Final, Literal, NamedTuple, cast
+from typing import Final, Literal, NamedTuple, NoReturn, cast
 
 import httpx
-from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI
+from openai import APIError, AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI
 from pydantic import ConfigDict, TypeAdapter, with_config
 from typing_extensions import ReadOnly, TypedDict
 
@@ -20,7 +21,7 @@ from litellm.constants import DEFAULT_MAX_RETRIES
 from litellm.litellm_core_utils.single_flight import SingleFlight, single_flight_lru_cache
 from litellm.litellm_core_utils.url_utils import ensure_api_route
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
-from litellm.llms.custom_httpx.http_handler import HTTPHandler
+from litellm.llms.custom_httpx.http_handler import HTTPHandler, MaskedHTTPStatusError
 from litellm.llms.openai.common_utils import BaseOpenAILLM, OpenAIAsyncHTTPClient, OpenAIHTTPClient
 from litellm.secret_managers.get_azure_ad_token_provider import (
     get_azure_ad_token_provider,
@@ -52,6 +53,13 @@ class _AsyncAzureClientParams(TypedDict, total=False, extra_items=object):
 
 
 _ASYNC_AZURE_CLIENT_PARAMS_ADAPTER: Final = TypeAdapter(_AsyncAzureClientParams)
+_AZURE_AD_TOKEN_PROVIDER_ADAPTER: Final = TypeAdapter[Callable[[], str | Awaitable[str]] | None](
+    Callable[[], str | Awaitable[str]] | None
+)
+
+
+def _callable_azure_ad_token_provider(provider: object) -> Callable[[], str | Awaitable[str]] | None:
+    return _AZURE_AD_TOKEN_PROVIDER_ADAPTER.validate_python(provider) if callable(provider) else None
 
 
 class _AzureGatewayClientParams(TypedDict, total=False, extra_items=object):
@@ -81,6 +89,160 @@ class AzureOpenAIError(BaseLLMException):
         )
 
 
+class EntraRejectionError(AzureOpenAIError):
+    pass
+
+
+_GUID: Final = re.compile(r"[0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}")
+_AADSTS_REASON: Final = re.compile(r"\bAADSTS(\d+)\b.*?(?:\.(?=[\s'\"\\]|$)|$)", re.MULTILINE)
+_ENTRA_CREDENTIAL_REJECTION_CODES: Final = frozenset(
+    {
+        50012,
+        50013,
+        50027,
+        50034,
+        50053,
+        50055,
+        50057,
+        50059,
+        50076,
+        50079,
+        50126,
+        50128,
+        53003,
+        65001,
+        70001,
+        70002,
+        70011,
+        70021,
+        70025,
+        70052,
+        90002,
+        90094,
+        500011,
+        500014,
+        700016,
+        700023,
+        700024,
+        700025,
+        700027,
+        700211,
+        700213,
+        700223,
+        700238,
+        7000112,
+        7000215,
+        7000218,
+        7000222,
+    }
+)
+
+
+def _entra_rejection_reason(message: str) -> str | None:
+    reason: Final = _AADSTS_REASON.search(message)
+    if reason is None or int(reason.group(1)) not in _ENTRA_CREDENTIAL_REJECTION_CODES:
+        return None
+    return _GUID.sub("<redacted>", reason.group(0))
+
+
+def _entra_rejection(reason: str) -> EntraRejectionError:
+    return EntraRejectionError(status_code=401, message=f"Failed to get Azure AD token: {reason}")
+
+
+def entra_rejection_as_auth_error(error: Exception) -> EntraRejectionError | None:
+    try:
+        from azure.core.exceptions import ClientAuthenticationError
+    except ImportError:
+        return None
+    if not isinstance(error, ClientAuthenticationError):
+        return None
+    status_code: Final = error.status_code
+    if status_code is not None and (status_code == 429 or status_code >= 500):
+        return None
+    reason: Final = _entra_rejection_reason(str(error))
+    return None if reason is None else _entra_rejection(reason)
+
+
+def _raise_entra_rejection_as_auth_error(error: Exception) -> NoReturn:
+    auth_error: Final = entra_rejection_as_auth_error(error)
+    if auth_error is None:
+        raise error
+    raise auth_error from error
+
+
+def entra_token(provider: Callable[[], str]) -> str:
+    try:
+        return provider()
+    except entra_rejection_types() as error:
+        _raise_entra_rejection_as_auth_error(error)
+
+
+def entra_rejection_types() -> tuple[type[Exception], ...]:
+    try:
+        from azure.core.exceptions import ClientAuthenticationError
+    except ImportError:
+        return ()
+    return (ClientAuthenticationError,)
+
+
+_EXCEPTION_CHAIN_MAX_DEPTH: Final = 16
+
+
+def _exception_chain(error: BaseException) -> Iterator[BaseException]:
+    link: BaseException | None = error  # rebind-ok: advances one link of the cause chain per iteration
+    for _ in range(_EXCEPTION_CHAIN_MAX_DEPTH):
+        if link is None:
+            return
+        yield link
+        link = link.__cause__ or link.__context__
+
+
+_REQUEST_FAILURE_TYPES: Final = (APIError, httpx.HTTPError)
+
+
+def _ends_the_failed_attempt(link: BaseException, failed_attempt: BaseException) -> bool:
+    return not isinstance(link, EntraRejectionError) and (
+        isinstance(link, _REQUEST_FAILURE_TYPES) or type(link) is type(failed_attempt)
+    )
+
+
+def _links_within_the_failed_attempt(error: BaseException) -> Iterator[BaseException]:
+    yield error
+    yield from itertools.takewhile(
+        lambda link: not _ends_the_failed_attempt(link, error), itertools.islice(_exception_chain(error), 1, None)
+    )
+
+
+def entra_rejection_in(error: BaseException) -> EntraRejectionError | None:
+    return next(
+        (link for link in _links_within_the_failed_attempt(error) if isinstance(link, EntraRejectionError)), None
+    )
+
+
+def caused_by_entra_rejection(error: BaseException) -> bool:
+    return entra_rejection_in(error) is not None
+
+
+async def _awaited_token(token: Awaitable[str]) -> str:
+    try:
+        return await token
+    except entra_rejection_types() as error:
+        _raise_entra_rejection_as_auth_error(error)
+
+
+def _entra_rejections_as_auth_errors(
+    provider: Callable[[], str | Awaitable[str]],
+) -> Callable[[], str | Awaitable[str]]:
+    def call_provider() -> str | Awaitable[str]:
+        try:
+            token: Final = provider()
+        except entra_rejection_types() as error:
+            _raise_entra_rejection_as_auth_error(error)
+        return token if isinstance(token, str) else _awaited_token(token)
+
+    return call_provider
+
+
 def process_azure_headers(headers: httpx.Headers | dict) -> dict:
     openai_headers: Final = {}
     if "x-ratelimit-limit-requests" in headers:
@@ -91,7 +253,7 @@ def process_azure_headers(headers: httpx.Headers | dict) -> dict:
         openai_headers["x-ratelimit-limit-tokens"] = headers["x-ratelimit-limit-tokens"]
     if "x-ratelimit-remaining-tokens" in headers:
         openai_headers["x-ratelimit-remaining-tokens"] = headers["x-ratelimit-remaining-tokens"]
-    llm_response_headers: Final = {"{}-{}".format("llm_provider", k): v for k, v in headers.items()}
+    llm_response_headers: Final = {f"llm_provider-{k}": v for k, v in headers.items()}
 
     return {**llm_response_headers, **openai_headers}
 
@@ -257,22 +419,34 @@ def _cached_oidc_token(cache_key: str) -> str | None:
     return cached if isinstance(cached, str) else None
 
 
+def _oidc_exchange_rejection_reason(error: MaskedHTTPStatusError) -> str | None:
+    if error.response.status_code not in (400, 401):
+        return None
+    return _entra_rejection_reason(error.text or error.response.text)
+
+
 def _exchange_oidc_token(cache_key: str, token_url: str, client_id: str, scope: str, oidc_token: str) -> str:
     cached_access_token: Final = _cached_oidc_token(cache_key)
     if cached_access_token is not None:
         return cached_access_token
 
     with closing(HTTPHandler(timeout=litellm.request_timeout)) as client:
-        req_token: Final = client.post(
-            token_url,
-            data={
-                "client_id": client_id,
-                "grant_type": "client_credentials",
-                "scope": scope,
-                "client_assertion_type": "urn:ietf:params:oauth:client-assertion-type:jwt-bearer",
-                "client_assertion": oidc_token,
-            },
-        )
+        try:
+            req_token: Final = client.post(
+                token_url,
+                data={
+                    "client_id": client_id,
+                    "grant_type": "client_credentials",
+                    "scope": scope,
+                    "client_assertion_type": "urn:ietf:params:oauth:client-assertion-type:jwt-bearer",
+                    "client_assertion": oidc_token,
+                },
+            )
+        except MaskedHTTPStatusError as error:
+            reason: Final = _oidc_exchange_rejection_reason(error)
+            if reason is None:
+                raise
+            raise _entra_rejection(reason) from error
 
     if req_token.status_code != 200:
         raise AzureOpenAIError(
@@ -483,6 +657,9 @@ def get_azure_ad_token(
             raise
         except Exception as e:
             verbose_logger.error("Error calling Azure AD token provider: %s", e)
+            auth_error: Final = entra_rejection_as_auth_error(e)
+            if auth_error is not None:
+                raise auth_error from e
             raise RuntimeError(f"Failed to get Azure AD token: {e}") from e
 
     return azure_ad_token
@@ -696,7 +873,7 @@ class BaseAzureLLM(BaseOpenAILLM):
         api_version: str | None,
         is_async: bool,
     ) -> dict:
-        azure_ad_token_provider = litellm_params.get("azure_ad_token_provider")
+        azure_ad_token_provider = _callable_azure_ad_token_provider(litellm_params.get("azure_ad_token_provider"))
         # If we have api_key, then we have higher priority
         azure_ad_token = litellm_params.get("azure_ad_token")
 
@@ -775,7 +952,7 @@ class BaseAzureLLM(BaseOpenAILLM):
             azure_client_params["timeout"] = timeout
 
         if azure_ad_token_provider is not None:
-            azure_client_params["azure_ad_token_provider"] = azure_ad_token_provider
+            azure_client_params["azure_ad_token_provider"] = _entra_rejections_as_auth_errors(azure_ad_token_provider)
         # this decides if we should set azure_endpoint or base_url on Azure OpenAI Client
         # required to support GPT-4 vision enhancements, since base_url needs to be set on Azure OpenAI Client
 

litellm/llms/azure/decisions/transformation.py

@@ -8,6 +8,7 @@ import litellm
 from litellm.llms.azure.common_utils import (
     BaseAzureLLM,
     cached_azure_ad_token_refresh_provider,
+    entra_token,
 )
 from litellm.llms.openai.decisions.transformation import OpenAIDecisionsConfig
 from litellm.secret_managers.main import get_secret_str
@@ -61,7 +62,7 @@ class AzureDecisionsConfig(OpenAIDecisionsConfig):
         scope: Final = (
             params.azure_scope or get_secret_str("AZURE_SCOPE") or "https://cognitiveservices.azure.com/.default"
         )
-        azure_headers["Authorization"] = f"Bearer {cached_azure_ad_token_refresh_provider(scope)()}"
+        azure_headers["Authorization"] = f"Bearer {entra_token(cached_azure_ad_token_refresh_provider(scope))}"
         return azure_headers
 
     async def avalidate_environment(

litellm/llms/azure/responses/transformation.py

@@ -3,11 +3,10 @@ from collections.abc import Callable, Mapping
 from copy import deepcopy
 from functools import partial
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, Literal
+from typing import TYPE_CHECKING, Any, Final, Literal, cast
 from urllib.parse import urlsplit
 
 import httpx
-from openai.types.responses import ResponseReasoningItem
 from pydantic import TypeAdapter
 
 from litellm._logging import verbose_logger
@@ -16,6 +15,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import prompt_cach
 from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config
 from litellm.llms.azure.common_utils import BaseAzureLLM
+from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
 from litellm.llms.base_llm.responses.transformation import SUMMARY_ONLY_REASONING_FIELD_VALUES, ReasoningFieldValues
 from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
 from litellm.types.llms.openai import *
@@ -72,6 +72,21 @@ def _with_responses_api_version(api_base: str | None, litellm_params: Mapping[st
     return MappingProxyType({**litellm_params, "api_version": _resolve_responses_api_version(api_base, litellm_params)})
 
 
+_AZURE_REJECTED_INPUT_FIELDS: Final = MappingProxyType(
+    {"message": frozenset({"status"}), "reasoning": frozenset({"status"})}
+)
+_AZURE_REJECTED_INPUT_FIELDS_UNDER_DROP_PARAMS: Final = MappingProxyType(
+    {**_AZURE_REJECTED_INPUT_FIELDS, "reasoning": frozenset({"status", "content"})}
+)
+
+
+def _without_fields_azure_rejects(item: object, rejected_fields: Mapping[str, frozenset[str]]) -> object:
+    if not is_str_keyed_mapping(item):
+        return item
+    rejected: Final = rejected_fields.get(str(item.get("type")), frozenset())
+    return {key: value for key, value in item.items() if key not in rejected} if rejected else item
+
+
 class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
     # Parameters not supported by Azure Responses API
     AZURE_UNSUPPORTED_PARAMS = ["context_management"]
@@ -122,68 +137,17 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
     def get_stripped_model_name(self, model: str) -> str:
         return model.replace("responses/", "").replace("o_series/", "").replace("azure_ai/", "")
 
-    def _handle_reasoning_item(self, item: dict[str, Any]) -> Mapping[str, object]:
-        """
-        Handle reasoning items to filter out the status field.
-        Issue: https://github.com/BerriAI/litellm/issues/13484
-
-        Azure OpenAI API does not accept 'status' field in reasoning input items.
-        """
-        if item.get("type") == "reasoning":
-            try:
-                # Ensure required fields are present for ResponseReasoningItem
-                item_data: Final = dict(item)
-                if "summary" not in item_data:
-                    item_data["summary"] = (
-                        item_data.get("reasoning_content", "")[:100] + "..."
-                        if len(item_data.get("reasoning_content", "")) > 100
-                        else item_data.get("reasoning_content", "")
-                    )
-
-                # Create ResponseReasoningItem object from the item data
-                reasoning_item: Final = ResponseReasoningItem.model_validate(item_data)
-
-                # Convert back to dict with exclude_none=True to exclude None fields
-                dict_reasoning_item: Final = reasoning_item.model_dump(exclude_none=True)
-                dict_reasoning_item.pop("status", None)
-
-                return dict_reasoning_item
-            except Exception as e:
-                verbose_logger.debug("Failed to create ResponseReasoningItem, falling back to manual filtering: %s", e)
-                # Fallback: manually filter out known None fields
-                filtered_item: Final = {
-                    k: v
-                    for k, v in item.items()
-                    if v is not None or k not in {"status", "content", "encrypted_content"}
-                }
-                return filtered_item
-        return item
-
     def _validate_input_param(
-        self, input: str | ResponseInputParam, *, drops_set_created_by: bool = False
+        self, input: str | ResponseInputParam, *, drop_params: bool = False
     ) -> str | ResponseInputParam:
-        """
-        Override parent method to also filter out 'status' field from message items.
-        Azure OpenAI API does not accept 'status' field in input messages.
-        """
-        from typing import cast
-
-        # First call parent's validation
-        validated_input: Final = super()._validate_input_param(input, drops_set_created_by=drops_set_created_by)
-
-        # Then filter out status from message items
-        if isinstance(validated_input, list):
-            filtered_input: Final[list[object]] = []
-            for item in validated_input:
-                if isinstance(item, dict) and item.get("type") == "message":
-                    # Filter out status field from message items
-                    filtered_item = {k: v for k, v in item.items() if k != "status"}
-                    filtered_input.append(filtered_item)
-                else:
-                    filtered_input.append(item)
-            return cast(ResponseInputParam, filtered_input)
-
-        return validated_input
+        validated_input: Final = super()._validate_input_param(input, drop_params=drop_params)
+        if not isinstance(validated_input, list):
+            return validated_input
+        rejected_fields: Final = (
+            _AZURE_REJECTED_INPUT_FIELDS_UNDER_DROP_PARAMS if drop_params else _AZURE_REJECTED_INPUT_FIELDS
+        )
+        sanitized_items: Final = [_without_fields_azure_rejects(item, rejected_fields) for item in validated_input]
+        return cast("ResponseInputParam", sanitized_items)  # cast-ok: items keep their shape minus rejected fields
 
     def transform_responses_api_request(
         self,

litellm/llms/azure/search/transformation.py

@@ -36,7 +36,7 @@ import httpx
 from pydantic import ConfigDict, ValidationError
 
 from litellm.litellm_core_utils.url_utils import ensure_api_route
-from litellm.llms.azure.common_utils import cached_azure_ad_token_refresh_provider
+from litellm.llms.azure.common_utils import cached_azure_ad_token_refresh_provider, entra_rejection_as_auth_error
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.base_llm.search.responses import (
     ResponsesSearchAnnotation,
@@ -270,6 +270,9 @@ class BingGroundingSearchConfig(BaseSearchConfig):
         try:
             return minter()
         except Exception as e:
+            rejection: Final = entra_rejection_as_auth_error(e)
+            if rejection is not None:
+                raise rejection from e
             raise ValueError(
                 f"Grounding with Bing Search: no credential available. Pass api_key, set {TOKEN_ENV} "
                 f"to an Entra bearer token, or configure azure-identity (AZURE_CLIENT_ID / "

litellm/llms/azure_ai/anthropic/count_tokens/handler.py

@@ -16,6 +16,7 @@ from litellm.anthropic_beta_headers_manager import aload_beta_headers_config
 from litellm.litellm_core_utils.asyncify import asyncify
 from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
 from litellm.llms.anthropic.common_utils import AnthropicError
+from litellm.llms.azure.common_utils import EntraRejectionError
 from litellm.llms.azure_ai.anthropic.count_tokens.transformation import (
     AzureAIAnthropicCountTokensConfig,
 )
@@ -135,6 +136,8 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
         except AnthropicError:
             # Re-raise Anthropic exceptions as-is
             raise
+        except EntraRejectionError as rejection:
+            raise AnthropicError(status_code=rejection.status_code, message=rejection.message) from rejection
         except httpx.HTTPStatusError as e:
             # HTTP errors - preserve the actual status code
             verbose_logger.error("HTTP error in CountTokens handler: %s", e)

litellm/llms/azure_ai/common_utils.py

@@ -1,5 +1,7 @@
 import asyncio
-from collections.abc import Mapping
+import os
+from collections.abc import Callable, Mapping
+from functools import partial
 from types import MappingProxyType
 from typing import Final, Literal
 from urllib.parse import urlparse
@@ -52,6 +54,28 @@ AZURE_ENTRA_CREDENTIAL_PARAM_KEYS: Final = frozenset({"azure_ad_token", "client_
 AZURE_ENTRA_LITELLM_PARAM_KEYS: Final = AZURE_ENTRA_CREDENTIAL_PARAM_KEYS | frozenset(
     {"tenant_id", "client_id", "azure_username", "azure_scope"}
 )
+AZURE_DEFAULT_SCOPE: Final = "https://cognitiveservices.azure.com/.default"
+_AZURE_ENTRA_ENVIRONMENT_VARIABLES: Final = MappingProxyType(
+    {
+        "tenant_id": "AZURE_TENANT_ID",
+        "client_id": "AZURE_CLIENT_ID",
+        "client_secret": "AZURE_CLIENT_SECRET",
+        "azure_username": "AZURE_USERNAME",
+        "azure_password": "AZURE_PASSWORD",
+        "azure_ad_token": "AZURE_AD_TOKEN",
+    }
+)
+
+
+def azure_entra_environment_defaults() -> Mapping[str, object]:
+    return MappingProxyType(
+        {
+            **{key: os.getenv(variable) for key, variable in _AZURE_ENTRA_ENVIRONMENT_VARIABLES.items()},
+            "azure_scope": os.getenv("AZURE_SCOPE", AZURE_DEFAULT_SCOPE),
+        }
+    )
+
+
 AZURE_ENTRA_CREDENTIAL_HELP: Final = (
     "Set `tenant_id` + `client_id` + `client_secret`, `azure_ad_token` (an `oidc/` token also needs "
     "`tenant_id` + `client_id`), or `client_id` + `azure_username` + `azure_password` in the agent's `litellm_params`"
@@ -70,6 +94,17 @@ def _resolve_config_secret(value: object) -> str | None:
     return get_secret_str(value) if value.startswith("os.environ/") else value
 
 
+def _agent_entra_token(provider: Callable[[], str], model: str) -> str:
+    from litellm.llms.azure.common_utils import EntraRejectionError, entra_token
+
+    try:
+        return entra_token(provider)
+    except EntraRejectionError as rejection:
+        raise litellm.AuthenticationError(
+            message=rejection.message, llm_provider="azure_ai", model=model
+        ) from rejection
+
+
 def get_azure_ai_agent_entra_token(litellm_params: Mapping[str, object]) -> str:
     """Mints the Entra bearer from the agent's own litellm_params, never from process-wide AZURE_* env vars."""
     from litellm.llms.azure.common_utils import (
@@ -88,18 +123,32 @@ def get_azure_ai_agent_entra_token(litellm_params: Mapping[str, object]) -> str:
     azure_username: Final = resolved["azure_username"]
     azure_password: Final = resolved["azure_password"]
     azure_ad_token: Final = resolved["azure_ad_token"]
+    model: Final = str(litellm_params.get("model") or "")
     if tenant_id and client_id and client_secret:
-        return get_azure_ad_token_from_entra_id(
-            tenant_id=tenant_id, client_id=client_id, client_secret=client_secret, scope=scope
-        )()
+        return _agent_entra_token(
+            get_azure_ad_token_from_entra_id(
+                tenant_id=tenant_id, client_id=client_id, client_secret=client_secret, scope=scope
+            ),
+            model,
+        )
     if client_id and azure_username and azure_password:
-        return get_azure_ad_token_from_username_password(
-            client_id=client_id, azure_username=azure_username, azure_password=azure_password, scope=scope
-        )()
+        return _agent_entra_token(
+            get_azure_ad_token_from_username_password(
+                client_id=client_id, azure_username=azure_username, azure_password=azure_password, scope=scope
+            ),
+            model,
+        )
     federated: Final = azure_ad_token is not None and azure_ad_token.startswith("oidc/")
     if azure_ad_token and federated and tenant_id and client_id:
-        return get_azure_ad_token_from_oidc(
-            azure_ad_token=azure_ad_token, azure_client_id=client_id, azure_tenant_id=tenant_id, scope=scope
+        return _agent_entra_token(
+            partial(
+                get_azure_ad_token_from_oidc,
+                azure_ad_token=azure_ad_token,
+                azure_client_id=client_id,
+                azure_tenant_id=tenant_id,
+                scope=scope,
+            ),
+            model,
         )
     if azure_ad_token and not federated:
         return azure_ad_token

litellm/llms/bedrock/count_tokens/kimi_k3.py

@@ -455,9 +455,7 @@ def _converse_request(
     if not all(_text_only(message) for message in messages):
         return None
     try:
-        chat_tools: Final = tuple(
-            chat_tool for tool in tools or () for chat_tool in _as_converse_input(tool)
-        )  # comprehension-ok: flattens each tool's translation
+        chat_tools: Final = tuple(chain.from_iterable(_as_converse_input(tool) for tool in tools or ()))
         return _ConverseRequest.model_validate(
             litellm.AmazonConverseConfig().transform_request(
                 model=model,

litellm/llms/bedrock/passthrough/guardrail_translation/handler.py

@@ -14,6 +14,7 @@ from pydantic import InstanceOf, TypeAdapter, ValidationError
 from typing_extensions import ReadOnly, TypedDict
 
 from litellm._logging import verbose_proxy_logger
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
 from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicMessagesHandler
 from litellm.llms.base_llm.guardrail_translation.base_translation import (
     BaseTranslation,
@@ -60,6 +61,8 @@ if TYPE_CHECKING:
 
 _CONVERSE_ACTIONS: Final = frozenset({"converse", "converse-stream"})
 _INVOKE_STREAM_ACTION: Final = "invoke-with-response-stream"
+_INVOKE_ACTIONS: Final = frozenset({"invoke", _INVOKE_STREAM_ACTION})
+_BLOCK_TYPES_THE_ANTHROPIC_HANDLER_DOES_NOT_SCAN: Final = frozenset({"document", "search_result"})
 _EVENT_STREAM_CONTENT_TYPE: Final = "vnd.amazon.eventstream"
 _EVENT_STREAM_MEDIA_TYPE: Final = "application/vnd.amazon.eventstream"
 
@@ -73,6 +76,83 @@ def _is_invoke_stream_endpoint(endpoint: str) -> bool:
     return endpoint.rstrip("/").split("/")[-1] == _INVOKE_STREAM_ACTION
 
 
+def _anthropic_invoke_body(
+    data: Mapping[str, object],
+) -> dict[str, object] | None:  # mutable-ok: the body guardrails write back into
+    """The body of an InvokeModel call in Anthropic Messages format, which Bedrock requires to carry anthropic_version"""
+    endpoint: Final = data.get("endpoint")
+    body: Final = data.get("data")
+    if not isinstance(endpoint, str) or endpoint.rstrip("/").split("/")[-1] not in _INVOKE_ACTIONS:
+        return None
+    if not isinstance(body, dict):
+        return None
+    invoke_body: Final = _BODY_BY_REFERENCE.validate_python(body)
+    if not isinstance(invoke_body.get("anthropic_version"), str) or not isinstance(invoke_body.get("messages"), list):
+        return None
+    messages: Final = _ARRAY_BY_REFERENCE.validate_python(invoke_body["messages"])
+    if not all(_is_well_formed_message(message) for message in messages):
+        return None
+    blocks: Final = _anthropic_message_blocks(messages)
+    if any(block.get("type") in _BLOCK_TYPES_THE_ANTHROPIC_HANDLER_DOES_NOT_SCAN for block in blocks):
+        return None
+    return invoke_body
+
+
+def _body_level_request_data(
+    passthrough_request: Mapping[str, object],
+) -> dict[str, object]:  # mutable-ok: the request dict guardrails receive
+    """What `/v1/messages` hands a guardrail as `request_data`: the Anthropic body at the top level, with the proxy's own keys (model, metadata, headers) on top"""
+    body: Final = as_str_mapping(passthrough_request.get("data")) or {}
+    proxy_keys: Final = {key: value for key, value in passthrough_request.items() if key != "data"}
+    return {**body, **proxy_keys}
+
+
+def _is_well_formed_message(message: object) -> bool:
+    """Whether the Messages translation can read `message`, so a malformed body still reaches Bedrock to be rejected there"""
+    fields: Final = as_str_mapping(message)
+    if fields is None or not isinstance(fields.get("role"), str):
+        return False
+    content: Final = fields.get("content")
+    if isinstance(content, str):
+        return True
+    return isinstance(content, list) and all(
+        as_str_mapping(block) is not None for block in _ARRAY_BY_REFERENCE.validate_python(content)
+    )
+
+
+def _anthropic_message_blocks(messages: Sequence[object]) -> tuple[Mapping[str, object], ...]:
+    fields: Final = (as_str_mapping(message) for message in messages)
+    return tuple(chain.from_iterable(_anthropic_content_blocks(field.get("content")) for field in fields if field))
+
+
+def _anthropic_content_blocks(content: object) -> tuple[Mapping[str, object], ...]:
+    blocks: Final[list[Mapping[str, object]]] = []  # mutable-ok: collected while walking the content
+    pending: Final[list[object]] = [content]  # mutable-ok: content lists still to walk, nested tool results included
+    while pending:
+        current = pending.pop()
+        if not isinstance(current, list):
+            continue
+        for item in _ARRAY_BY_REFERENCE.validate_python(current):
+            block = as_str_mapping(item)
+            if block is None:
+                continue
+            blocks.append(block)
+            if block.get("type") == "tool_result":
+                pending.append(block.get("content"))
+    return tuple(blocks)
+
+
+def _anthropic_invoke_reply(
+    response: object,
+    request_data: object,
+) -> dict[str, object] | None:  # mutable-ok: the reply guardrails write back into
+    if not isinstance(response, dict) or not isinstance(request_data, dict):
+        return None
+    if _anthropic_invoke_body(_OBJECT_BY_REFERENCE.validate_python(request_data)) is None:
+        return None
+    return _BODY_BY_REFERENCE.validate_python(response)
+
+
 def _generic_passthrough_handler() -> "PassThroughEndpointHandler":
     """
     Fallback for non-Converse Bedrock routes (e.g. invoke). The generic
@@ -1285,6 +1365,17 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
         body: Final = data.get("data")
 
         if not _is_converse_endpoint(endpoint):
+            passthrough_request: Final = _BODY_BY_REFERENCE.validate_python(data)
+            anthropic_body: Final = _anthropic_invoke_body(passthrough_request)
+            if anthropic_body is not None:
+                await AnthropicMessagesHandler().process_input_messages(  # pyright: ignore[reportUnknownMemberType]  # takes the untyped request dict
+                    data=anthropic_body,
+                    guardrail_to_apply=guardrail_to_apply,
+                    litellm_logging_obj=litellm_logging_obj,
+                    request_data=_body_level_request_data(passthrough_request),
+                    keep_cache_control=True,
+                )
+                return passthrough_request
             return await _generic_passthrough_handler().process_input_messages(
                 data=data,
                 guardrail_to_apply=guardrail_to_apply,
@@ -1410,6 +1501,15 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
             and not _is_converse_endpoint(endpoint)
             and not _is_invoke_stream_endpoint(endpoint)
         ):
+            anthropic_reply: Final = _anthropic_invoke_reply(response, request_data)  # pyright: ignore[reportUnknownArgumentType]  # the base signature types request_data as a bare dict
+            if anthropic_reply is not None:
+                return await AnthropicMessagesHandler().process_output_response(  # pyright: ignore[reportUnknownMemberType]  # takes the untyped request dict
+                    response=anthropic_reply,  # pyright: ignore[reportArgumentType]  # the Anthropic reply dict, written back in place
+                    guardrail_to_apply=guardrail_to_apply,
+                    litellm_logging_obj=litellm_logging_obj,
+                    user_api_key_dict=user_api_key_dict,
+                    request_data=_body_level_request_data(_OBJECT_BY_REFERENCE.validate_python(request_data or {})),
+                )
             return await _generic_passthrough_handler().process_output_response(
                 response=response,
                 guardrail_to_apply=guardrail_to_apply,

litellm/llms/chatgpt/authenticator.py

@@ -66,6 +66,13 @@ def get_cached_authenticator(auth_file: str) -> "Authenticator":
     return Authenticator(auth_file=auth_file)
 
 
+def resolve_chatgpt_authenticator(
+    litellm_params: "Mapping[str, object] | GenericLiteLLMParams | None", default: "Authenticator"
+) -> "Authenticator":
+    auth_file: Final = get_chatgpt_auth_file(litellm_params)
+    return get_cached_authenticator(auth_file) if auth_file else default
+
+
 def _token_refresh_timeout() -> httpx.Timeout:
     configured: Final = get_configured_request_timeout()
     seconds: Final = (

litellm/llms/chatgpt/chat/transformation.py

@@ -6,7 +6,7 @@ from litellm.llms.openai.openai import OpenAIConfig
 from litellm.types.llms.openai import AllMessageValues
 from litellm.types.router import GenericLiteLLMParams
 
-from ..authenticator import Authenticator, get_cached_authenticator, get_chatgpt_auth_file
+from ..authenticator import Authenticator, get_chatgpt_auth_file, resolve_chatgpt_authenticator
 from ..common_utils import (
     GetAccessTokenError,
     chatgpt_client_api_base,
@@ -30,12 +30,6 @@ class ChatGPTConfig(OpenAIConfig):
         super().__init__()
         self.authenticator = authenticator if authenticator is not None else Authenticator()
 
-    def _resolve_authenticator(
-        self, litellm_params: Mapping[str, object] | GenericLiteLLMParams | None
-    ) -> Authenticator:
-        auth_file: Final = get_chatgpt_auth_file(litellm_params)
-        return get_cached_authenticator(auth_file) if auth_file else self.authenticator
-
     @staticmethod
     def _get_access_token_or_raise(authenticator: Authenticator, model: str, llm_provider: str) -> str:
         try:
@@ -63,7 +57,7 @@ class ChatGPTConfig(OpenAIConfig):
             return chatgpt_client_api_base(api_base), client_credential.access_token, custom_llm_provider
         if api_key:
             return api_base or self.api_base_without_login(), api_key, custom_llm_provider
-        authenticator: Final = self._resolve_authenticator(litellm_params)
+        authenticator: Final = resolve_chatgpt_authenticator(litellm_params, self.authenticator)
         dynamic_api_base: Final = authenticator.get_api_base()
         dynamic_api_key: Final = self._get_access_token_or_raise(authenticator, model, custom_llm_provider)
         return dynamic_api_base, dynamic_api_key, custom_llm_provider
@@ -98,7 +92,7 @@ class ChatGPTConfig(OpenAIConfig):
                 ),
                 **strip_chatgpt_client_credential_headers(headers),
             )
-        authenticator: Final = self._resolve_authenticator(litellm_params)
+        authenticator: Final = resolve_chatgpt_authenticator(litellm_params, self.authenticator)
         resolved_api_key: Final = (
             self._get_access_token_or_raise(authenticator, model, "chatgpt")
             if get_chatgpt_auth_file(litellm_params)

litellm/llms/chatgpt/common_utils.py

@@ -7,7 +7,7 @@ import hashlib
 import json
 import os
 import platform
-from collections.abc import Mapping, Sequence
+from collections.abc import Mapping
 from dataclasses import dataclass
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Final, TypeAlias
@@ -20,14 +20,10 @@ from pydantic import JsonValue, TypeAdapter
 from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils
 from litellm.litellm_core_utils.provider_affinity import get_caller_litellm_session_id, get_requested_session_id
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
-from litellm.types.utils import ProviderSpecificHeader
 
 if TYPE_CHECKING:
     from litellm.types.router import GenericLiteLLMParams
 
-_SCOPED_HEADERS: Final[TypeAdapter[ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None]] = TypeAdapter(
-    ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None
-)
 _STRING_HEADERS: Final = TypeAdapter(Mapping[str, str])
 _PARAMS: Final = TypeAdapter(Mapping[str, object])
 
@@ -357,8 +353,8 @@ def get_chatgpt_request_credential(
     normalized: Final = _normalize_litellm_params(params)
     scoped: Final = normalized.get("provider_specific_header")
     extra: Final = normalized.get("extra_headers")
-    headers: Final = ProviderSpecificHeaderUtils.get_provider_specific_headers(
-        provider_specific_header=_SCOPED_HEADERS.validate_python(scoped), custom_llm_provider="chatgpt"
+    headers: Final = ProviderSpecificHeaderUtils.validated_provider_specific_headers(
+        provider_specific_header=scoped, custom_llm_provider="chatgpt"
     )
     return get_chatgpt_client_credential(headers) or get_chatgpt_client_credential(
         _STRING_HEADERS.validate_python(extra) if extra is not None else MappingProxyType({})

litellm/llms/chatgpt/responses/transformation.py

@@ -28,7 +28,7 @@ from litellm.types.llms.openai import (
 from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import LlmProviders
 
-from ..authenticator import Authenticator, get_cached_authenticator, get_chatgpt_auth_file
+from ..authenticator import Authenticator, get_chatgpt_auth_file, resolve_chatgpt_authenticator
 from ..common_utils import (
     CHATGPT_API_BASE,
     GetAccessTokenError,
@@ -64,10 +64,6 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig):
     def custom_llm_provider(self) -> LlmProviders:
         return LlmProviders.CHATGPT
 
-    def _resolve_authenticator(self, litellm_params: GenericLiteLLMParams | None) -> Authenticator:
-        auth_file: Final = get_chatgpt_auth_file(litellm_params)
-        return get_cached_authenticator(auth_file) if auth_file else self.authenticator
-
     def validate_environment(
         self,
         headers: dict[str, str],
@@ -81,7 +77,7 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig):
                 **get_chatgpt_default_headers(client_credential.access_token, client_credential.account_id, session_id),
                 **strip_chatgpt_client_credential_headers(headers),
             }
-        authenticator: Final = self._resolve_authenticator(litellm_params)
+        authenticator: Final = resolve_chatgpt_authenticator(litellm_params, self.authenticator)
         safe_headers: Final = strip_chatgpt_client_credential_headers(headers)
         configured_api_key: Final = litellm_params.api_key if litellm_params is not None else None
         if configured_api_key and not get_chatgpt_auth_file(litellm_params):

litellm/llms/databricks/messages/transformation.py

@@ -192,7 +192,7 @@ class DatabricksAnthropicMessagesConfig(DatabricksBase, AnthropicMessagesConfig)
             headers=headers,
         )
         self._normalize_system_role_messages(  # pyright: ignore[reportUnknownMemberType]  # the shared helper takes a bare dict
-            anthropic_messages_request, model=model, drop_params=getattr(litellm_params, "drop_params", None)
+            anthropic_messages_request, model=model, drop_params=litellm_params.drop_params
         )
         self._remove_scope_from_cache_control(anthropic_messages_request)  # pyright: ignore[reportUnknownMemberType]  # the shared helper takes a bare dict
         return anthropic_messages_request

litellm/llms/empiriolabs/decisions/transformation.py

@@ -0,0 +1,10 @@
+from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig
+
+
+class EmpirioLabsDecisionsConfig(BaseDecisionsConfig):
+    path = "/v1/decisions"
+    api_key_env = ("EMPIRIOLABS_API_KEY",)
+    api_base_env = ("EMPIRIOLABS_API_BASE",)
+
+    def get_default_api_base(self) -> str | None:
+        return "https://api.empiriolabs.ai"

litellm/llms/gemini/realtime/transformation.py

@@ -218,7 +218,7 @@ def _system_instruction_text(system_instruction: HttpxContentType | None) -> str
 class GeminiRealtimeConfig(BaseRealtimeConfig):
     _TOOL_CALL_ID_TO_NAME_MAX = 256  # LRU cap for call_id→name mapping
 
-    def __init__(self, drop_params: bool = False):
+    def __init__(self, drop_params: bool = False) -> None:
         super().__init__()
         self.drop_params = drop_params
         self._tool_call_id_to_name: OrderedDict[str, str] = OrderedDict()

litellm/llms/github_copilot/github_host.py

@@ -21,8 +21,10 @@ class GithubHostEndpoints:
     device_code_url: str
     access_token_url: str
     api_key_url: str
+    copilot_user_url: str
     user_url: str
     client_id: str
+    exchanges_copilot_token: bool
     copilot_api_hosts: frozenset[str]
     copilot_api_host_suffix: str | None
     default_copilot_api_base: str
@@ -49,14 +51,17 @@ def github_host_endpoints(host: str) -> GithubHostEndpoints:
     if normalized is None:
         raise ValueError("host must be github.com or a single-subdomain GHE.com hostname")
     client_id: Final = os.getenv("GITHUB_COPILOT_CLIENT_ID", DEFAULT_GITHUB_CLIENT_ID)
+    exchanges_copilot_token: Final = client_id == DEFAULT_GITHUB_CLIENT_ID
     if normalized == GITHUB_COPILOT_DEFAULT_HOST:
         return GithubHostEndpoints(
             host=normalized,
             device_code_url=os.getenv("GITHUB_COPILOT_DEVICE_CODE_URL", DEFAULT_GITHUB_DEVICE_CODE_URL),
             access_token_url=os.getenv("GITHUB_COPILOT_ACCESS_TOKEN_URL", DEFAULT_GITHUB_ACCESS_TOKEN_URL),
             api_key_url=os.getenv("GITHUB_COPILOT_API_KEY_URL", DEFAULT_GITHUB_API_KEY_URL),
+            copilot_user_url="https://api.github.com/copilot_internal/user",
             user_url="https://api.github.com/user",
             client_id=client_id,
+            exchanges_copilot_token=exchanges_copilot_token,
             copilot_api_hosts=frozenset({"githubcopilot.com"}),
             copilot_api_host_suffix=".githubcopilot.com",
             default_copilot_api_base="https://api.githubcopilot.com",
@@ -70,8 +75,10 @@ def github_host_endpoints(host: str) -> GithubHostEndpoints:
         device_code_url=f"https://{normalized}/login/device/code",
         access_token_url=f"https://{normalized}/login/oauth/access_token",
         api_key_url=f"https://{api_host}/copilot_internal/v2/token",
+        copilot_user_url=f"https://{api_host}/copilot_internal/user",
         user_url=f"https://{api_host}/user",
         client_id=client_id,
+        exchanges_copilot_token=exchanges_copilot_token,
         copilot_api_hosts=frozenset({copilot_api_host}),
         copilot_api_host_suffix=None,
         default_copilot_api_base=f"https://{copilot_api_host}",

litellm/llms/github_copilot/per_user_auth.py

@@ -46,6 +46,7 @@ _LLM_PROVIDER: Final = "github_copilot"
 _DEVICE_FLOW_SCOPE: Final = "read:user"
 _SESSION_CACHE_MAX_SIZE: Final = 1000
 _SYNC_LOCK_COUNT: Final = 64
+_ACCOUNT_LOOKUP_TTL_SECONDS: Final = 1800
 
 
 class _SessionCache(Protocol):
@@ -125,12 +126,20 @@ class _CopilotEndpointsPayload(TypedDict):
     api: ReadOnly[NotRequired[object]]
 
 
+@with_config(ConfigDict(extra="allow", strict=True))
+class _CopilotAccountPayload(TypedDict):
+    access_type_sku: ReadOnly[NotRequired[object]]
+    chat_enabled: ReadOnly[NotRequired[object]]
+    endpoints: ReadOnly[NotRequired[object]]
+
+
 @with_config(ConfigDict(extra="allow", strict=True))
 class _GithubUserPayload(TypedDict):
     login: ReadOnly[str]
 
 
 _COPILOT_TOKEN: Final = TypeAdapter(_CopilotTokenPayload)
+_COPILOT_ACCOUNT: Final = TypeAdapter(_CopilotAccountPayload)
 _DEVICE_CODE: Final = TypeAdapter(_DeviceCodePayload)
 _ACCESS_TOKEN_POLL: Final = TypeAdapter(_AccessTokenPollPayload)
 _GITHUB_USER: Final = TypeAdapter(_GithubUserPayload)
@@ -273,6 +282,8 @@ def _session_from_response(
             llm_provider=_LLM_PROVIDER,
             model="",
         )
+    if not endpoints.exchanges_copilot_token:
+        return _session_from_account(response, github_token, credential_name, cache_key, endpoints)
     try:
         payload: Final = _COPILOT_TOKEN.validate_python(response.json())
     except Exception as e:
@@ -282,13 +293,7 @@ def _session_from_response(
             model="",
         ) from e
 
-    payload_endpoints: Final[object] = payload.get("endpoints")
-    endpoints_api: Final[object] = (
-        _COPILOT_ENDPOINTS.validate_python(payload_endpoints).get("api")
-        if isinstance(payload_endpoints, Mapping)
-        else None
-    )
-    api_base: Final = validated_copilot_api_base(endpoints_api, endpoints)
+    api_base: Final = validated_copilot_api_base(_endpoints_api(payload.get("endpoints")), endpoints)
     session: Final = GithubCopilotUserSession(token=payload["token"], api_base=api_base)
     expires_at: Final = _expires_at_seconds(payload.get("expires_at"))
     if expires_at > 0:
@@ -296,6 +301,41 @@ def _session_from_response(
     return session
 
 
+def _session_from_account(
+    response: httpx.Response,
+    github_token: str,
+    credential_name: str,
+    cache_key: str,
+    endpoints: GithubHostEndpoints,
+) -> GithubCopilotUserSession:
+    try:
+        payload: Final = _COPILOT_ACCOUNT.validate_python(response.json())
+    except Exception as e:
+        raise ServiceUnavailableError(
+            message="GitHub Copilot account lookup returned an invalid response",
+            llm_provider=_LLM_PROVIDER,
+            model="",
+        ) from e
+    if payload.get("chat_enabled") is False or payload.get("access_type_sku") == "no_access":
+        _SESSION_CACHE.delete_cache(cache_key)
+        raise _reject_connection(credential_name)
+
+    api_base: Final = validated_copilot_api_base(_endpoints_api(payload.get("endpoints")), endpoints)
+    session: Final = GithubCopilotUserSession(token=github_token, api_base=api_base)
+    _cache_session(cache_key, github_token, session, time.time() + _ACCOUNT_LOOKUP_TTL_SECONDS)
+    return session
+
+
+def _endpoints_api(payload_endpoints: object) -> object:
+    if not isinstance(payload_endpoints, Mapping):
+        return None
+    return _COPILOT_ENDPOINTS.validate_python(payload_endpoints).get("api")
+
+
+def _session_lookup_url(endpoints: GithubHostEndpoints) -> str:
+    return endpoints.api_key_url if endpoints.exchanges_copilot_token else endpoints.copilot_user_url
+
+
 def _to_int(raw: object) -> int:
     if isinstance(raw, bool):
         return 0
@@ -337,7 +377,7 @@ def _do_exchange(
             _SyncGetClient, litellm.module_level_client
         )
         response: Final = client.get(
-            endpoints.api_key_url,
+            _session_lookup_url(endpoints),
             headers=github_api_headers(github_token),
         )
     except Exception as e:
@@ -367,7 +407,7 @@ async def _do_aexchange(
     client: Final = _new_async_github_client()
     try:
         response: Final = await client.get(
-            endpoints.api_key_url,
+            _session_lookup_url(endpoints),
             headers=github_api_headers(github_token),
         )
     except Exception as e:

litellm/llms/openai/cost_calculation.py

@@ -31,9 +31,9 @@ def cost_per_token(
     Input:
         - model: str, the model name without provider prefix
         - usage: LiteLLM Usage block, containing anthropic caching information
-        - data_residency: optional OpenAI data-residency region (e.g. "eu", "us"),
-          inferred from api_base. Applies the model's regional-processing
-          uplift multiplier when set.
+        - data_residency: optional OpenAI data-residency mode (e.g. "eu", "us", "fedramp"),
+          inferred from api_base, including gov.api.openai.com. Applies the
+          model's regional-processing uplift multiplier when set.
 
     Returns:
         Tuple[float, float] - prompt_cost_in_usd, completion_cost_in_usd

litellm/llms/openai/data_residency.py

@@ -7,9 +7,12 @@ enabled and rejects requests sent to the wrong host, so the api_base hostname
 is the authoritative signal of which region a request was processed in.
 """
 
+import os
 from typing import Final
 from urllib.parse import urlparse
 
+import litellm
+
 # Mapping of OpenAI regional hostnames to the corresponding data-residency
 # value used by the cost calculator. See
 # https://developers.openai.com/api/docs/pricing for the regional-processing
@@ -17,6 +20,7 @@ from urllib.parse import urlparse
 _OPENAI_REGIONAL_HOSTS: Final[dict[str, str]] = {
     "eu.api.openai.com": "eu",
     "us.api.openai.com": "us",
+    "gov.api.openai.com": "fedramp",
 }
 
 
@@ -25,13 +29,22 @@ def infer_openai_data_residency(custom_llm_provider: str | None, api_base: str |
     Derive the OpenAI data-residency region from an api_base URL.
 
     Returns ``"eu"`` for the EU regional host, ``"us"`` for the US regional
-    host, and ``None`` for the default global host, any non-OpenAI provider,
-    or any non-OpenAI URL.
+    host, ``"fedramp"`` for ``gov.api.openai.com``, and ``None`` for the
+    default global host, any non-OpenAI provider, or any non-OpenAI URL.
+
+    When ``api_base`` is unset, falls back to the same defaults the OpenAI
+    handler uses (``litellm.api_base``, ``OPENAI_BASE_URL``, ``OPENAI_API_BASE``)
+    so env-configured regional endpoints are billed with their uplift.
     """
-    if custom_llm_provider != "openai" or not api_base:
+    if custom_llm_provider != "openai":
+        return None
+    resolved_api_base: Final = (
+        api_base or litellm.api_base or os.getenv("OPENAI_BASE_URL") or os.getenv("OPENAI_API_BASE")
+    )
+    if not resolved_api_base:
         return None
     try:
-        host: Final = urlparse(api_base).hostname
+        host: Final = urlparse(resolved_api_base).hostname
     except (TypeError, ValueError):
         return None
     if not host:

litellm/llms/openai/responses/transformation.py

@@ -669,7 +669,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         prompt_cache_options: object = None,
     ) -> tuple[str | ResponseInputParam, Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None]:
         validated_input: Final = self._validate_input_param(
-            self._drop_bridge_minted_reasoning_items(input), drops_set_created_by=drops_params(litellm_params)
+            self._drop_bridge_minted_reasoning_items(input), drop_params=drops_params(litellm_params)
         )
         reads_breakpoints: Final = self._reads_prompt_cache_breakpoints(
             model=model, litellm_params=litellm_params, prompt_cache_options=prompt_cache_options
@@ -951,7 +951,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         return tools if all(new is old for new, old in zip(sanitized, tools, strict=True)) else sanitized
 
     def _validate_input_param(
-        self, input: str | ResponseInputParam, *, drops_set_created_by: bool = False
+        self, input: str | ResponseInputParam, *, drop_params: bool = False
     ) -> str | ResponseInputParam:
         """
         Ensure all input fields if pydantic are converted to dict
@@ -965,9 +965,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
                 # if it's pydantic, convert to dict
                 if isinstance(item, BaseModel):
                     validated_input.append(
-                        replayable_responses_item(
-                            item.model_dump(exclude_none=True), drops_set_created_by=drops_set_created_by
-                        )
+                        replayable_responses_item(item.model_dump(exclude_none=True), drops_set_created_by=drop_params)
                     )
                 elif isinstance(item, dict):
                     validated_input.append(
@@ -975,7 +973,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
                             self._handle_reasoning_item(cast(dict[str, object], item))
                             if item.get("type") == "reasoning"
                             else item,
-                            drops_set_created_by=drops_set_created_by,
+                            drops_set_created_by=drop_params,
                         )
                     )
                 else:

litellm/llms/openai_like/dynamic_config.py

@@ -2,9 +2,11 @@
 Dynamic configuration class generator for JSON-based providers.
 """
 
-from collections.abc import Coroutine
+from collections.abc import Coroutine, Mapping
 from typing import TYPE_CHECKING, Final, Literal, overload
 
+from pydantic import TypeAdapter, ValidationError
+
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     handle_messages_with_content_list_to_str_conversion,
@@ -20,6 +22,26 @@ from .json_loader import SimpleProviderConfig
 if TYPE_CHECKING:
     from litellm.llms.openai_like.responses.transformation import OpenAILikeResponsesConfig
 
+_PAID_SEARCH_TOOLS: Final = frozenset({"web_search", "web_search_image", "image_search"})
+_PAYLOAD_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
+_TOOL_USAGE_ADAPTER: Final[TypeAdapter[Mapping[str, int]]] = TypeAdapter(Mapping[str, int])
+
+
+def web_search_requests_from_tool_usage(payload: object) -> int | None:
+    """
+    Billable search calls in a parsed OpenAI-compatible chat response whose usage carries a
+    tool_usage {tool_name: count} map, as EmpirioLabs reports built-in tool invocations.
+    Free tools (code_interpreter, web_extractor) are not counted.
+    """
+    try:
+        body: Final = _PAYLOAD_ADAPTER.validate_python(payload)
+        usage: Final = _PAYLOAD_ADAPTER.validate_python(body.get("usage"))
+        tool_usage: Final = _TOOL_USAGE_ADAPTER.validate_python(usage.get("tool_usage"), strict=True)
+    except ValidationError:
+        return None
+    calls: Final = sum(count for tool, count in tool_usage.items() if tool in _PAID_SEARCH_TOOLS and count > 0)
+    return calls if calls > 0 else None
+
 
 def create_config_class(provider: SimpleProviderConfig):
     """Generate config class dynamically from JSON configuration"""
@@ -179,6 +201,25 @@ def create_config_class(provider: SimpleProviderConfig):
 
             return optional_params
 
+        def transform_parsed_response_dict(
+            self,
+            parsed_response: dict[str, object],  # mutable-ok: BaseConfig signature
+        ) -> dict[str, object]:  # mutable-ok: BaseConfig signature
+            """
+            Carry the provider's usage.tool_usage search count as usage.server_tool_use.web_search_requests,
+            the field the built-in tool cost path bills per call.
+            """
+            repaired: Final = _PAYLOAD_ADAPTER.validate_python(
+                super().transform_parsed_response_dict(parsed_response)  # pyright: ignore[reportUnknownMemberType]  # base_class is picked at runtime
+            )
+            if not provider.special_handling.get("web_search_requests_from_tool_usage"):
+                return {**repaired}
+            web_search_requests: Final = web_search_requests_from_tool_usage(repaired)
+            if web_search_requests is None:
+                return {**repaired}
+            usage: Final = _PAYLOAD_ADAPTER.validate_python(repaired["usage"])
+            return {**repaired, "usage": {**usage, "server_tool_use": {"web_search_requests": web_search_requests}}}
+
         @property
         def custom_llm_provider(self) -> str | None:
             return provider.slug