Merge pull request #32400 from BerriAI/litellm_internal_staging

chore(ci): promote internal staging to main

.circleci/config.yml

@@ -5,6 +5,16 @@ orbs:
   win: circleci/windows@5.0 # Add Windows orb
 
 commands:
+  skip_if_unrelated_changes:
+    parameters:
+      category:
+        type: enum
+        enum: ["backend", "client"]
+        default: "backend"
+    steps:
+      - run:
+          name: "Skip job when no << parameters.category >>-relevant files changed"
+          command: bash .circleci/scripts/path_filter.sh << parameters.category >>
   setup_google_dns:
     steps:
       - run:
@@ -282,6 +292,7 @@ jobs:
     parallelism: 4
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - restore_cache:
           keys:
@@ -354,6 +365,7 @@ jobs:
     parallelism: 4
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - restore_cache:
           keys:
@@ -427,6 +439,7 @@ jobs:
 
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - restore_cache:
           keys:
@@ -480,6 +493,7 @@ jobs:
 
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - run:
@@ -545,6 +559,7 @@ jobs:
       DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - run:
@@ -584,6 +599,7 @@ jobs:
       DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - run:
@@ -624,6 +640,7 @@ jobs:
       DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - run:
@@ -656,6 +673,7 @@ jobs:
       FAKE_OPENAI_API_BASE: http://127.0.0.1:8190
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - restore_cache:
@@ -705,6 +723,7 @@ jobs:
 
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - restore_cache:
@@ -755,6 +774,7 @@ jobs:
 
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - run:
@@ -787,6 +807,7 @@ jobs:
 
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - restore_cache:
@@ -832,6 +853,7 @@ jobs:
 
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - run:
@@ -877,6 +899,7 @@ jobs:
 
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - run:
@@ -918,6 +941,7 @@ jobs:
 
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - run:
@@ -963,6 +987,7 @@ jobs:
 
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - run:
@@ -1007,6 +1032,7 @@ jobs:
 
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - restore_cache:
@@ -1045,6 +1071,7 @@ jobs:
 
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - run:
@@ -1089,6 +1116,7 @@ jobs:
 
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - run:
@@ -1132,6 +1160,7 @@ jobs:
 
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - run:
@@ -1163,6 +1192,7 @@ jobs:
 
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - run:
@@ -1205,6 +1235,7 @@ jobs:
 
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - run:
@@ -1248,6 +1279,7 @@ jobs:
 
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - run:
@@ -1291,6 +1323,7 @@ jobs:
 
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - run:
@@ -1321,6 +1354,7 @@ jobs:
 
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - run:
@@ -1366,6 +1400,7 @@ jobs:
 
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - run:
@@ -1407,6 +1442,7 @@ jobs:
 
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - restore_cache:
           keys:
@@ -1459,6 +1495,7 @@ jobs:
 
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - run:
@@ -1482,6 +1519,7 @@ jobs:
 
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - run:
@@ -1507,6 +1545,7 @@ jobs:
 
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - run:
@@ -1531,6 +1570,7 @@ jobs:
 
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - attach_workspace:
           at: ~/project
       - setup_google_dns
@@ -1570,14 +1610,14 @@ jobs:
       - run:
           name: Run helm lint
           command: |
-            helm lint ./deploy/charts/litellm-helm
+            helm lint ./helm/litellm-helm
 
       # Run helm tests
       - run:
           name: Run helm tests
           command: |
             IMAGE_TAG=${CIRCLE_SHA1:-ci}
-            helm install litellm ./deploy/charts/litellm-helm -f ./deploy/charts/litellm-helm/ci/test-values.yaml \
+            helm install litellm ./helm/litellm-helm -f ./helm/litellm-helm/ci/test-values.yaml \
               --set image.repository=litellm-ci \
               --set image.tag=${IMAGE_TAG} \
               --set image.pullPolicy=Never
@@ -1606,6 +1646,7 @@ jobs:
     working_directory: ~/project
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - run:
@@ -1698,6 +1739,7 @@ jobs:
     working_directory: ~/project
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - attach_workspace:
           at: ~/project
       - setup_google_dns
@@ -1787,6 +1829,7 @@ jobs:
     working_directory: ~/project
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - run:
@@ -1869,6 +1912,7 @@ jobs:
     working_directory: ~/project
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - run:
@@ -2000,6 +2044,7 @@ jobs:
     working_directory: ~/project
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - run:
@@ -2085,6 +2130,7 @@ jobs:
     working_directory: ~/project
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - run:
@@ -2180,6 +2226,7 @@ jobs:
     working_directory: ~/project
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - run:
@@ -2201,6 +2248,7 @@ jobs:
           # the OTEL test - should get this as a trace
           command: |
             docker run -d \
+              --restart on-failure \
               -p 4000:4000 \
               -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
               -e STORE_MODEL_IN_DB="True" \
@@ -2252,6 +2300,7 @@ jobs:
     working_directory: ~/project
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       # Remove Docker CLI installation since it's already available in machine executor
       - install_uv
@@ -2333,6 +2382,7 @@ jobs:
     working_directory: ~/project
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - run:
@@ -2471,6 +2521,7 @@ jobs:
     working_directory: ~/project
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - setup_google_dns
       - install_uv
       - run:
@@ -2537,6 +2588,7 @@ jobs:
       - *python312_image
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - attach_workspace:
           at: .
       # Check file locations
@@ -2567,6 +2619,8 @@ jobs:
     working_directory: ~/project
     steps:
       - checkout
+      - skip_if_unrelated_changes:
+          category: client
       - setup_google_dns
       - restore_cache:
           keys:
@@ -2609,6 +2663,8 @@ jobs:
     working_directory: ~/project
     steps:
       - checkout
+      - skip_if_unrelated_changes:
+          category: client
       - setup_google_dns
       - restore_cache:
           keys:
@@ -2654,6 +2710,8 @@ jobs:
       PROXY_LOGOUT_URL: "https://www.example.com"
     steps:
       - checkout
+      - skip_if_unrelated_changes:
+          category: client
       - setup_google_dns
       - install_uv
       - restore_cache:
@@ -2791,6 +2849,8 @@ jobs:
       SERVER_ROOT_PATH: "/litellm"
     steps:
       - checkout
+      - skip_if_unrelated_changes:
+          category: client
       - setup_google_dns
       - install_uv
       - restore_cache:
@@ -2892,6 +2952,7 @@ jobs:
     working_directory: ~/project
     steps:
       - checkout
+      - skip_if_unrelated_changes
 
       - run:
           name: Build Docker image
@@ -2917,6 +2978,7 @@ jobs:
     working_directory: ~/project
     steps:
       - checkout
+      - skip_if_unrelated_changes
       - attach_workspace:
           at: ~/project
       - setup_google_dns

.circleci/scripts/classify_changes.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -uo pipefail
+
+category="${1:?usage: classify_changes.sh <backend|client>}"
+
+has_client=false
+has_backend=false
+while IFS= read -r file || [ -n "$file" ]; do
+  [ -n "$file" ] || continue
+  case "$file" in
+    ui/*) has_client=true ;;
+    docs/* | *.md | *.mdx) : ;;
+    *) has_backend=true ;;
+  esac
+done
+
+case "$category" in
+  backend)
+    [ "$has_backend" = true ] && echo run || echo skip
+    ;;
+  client)
+    { [ "$has_client" = true ] || [ "$has_backend" = true ]; } && echo run || echo skip
+    ;;
+  *)
+    echo run
+    ;;
+esac

.circleci/scripts/path_filter.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+set -uo pipefail
+
+category="${1:?usage: path_filter.sh <backend|client>}"
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+run_full() {
+  echo "path-filter[$category]: running job ($1)"
+  exit 0
+}
+
+[ -n "${CIRCLE_PULL_REQUEST:-}" ] || run_full "not a pull request"
+
+candidate_bases="main litellm_internal_staging litellm_oss_staging"
+merge_base=""
+for base in $candidate_bases; do
+  git fetch --quiet origin "$base" 2>/dev/null || continue
+  candidate="$(git merge-base HEAD FETCH_HEAD 2>/dev/null)" || continue
+  [ -n "$candidate" ] || continue
+  if [ -z "$merge_base" ] || git merge-base --is-ancestor "$merge_base" "$candidate" 2>/dev/null; then
+    merge_base="$candidate"
+  fi
+done
+
+[ -n "$merge_base" ] || run_full "could not resolve a merge base against $candidate_bases"
+
+changed="$(git diff --name-only "$merge_base" HEAD 2>/dev/null)" || run_full "git diff failed"
+[ -n "$changed" ] || run_full "no files changed vs $merge_base"
+
+echo "path-filter[$category]: changed files vs ${merge_base}:"
+printf '%s\n' "$changed" | sed 's/^/  /' || true
+
+decision="$(printf '%s\n' "$changed" | bash "$here/classify_changes.sh" "$category")" || run_full "classify_changes.sh failed"
+
+if [ "$decision" = run ]; then
+  run_full "$category-relevant changes detected"
+fi
+
+echo "path-filter[$category]: only unrelated (docs/client) changes detected; halting job as successful"
+circleci-agent step halt

.github/pull_request_template.md

@@ -13,7 +13,7 @@
 - [ ] I have added meaningful tests
 - [ ] My PR passes all CI/CD checks (e.g., lint, format, unit tests)
 - [ ] My PR's scope is as isolated as possible; it only solves 1 specific problem
-- [ ] I have requested a Greptile review by commenting `@greptileai` and received a **Confidence Score of at least 4/5** before requesting a maintainer review
+- [ ] I have received a Greptile **Confidence Score of at least 4/5** before requesting a maintainer review (Greptile reviews automatically once the PR is opened; only comment `@greptileai` to re-request a review after pushing changes)
 
 ## Delays in PR merge?
 
@@ -24,6 +24,7 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
 <!-- Include screenshots, screen recordings, or command (e.g., curl) + output demonstrating that your changes work as expected
      The proof must be completely e2e with no mocks, using, for example, actual LLM calls costing real $. `pytest` commands are not enough
      For bug fixes: show reproduction before the fix and passing behavior after
+     Include the commit hash each proof was captured at, for both the before and the after runs
      For new features: show the feature working end-to-end
      For UI changes: include before/after screenshots -->
 

.github/workflows/codspeed.yml

@@ -4,9 +4,11 @@ on:
   push:
     branches:
       - main
+      - litellm_internal_staging
   pull_request:
     branches:
       - main
+      - litellm_internal_staging
   # Allow CodSpeed to trigger backtest performance analysis
   # in order to generate initial data
   workflow_dispatch:
@@ -22,7 +24,7 @@ concurrency:
 jobs:
   benchmarks:
     runs-on: ubuntu-24.04
-    timeout-minutes: 15
+    timeout-minutes: 60
 
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0

.github/workflows/helm_unit_test.yml

@@ -38,4 +38,6 @@ jobs:
           echo "Helm unittest plugin integrity verified: $ACTUAL_SHA"
 
       - name: Run unit tests
-        run: helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm
+        run: |
+          helm unittest -f 'tests/*.yaml' helm/litellm-helm
+          helm unittest -f 'tests/*.yaml' helm/litellm

.github/workflows/test-terraform-provider.yml

@@ -0,0 +1,113 @@
+name: Terraform Provider
+
+on:
+  push:
+    paths:
+      - "terraform/provider/**"
+      - ".github/workflows/test-terraform-provider.yml"
+  pull_request:
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_staging
+      - "litellm_**"
+    paths:
+      - "terraform/provider/**"
+      - "litellm/proxy/**"
+      - ".github/workflows/test-terraform-provider.yml"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  provider-checks:
+    name: gofmt, vet, build, test
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    defaults:
+      run:
+        working-directory: terraform/provider
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0
+        with:
+          go-version-file: terraform/provider/go.mod
+          cache: true
+          cache-dependency-path: terraform/provider/go.sum
+
+      - name: gofmt
+        run: |
+          UNFORMATTED=$(gofmt -l .)
+          if [ -n "${UNFORMATTED}" ]; then
+            echo "::error::gofmt required for: ${UNFORMATTED}"
+            exit 1
+          fi
+
+      - name: go vet
+        run: go vet ./...
+
+      - name: Build
+        run: go build ./...
+
+      - name: Test
+        run: go test -timeout 120s ./...
+
+  endpoint-drift:
+    name: Provider endpoints vs proxy OpenAPI schema
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
+        with:
+          version: "0.10.9"
+
+      - name: Cache uv dependencies
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        with:
+          path: |
+            ~/.cache/uv
+            .venv
+          key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-
+
+      - name: Install dependencies
+        run: |
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
+
+      - name: Generate Prisma client
+        env:
+          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
+        run: |
+          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+
+      - name: Generate proxy OpenAPI schema
+        run: |
+          uv run --no-sync python terraform/provider/tools/dump_openapi.py "${RUNNER_TEMP}/openapi.json"
+
+      - uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0
+        with:
+          go-version-file: terraform/provider/go.mod
+          cache: true
+          cache-dependency-path: terraform/provider/go.sum
+
+      - name: Audit provider endpoints against the schema
+        working-directory: terraform/provider
+        run: go run ./tools/endpointaudit -provider-dir ./litellm -spec "${RUNNER_TEMP}/openapi.json"

.gitignore

@@ -52,9 +52,8 @@ ui/litellm-dashboard/node_modules
 ui/litellm-dashboard/next-env.d.ts
 ui/litellm-dashboard/package.json
 ui/litellm-dashboard/package-lock.json
-deploy/charts/litellm/*.tgz
-deploy/charts/litellm/charts/*
-deploy/charts/*.tgz
+helm/litellm-helm/*.tgz
+helm/*.tgz
 litellm/proxy/vertex_key.json
 **/.vim/
 **/node_modules

CLAUDE.md

@@ -21,11 +21,11 @@ End-to-end tests belong in `tests/e2e/` and must follow the harness conventions
 
 When creating PRs, don't set base to `main`. `litellm_internal_staging` serves that purpose
 
-When writing a PR body, treat the comments and imperative instructions inside @.github/pull_request_template.md as rules to follow, not just layout
+When writing a PR body, treat the comments and imperative instructions inside @.github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule
 
 If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just leave the section blank
 
-Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, just tell me which URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, what fields to fill out, etc. along with the other commands to run in an ordered list, and I'll do it myself and post the screenshots after you make the PR
+Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`; the Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, just tell me which URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, what fields to fill out, etc. along with the other commands to run in an ordered list, and I'll do it myself and post the screenshots after you make the PR
 
 If you ever make public-facing PR descriptions, comments, issues, commit messages, etc., always follow these guidelines to sound less AI-y:
 - don't use emojis
@@ -47,9 +47,11 @@ If you're trying to create a new function that relies on untyped stuff, instead
 
 If you get an LIT001 or LIT002 fail, refactor the code to follow functional programming best practices rather than introducing mutable data structures. For example, build values in one shot with comprehensions or generators wrapped in `tuple()` / `frozenset()` instead of seeding an empty `list`/`dict`/`set` and mutating it over time. Ideally `# mutable-ok` is never used; reach for it only as a genuine last resort when an immutable rewrite is truly impossible, and always pair it with a real reason
 
+Every lint or type suppression must name the exact rule inside brackets and carry a reason comment, e.g. `# pyright: ignore[reportArgumentType]  # stubs lack async overload` or `# noqa: TID251  # <reason>`. `# type: ignore` is banned (LIT009): pyrightconfig.json sets `enableTypeIgnoreComments` to false, so it silently does nothing
+
 Commit and push your work when you're done without asking
 
-When you must use real LLM models to, for example, write e2e tests, write a QA runbook, etc., make sure to use the latest models (doesn't have to be smartest, can also be a modern small, fast one. No strong preference for smart vs fast here, just use something modern) as of the year and month of the current date. Do a web search as necessary to figure that out
+When referencing or running models (coding, QA'ing, writing docs, writing tests, etc.), use the latest model in that model family unless otherwise specified; treat your training knowledge, memories, configs, and tests as stale, and determine the family's latest with model_prices_and_context_window.json or the web
 
 If you're an internal contributor, when creating a new PR, the typical flow is to branch off litellm_internal_staging and create a branch prefixed with litellm_. Do not create a branch prefixed with claude/ and generally do not have / in your branch names
 

Dockerfile

@@ -1,10 +1,10 @@
 # syntax=docker/dockerfile:1.7
 
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
 ARG UI_BUILD_IMAGE=node:20.18-alpine3.20@sha256:3488b10bf958af7125a176419d2d8a9937d895bf124012aae811651988d2ffe6

Makefile

@@ -265,7 +265,7 @@ test-integration: install-test-deps
 	$(UV_RUN) pytest tests/ -k "not test_litellm"
 
 test-unit-helm: install-helm-unittest
-	helm unittest -f 'tests/*.yaml' deploy/charts/litellm-helm
+	helm unittest -f 'tests/*.yaml' helm/litellm-helm
 
 # LLM Translation testing targets
 test-llm-translation: install-test-deps

backend/Dockerfile

@@ -1,5 +1,5 @@
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 
 FROM $UV_IMAGE AS uvbin

basedpyright-code-budget.json

@@ -3,16 +3,16 @@
     "limit": 37484
   },
   "reportArgumentType": {
-    "limit": 2721
+    "limit": 2704
   },
   "reportAssignmentType": {
     "limit": 330
   },
   "reportAttributeAccessIssue": {
-    "limit": 519
+    "limit": 516
   },
   "reportCallIssue": {
-    "limit": 131
+    "limit": 124
   },
   "reportConstantRedefinition": {
     "limit": 59
@@ -42,7 +42,7 @@
     "limit": 18
   },
   "reportIndexIssue": {
-    "limit": 39
+    "limit": 37
   },
   "reportInvalidTypeForm": {
     "limit": 35
@@ -51,7 +51,7 @@
     "limit": 5
   },
   "reportMatchNotExhaustive": {
-    "limit": 2
+    "limit": 0
   },
   "reportMissingParameterType": {
     "limit": 5900
@@ -63,25 +63,25 @@
     "limit": 41
   },
   "reportOperatorIssue": {
-    "limit": 9
+    "limit": 0
   },
   "reportOptionalCall": {
-    "limit": 7
+    "limit": 0
   },
   "reportOptionalIterable": {
-    "limit": 6
+    "limit": 0
   },
   "reportOptionalMemberAccess": {
-    "limit": 1086
+    "limit": 1085
   },
   "reportOptionalOperand": {
-    "limit": 6
+    "limit": 0
   },
   "reportOptionalSubscript": {
-    "limit": 17
+    "limit": 0
   },
   "reportPossiblyUnboundVariable": {
-    "limit": 78
+    "limit": 77
   },
   "reportPrivateUsage": {
     "limit": 2438
@@ -90,28 +90,28 @@
     "limit": 12
   },
   "reportReturnType": {
-    "limit": 226
+    "limit": 225
   },
   "reportTypedDictNotRequiredAccess": {
-    "limit": 30
+    "limit": 27
   },
   "reportUndefinedVariable": {
-    "limit": 5
+    "limit": 0
   },
   "reportUnknownArgumentType": {
-    "limit": 45905
+    "limit": 45894
   },
   "reportUnknownLambdaType": {
     "limit": 113
   },
   "reportUnknownMemberType": {
-    "limit": 40556
+    "limit": 40541
   },
   "reportUnknownParameterType": {
     "limit": 20418
   },
   "reportUnknownVariableType": {
-    "limit": 32168
+    "limit": 32151
   },
   "reportUnnecessaryCast": {
     "limit": 177

codecov.yaml

@@ -15,6 +15,16 @@ ignore:
 flag_management:
   default_rules:
     carryforward: true
+  # Dead flags no CI job uploads anymore: their carried-forward sessions were
+  # measured against old revisions, and the stale line maps mark comment lines
+  # of since-edited files as missed, sinking patch coverage on unrelated PRs.
+  individual_flags:
+    - name: proxy-mgmt-behavior
+      carryforward: false
+    - name: security
+      carryforward: false
+    - name: proxy-db-schema-migration
+      carryforward: false
 
 component_management:
   individual_components:

deploy/azure_resource_manager/azure_marketplace/createUiDefinition.json

@@ -1,15 +0,0 @@
-{
-    "$schema": "https://schema.management.azure.com/schemas/0.1.2-preview/CreateUIDefinition.MultiVm.json#",
-    "handler": "Microsoft.Azure.CreateUIDef",
-    "version": "0.1.2-preview",
-    "parameters": {
-        "config": {
-            "isWizard": false,
-            "basics": { }
-        },
-        "basics": [ ],
-        "steps": [ ],
-        "outputs": { },
-        "resourceTypes": [ ]
-    }
-}

deploy/azure_resource_manager/azure_marketplace/mainTemplate.json

@@ -1,63 +0,0 @@
-{
-    "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
-    "contentVersion": "1.0.0.0",
-    "parameters": {
-      "imageName": {
-        "type": "string",
-        "defaultValue": "ghcr.io/berriai/litellm:main-latest"
-      },
-      "containerName": {
-        "type": "string",
-        "defaultValue": "litellm-container"
-      },
-      "dnsLabelName": {
-        "type": "string",
-        "defaultValue": "litellm"
-      },
-      "portNumber": {
-        "type": "int",
-        "defaultValue": 4000
-      }
-    },
-    "resources": [
-      {
-        "type": "Microsoft.ContainerInstance/containerGroups",
-        "apiVersion": "2021-03-01",
-        "name": "[parameters('containerName')]",
-        "location": "[resourceGroup().location]",
-        "properties": {
-          "containers": [
-            {
-              "name": "[parameters('containerName')]",
-              "properties": {
-                "image": "[parameters('imageName')]",
-                "resources": {
-                  "requests": {
-                    "cpu": 1,
-                    "memoryInGB": 2
-                  }
-                },
-                "ports": [
-                  {
-                    "port": "[parameters('portNumber')]"
-                  }
-                ]
-              }
-            }
-          ],
-          "osType": "Linux",
-          "restartPolicy": "Always",
-          "ipAddress": {
-            "type": "Public",
-            "ports": [
-              {
-                "protocol": "tcp",
-                "port": "[parameters('portNumber')]"
-              }
-            ],
-            "dnsNameLabel": "[parameters('dnsLabelName')]"
-          }
-        }
-      }
-    ]
-  }

deploy/azure_resource_manager/main.bicep

@@ -1,42 +0,0 @@
-param imageName string = 'ghcr.io/berriai/litellm:main-latest'
-param containerName string = 'litellm-container'
-param dnsLabelName string = 'litellm'
-param portNumber int = 4000
-
-resource containerGroupName 'Microsoft.ContainerInstance/containerGroups@2021-03-01' = {
-  name: containerName
-  location: resourceGroup().location
-  properties: {
-    containers: [
-      {
-        name: containerName
-        properties: {
-          image: imageName
-          resources: {
-            requests: {
-              cpu: 1
-              memoryInGB: 2
-            }
-          }
-          ports: [
-            {
-              port: portNumber
-            }
-          ]
-        }
-      }
-    ]
-    osType: 'Linux'
-    restartPolicy: 'Always'
-    ipAddress: {
-      type: 'Public'
-      ports: [
-        {
-          protocol: 'tcp'
-          port: portNumber
-        }
-      ]
-      dnsNameLabel: dnsLabelName
-    }
-  }
-}

docker/Dockerfile.database

@@ -1,10 +1,10 @@
 # syntax=docker/dockerfile:1.7
 
 # Base image for building
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
 
 # Runtime image
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.
 ARG UI_BUILD_IMAGE=node:20.18-alpine3.20@sha256:3488b10bf958af7125a176419d2d8a9937d895bf124012aae811651988d2ffe6

docker/Dockerfile.non_root

@@ -1,8 +1,8 @@
 # syntax=docker/dockerfile:1.7
 
 # Base images
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
 ARG PROXY_EXTRAS_SOURCE=published
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 # Pinned by digest like the other base images; bump explicitly on Node upgrades.

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -17,6 +17,7 @@ if TYPE_CHECKING:
     from litellm.proxy._types import LiteLLM_ManagedObjectTable
     from litellm.proxy.utils import PrismaClient, ProxyLogging
     from litellm.router import Router
+    from litellm.types.utils import LiteLLMBatch
 
 
 CHECK_BATCH_COST_USER_AGENT = "LiteLLM Proxy/CheckBatchCost"
@@ -277,13 +278,20 @@ class CheckBatchCost:
         except Exception:
             return None
 
-    async def check_batch_cost(self):
+    async def _track_completed_batch_cost(
+        self,
+        job: "LiteLLM_ManagedObjectTable",
+        response: "LiteLLMBatch",
+        model_id: str,
+        batch_id: str,
+        prom_logger: Optional["PrometheusLogger"],
+    ) -> Optional[Tuple[Optional[str], Optional[str]]]:
         """
-        Check if the batch JOB has been tracked.
-        - get all status="validating" and file_purpose="batch" jobs
-        - check if batch is now complete
-        - if not, return False
-        - if so, return True
+        Fetch a completed batch's results, compute cost/usage, and emit the
+        aretrieve_batch spend log. Returns (model_name, llm_provider) on
+        success, None when the job can't be routed to a deployment. Raises on
+        results-fetch or cost-computation failures so the caller can leave the
+        job unprocessed and retry it on a later poll.
         """
         from litellm.batches.batch_utils import (
             _get_file_content_as_dictionary,
@@ -296,6 +304,184 @@ class CheckBatchCost:
             _is_base64_encoded_unified_file_id,
         )
 
+        verbose_proxy_logger.info(
+            f"Batch ID: {batch_id} is complete, tracking cost and usage"
+        )
+
+        # aretrieve_batch is called with the raw provider batch ID, so response.id
+        # is the raw provider value (e.g. "batch_20260223-0518.234"). We need the
+        # unified base64 ID in the S3 log so downstream consumers can correlate it
+        # back to the batch they submitted via the proxy.
+        #
+        # CheckBatchCost builds its own LiteLLMLogging object (logging_obj below) and
+        # calls async_success_handler(result=response) directly. That handler calls
+        # _build_standard_logging_payload(response, ...) which reads response.id at
+        # that point — so setting response.id here is sufficient.
+        #
+        # The HTTP endpoint does this substitution via the managed files hook
+        # (async_post_call_success_hook). CheckBatchCost bypasses that hook entirely,
+        # so we do it explicitly here.
+        response.id = job.unified_object_id
+
+        # This background job runs as default_user_id, so going through the HTTP endpoint
+        # would trigger check_managed_file_id_access and get 403. Instead, extract the raw
+        # provider file ID and call afile_content directly with deployment credentials.
+        raw_output_file_id = response.output_file_id
+        decoded = _is_base64_encoded_unified_file_id(raw_output_file_id)
+        if decoded:
+            try:
+                raw_output_file_id = decoded.split("llm_output_file_id,")[1].split(";")[0]
+            except (IndexError, AttributeError):
+                pass
+
+        credentials = self.llm_router.get_deployment_credentials_with_provider(model_id) or {}
+        _file_content = await afile_content(
+            file_id=raw_output_file_id,
+            **credentials,
+        )
+
+        # Access content - handle both direct attribute and method call
+        if hasattr(_file_content, 'content'):
+            content_bytes = _file_content.content  # type: ignore[union-attr]
+        elif hasattr(_file_content, 'read'):
+            content_bytes = await _file_content.read()  # type: ignore[misc]
+        else:
+            content_bytes = _file_content  # type: ignore[assignment]
+
+        file_content_as_dict = _get_file_content_as_dictionary(
+            content_bytes  # type: ignore[arg-type]
+        )
+
+        # Record output file size
+        if prom_logger and content_bytes:
+            try:
+                prom_logger.record_managed_file_size(
+                    size_bytes=len(content_bytes),  # type: ignore
+                    purpose="batch",
+                    file_type="output",
+                    model=model_id,
+                )
+            except Exception:
+                pass
+
+        deployment_info = self.llm_router.get_deployment(model_id=model_id)
+        if deployment_info is None:
+            verbose_proxy_logger.info(
+                f"Skipping job {job.unified_object_id} because it is not a valid deployment info"
+            )
+            self._record_error(prom_logger, "deployment_not_found")
+            return None
+        custom_llm_provider = deployment_info.litellm_params.custom_llm_provider
+        litellm_model_name = deployment_info.litellm_params.model
+
+        model_name, llm_provider, _, _ = get_llm_provider(
+            model=litellm_model_name,
+            custom_llm_provider=custom_llm_provider,
+        )
+
+        # CheckBatchCost bypasses async_post_call_success_hook, so convert raw
+        # output/error file IDs to managed base64 IDs before the DB write here.
+        managed_files_hook = self.proxy_logging_obj.get_proxy_hook("managed_files")
+        if managed_files_hook is not None:
+            from litellm.proxy._types import UserAPIKeyAuth
+            _minimal_auth = UserAPIKeyAuth(
+                user_id=job.created_by or "default-user-id",
+                team_id=getattr(job, "team_id", None),
+            )
+            for _file_attr in ["output_file_id", "error_file_id"]:
+                _raw_file_id = getattr(response, _file_attr, None)
+                if _raw_file_id and not _is_base64_encoded_unified_file_id(_raw_file_id):
+                    try:
+                        _unified_file_id = managed_files_hook.get_unified_output_file_id(
+                            output_file_id=_raw_file_id,
+                            model_id=model_id,
+                            model_name=str(model_name) if model_name else deployment_info.model_name or None,
+                        )
+                        await managed_files_hook.store_unified_file_id(
+                            file_id=_unified_file_id,
+                            file_object=None,
+                            litellm_parent_otel_span=None,
+                            model_mappings={model_id: _raw_file_id},
+                            user_api_key_dict=_minimal_auth,
+                        )
+                        setattr(response, _file_attr, _unified_file_id)
+                        verbose_proxy_logger.info(
+                            f"CheckBatchCost: converted {_file_attr} "
+                            f"{_raw_file_id!r} -> managed ID for batch {batch_id}"
+                        )
+                    except Exception as _e:
+                        verbose_proxy_logger.warning(
+                            f"CheckBatchCost: failed to create managed file ID for "
+                            f"{_file_attr}={_raw_file_id!r}: {_e}"
+                        )
+
+        # Pass deployment model_info so custom batch pricing
+        # (input_cost_per_token_batches etc.) is used for cost calc
+        deployment_model_info = deployment_info.model_info.model_dump() if deployment_info.model_info else {}
+        batch_cost, batch_usage, batch_models = (
+            await calculate_batch_cost_and_usage(
+                file_content_dictionary=file_content_as_dict,
+                custom_llm_provider=llm_provider,  # type: ignore
+                model_name=model_name,
+                model_info=deployment_model_info,  # type: ignore[arg-type]
+            )
+        )
+        logging_obj = LiteLLMLogging(
+            model=batch_models[0],
+            messages=[{"role": "user", "content": "<retrieve_batch>"}],
+            stream=False,
+            call_type="aretrieve_batch",
+            start_time=datetime.now(),
+            litellm_call_id=str(uuid.uuid4()),
+            function_id=str(uuid.uuid4()),
+        )
+
+        creator_user_id = job.created_by
+        user_info = await self._get_user_info(batch_id, job.created_by)
+
+        logging_obj.update_environment_variables(
+            litellm_params={
+                # set the user-agent header so that S3 callback consumers can easily identify CheckBatchCost callbacks
+                "proxy_server_request": {
+                    "headers": {
+                        "user-agent": CHECK_BATCH_COST_USER_AGENT,
+                    }
+                },
+                "metadata": {
+                    "user_api_key_user_id": creator_user_id,
+                    **user_info,
+                },
+            },
+            optional_params={},
+        )
+
+        await logging_obj.async_success_handler(
+            result=response,
+            batch_cost=batch_cost,
+            batch_usage=batch_usage,
+            batch_models=batch_models,
+        )
+
+        # Record batch duration (completed_at - created_at)
+        if prom_logger and response.completed_at and response.created_at:
+            duration_seconds = float(response.completed_at - response.created_at)
+            if duration_seconds >= 0:
+                prom_logger.record_managed_batch_duration(
+                    duration_seconds=duration_seconds,
+                    model=model_name,
+                    api_provider=str(llm_provider) if llm_provider else None,
+                )
+
+        return model_name, str(llm_provider) if llm_provider else None
+
+    async def check_batch_cost(self):
+        """
+        Check if the batch JOB has been tracked.
+        - get all status="validating" and file_purpose="batch" jobs
+        - check if batch is now complete
+        - if not, return False
+        - if so, return True
+        """
         try:
             from litellm.integrations.prometheus import PrometheusLogger
             prom_logger = PrometheusLogger.get_instance()
@@ -381,177 +567,26 @@ class CheckBatchCost:
                 response.status == "completed"
                 and response.output_file_id is not None
             ):
-                verbose_proxy_logger.info(
-                    f"Batch ID: {batch_id} is complete, tracking cost and usage"
-                )
-
-                # aretrieve_batch is called with the raw provider batch ID, so response.id
-                # is the raw provider value (e.g. "batch_20260223-0518.234"). We need the
-                # unified base64 ID in the S3 log so downstream consumers can correlate it
-                # back to the batch they submitted via the proxy.
-                #
-                # CheckBatchCost builds its own LiteLLMLogging object (logging_obj below) and
-                # calls async_success_handler(result=response) directly. That handler calls
-                # _build_standard_logging_payload(response, ...) which reads response.id at
-                # that point — so setting response.id here is sufficient.
-                #
-                # The HTTP endpoint does this substitution via the managed files hook
-                # (async_post_call_success_hook). CheckBatchCost bypasses that hook entirely,
-                # so we do it explicitly here.
-                response.id = job.unified_object_id
-
-                # This background job runs as default_user_id, so going through the HTTP endpoint
-                # would trigger check_managed_file_id_access and get 403. Instead, extract the raw
-                # provider file ID and call afile_content directly with deployment credentials.
-                raw_output_file_id = response.output_file_id
-                decoded = _is_base64_encoded_unified_file_id(raw_output_file_id)
-                if decoded:
-                    try:
-                        raw_output_file_id = decoded.split("llm_output_file_id,")[1].split(";")[0]
-                    except (IndexError, AttributeError):
-                        pass
-
-                credentials = self.llm_router.get_deployment_credentials_with_provider(model_id) or {}
-                _file_content = await afile_content(
-                    file_id=raw_output_file_id,
-                    **credentials,
-                )
-
-                # Access content - handle both direct attribute and method call
-                if hasattr(_file_content, 'content'):
-                    content_bytes = _file_content.content  # type: ignore[union-attr]
-                elif hasattr(_file_content, 'read'):
-                    content_bytes = await _file_content.read()  # type: ignore[misc]
-                else:
-                    content_bytes = _file_content  # type: ignore[assignment]
-
-                file_content_as_dict = _get_file_content_as_dictionary(
-                    content_bytes  # type: ignore[arg-type]
-                )
-
-                # Record output file size
-                if prom_logger and content_bytes:
-                    try:
-                        prom_logger.record_managed_file_size(
-                            size_bytes=len(content_bytes),  # type: ignore
-                            purpose="batch",
-                            file_type="output",
-                            model=model_id,
-                        )
-                    except Exception:
-                        pass
-
-                deployment_info = self.llm_router.get_deployment(model_id=model_id)
-                if deployment_info is None:
-                    verbose_proxy_logger.info(
-                        f"Skipping job {job.unified_object_id} because it is not a valid deployment info"
+                try:
+                    tracked = await self._track_completed_batch_cost(
+                        job=job,
+                        response=response,
+                        model_id=model_id,
+                        batch_id=batch_id,
+                        prom_logger=prom_logger,
                     )
-                    if prom_logger:
-                        prom_logger.record_check_batch_cost_error("deployment_not_found")
+                except Exception as tracking_err:
+                    verbose_proxy_logger.error(
+                        f"CheckBatchCost: failed to track cost for batch {batch_id} "
+                        f"(job {job.id}); leaving it unprocessed so the next poll retries: {tracking_err}"
+                    )
+                    self._record_error(prom_logger, "cost_tracking_error")
+                    continue
+                if tracked is None:
                     continue
-                custom_llm_provider = deployment_info.litellm_params.custom_llm_provider
-                litellm_model_name = deployment_info.litellm_params.model
-
-                model_name, llm_provider, _, _ = get_llm_provider(
-                    model=litellm_model_name,
-                    custom_llm_provider=custom_llm_provider,
-                )
-
-                # CheckBatchCost bypasses async_post_call_success_hook, so convert raw
-                # output/error file IDs to managed base64 IDs before the DB write here.
-                managed_files_hook = self.proxy_logging_obj.get_proxy_hook("managed_files")
-                if managed_files_hook is not None:
-                    from litellm.proxy._types import UserAPIKeyAuth
-                    _minimal_auth = UserAPIKeyAuth(
-                        user_id=job.created_by or "default-user-id",
-                        team_id=getattr(job, "team_id", None),
-                    )
-                    for _file_attr in ["output_file_id", "error_file_id"]:
-                        _raw_file_id = getattr(response, _file_attr, None)
-                        if _raw_file_id and not _is_base64_encoded_unified_file_id(_raw_file_id):
-                            try:
-                                _unified_file_id = managed_files_hook.get_unified_output_file_id(
-                                    output_file_id=_raw_file_id,
-                                    model_id=model_id,
-                                    model_name=str(model_name) if model_name else deployment_info.model_name or None,
-                                )
-                                await managed_files_hook.store_unified_file_id(
-                                    file_id=_unified_file_id,
-                                    file_object=None,
-                                    litellm_parent_otel_span=None,
-                                    model_mappings={model_id: _raw_file_id},
-                                    user_api_key_dict=_minimal_auth,
-                                )
-                                setattr(response, _file_attr, _unified_file_id)
-                                verbose_proxy_logger.info(
-                                    f"CheckBatchCost: converted {_file_attr} "
-                                    f"{_raw_file_id!r} -> managed ID for batch {batch_id}"
-                                )
-                            except Exception as _e:
-                                verbose_proxy_logger.warning(
-                                    f"CheckBatchCost: failed to create managed file ID for "
-                                    f"{_file_attr}={_raw_file_id!r}: {_e}"
-                                )
-
-                # Pass deployment model_info so custom batch pricing
-                # (input_cost_per_token_batches etc.) is used for cost calc
-                deployment_model_info = deployment_info.model_info.model_dump() if deployment_info.model_info else {}
-                batch_cost, batch_usage, batch_models = (
-                    await calculate_batch_cost_and_usage(
-                        file_content_dictionary=file_content_as_dict,
-                        custom_llm_provider=llm_provider,  # type: ignore
-                        model_name=model_name,
-                        model_info=deployment_model_info,  # type: ignore[arg-type]
-                    )
-                )
-                logging_obj = LiteLLMLogging(
-                    model=batch_models[0],
-                    messages=[{"role": "user", "content": "<retrieve_batch>"}],
-                    stream=False,
-                    call_type="aretrieve_batch",
-                    start_time=datetime.now(),
-                    litellm_call_id=str(uuid.uuid4()),
-                    function_id=str(uuid.uuid4()),
-                )
-
-                creator_user_id = job.created_by
-                user_info = await self._get_user_info(batch_id, job.created_by)
-
-                logging_obj.update_environment_variables(
-                    litellm_params={
-                        # set the user-agent header so that S3 callback consumers can easily identify CheckBatchCost callbacks
-                        "proxy_server_request": {
-                            "headers": {
-                                "user-agent": CHECK_BATCH_COST_USER_AGENT,
-                            }
-                        },
-                        "metadata": {
-                            "user_api_key_user_id": creator_user_id,
-                            **user_info,
-                        },
-                    },
-                    optional_params={},
-                )
-
-                await logging_obj.async_success_handler(
-                    result=response,
-                    batch_cost=batch_cost,
-                    batch_usage=batch_usage,
-                    batch_models=batch_models,
-                )
-
-                # Record batch duration (completed_at - created_at)
-                if prom_logger and response.completed_at and response.created_at:
-                    duration_seconds = float(response.completed_at - response.created_at)
-                    if duration_seconds >= 0:
-                        prom_logger.record_managed_batch_duration(
-                            duration_seconds=duration_seconds,
-                            model=model_name,
-                            api_provider=str(llm_provider) if llm_provider else None,
-                        )
 
                 # Track this job for the final metrics summary
-                processed_models.append((model_name, str(llm_provider) if llm_provider else None))
+                processed_models.append(tracked)
 
                 # mark the job as complete
                 try:

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.47"
+version = "0.1.48"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.47"
+version = "0.1.48"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

gateway/Dockerfile

@@ -1,5 +1,5 @@
-ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
-ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:c61ac6919b811ea53c4782d69f1fe05218ba3c25d53f01b6ab7892e621bd4370
+ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
+ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:42df77a9974d6ec8b17a5ee8bc23b532600a44d705acef2409e0933c1251b45f
 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
 
 FROM $UV_IMAGE AS uvbin

gateway/main.py

@@ -25,17 +25,25 @@ DatabaseURLSettings.from_env().apply_to_env()
 
 from litellm.proxy.proxy_server import app
 
-from gateway.routes.allowlist import GATEWAY_EXACT_PATHS, GATEWAY_PATH_PREFIXES
+from gateway.routes.allowlist import (
+    GATEWAY_EXACT_PATHS,
+    GATEWAY_MOUNT_PATHS,
+    GATEWAY_PATH_PREFIXES,
+)
 
 
 def _is_gateway_route(route) -> bool:
-    """Keep the route on the gateway if its path is in the LLM data-plane surface."""
+    """Keep the route on the gateway if its path is in the LLM data-plane surface.
+
+    Prometheus registers /metrics as a Mount (``app.mount("/metrics", make_asgi_app())``),
+    so Mounts are matched against GATEWAY_MOUNT_PATHS instead of being dropped with
+    the UI static mounts.
+    """
     path = getattr(route, "path", None)
     if path is None:
         return False
     if isinstance(route, Mount):
-        # Gateway never serves the static UI or its asset bundles.
-        return False
+        return path in GATEWAY_MOUNT_PATHS
     if path in GATEWAY_EXACT_PATHS:
         return True
     return any(path.startswith(prefix) for prefix in GATEWAY_PATH_PREFIXES)

gateway/routes/allowlist.py

@@ -106,7 +106,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
     # Health & ops
     "/health",
     "/metrics",
-    "/watsonx"
+    "/watsonx",
 )
 
 GATEWAY_EXACT_PATHS: frozenset[str] = frozenset(
@@ -120,3 +120,9 @@ GATEWAY_EXACT_PATHS: frozenset[str] = frozenset(
         "/test",
     }
 )
+
+GATEWAY_MOUNT_PATHS: frozenset[str] = frozenset(
+    {
+        "/metrics",
+    }
+)

helm/litellm/templates/backend/deployment.yaml

@@ -45,11 +45,16 @@ spec:
               value: /app/config/config.yaml
             {{- end }}
           {{- include "litellm.envFrom" .Values.backend | nindent 10 }}
-          {{- if .Values.gateway.config.create }}
+          {{- if or .Values.gateway.config.create .Values.backend.volumeMounts }}
           volumeMounts:
+            {{- if .Values.gateway.config.create }}
             - name: gateway-config
               mountPath: /app/config/config.yaml
               subPath: config.yaml
+            {{- end }}
+            {{- with .Values.backend.volumeMounts }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
           {{- end }}
           {{- with .Values.backend.livenessProbe }}
           livenessProbe:
@@ -61,11 +66,16 @@ spec:
           {{- end }}
           resources:
             {{- toYaml .Values.backend.resources | nindent 12 }}
-      {{- if .Values.gateway.config.create }}
+      {{- if or .Values.gateway.config.create .Values.backend.volumes }}
       volumes:
+        {{- if .Values.gateway.config.create }}
         - name: gateway-config
           configMap:
             name: {{ include "litellm.gateway.fullname" . }}-config
+        {{- end }}
+        {{- with .Values.backend.volumes }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
       {{- end }}
       {{- with .Values.backend.nodeSelector }}
       nodeSelector:

helm/litellm/templates/gateway/deployment.yaml

@@ -47,11 +47,16 @@ spec:
               value: {{ .Values.gateway.numWorkers | quote }}
             {{- end }}
           {{- include "litellm.envFrom" .Values.gateway | nindent 10 }}
-          {{- if .Values.gateway.config.create }}
+          {{- if or .Values.gateway.config.create .Values.gateway.volumeMounts }}
           volumeMounts:
+            {{- if .Values.gateway.config.create }}
             - name: gateway-config
               mountPath: /app/config/config.yaml
               subPath: config.yaml
+            {{- end }}
+            {{- with .Values.gateway.volumeMounts }}
+            {{- toYaml . | nindent 12 }}
+            {{- end }}
           {{- end }}
           {{- with .Values.gateway.livenessProbe }}
           livenessProbe:
@@ -63,11 +68,16 @@ spec:
           {{- end }}
           resources:
             {{- toYaml .Values.gateway.resources | nindent 12 }}
-      {{- if .Values.gateway.config.create }}
+      {{- if or .Values.gateway.config.create .Values.gateway.volumes }}
       volumes:
+        {{- if .Values.gateway.config.create }}
         - name: gateway-config
           configMap:
             name: {{ include "litellm.gateway.fullname" . }}-config
+        {{- end }}
+        {{- with .Values.gateway.volumes }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
       {{- end }}
       {{- with .Values.gateway.nodeSelector }}
       nodeSelector:

helm/litellm/templates/ui/deployment.yaml

@@ -46,6 +46,10 @@ spec:
             {{- toYaml . | nindent 12 }}
             {{- end }}
           {{- include "litellm.envFrom" .Values.ui | nindent 10 }}
+          {{- with .Values.ui.volumeMounts }}
+          volumeMounts:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
           {{- with .Values.ui.livenessProbe }}
           livenessProbe:
             {{- toYaml . | nindent 12 }}
@@ -56,6 +60,10 @@ spec:
           {{- end }}
           resources:
             {{- toYaml .Values.ui.resources | nindent 12 }}
+      {{- with .Values.ui.volumes }}
+      volumes:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       {{- with .Values.ui.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

helm/litellm/tests/deployment_volumes_tests.yaml

@@ -0,0 +1,172 @@
+suite: test deployment volumes and volumeMounts
+templates:
+  - gateway/deployment.yaml
+  - gateway/configmap.yaml
+  - backend/deployment.yaml
+  - ui/deployment.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: gateway renders only the config volume by default
+    template: gateway/deployment.yaml
+    asserts:
+      - equal:
+          path: spec.template.spec.volumes
+          value:
+            - name: gateway-config
+              configMap:
+                name: RELEASE-NAME-litellm-gateway-config
+      - equal:
+          path: spec.template.spec.containers[0].volumeMounts
+          value:
+            - name: gateway-config
+              mountPath: /app/config/config.yaml
+              subPath: config.yaml
+
+  - it: gateway merges user volumes and volumeMounts with the config volume
+    template: gateway/deployment.yaml
+    set:
+      gateway.volumes:
+        - name: custom-callbacks
+          configMap:
+            name: custom-callbacks
+      gateway.volumeMounts:
+        - name: custom-callbacks
+          mountPath: /app/custom_callbacks.py
+          subPath: custom_callbacks.py
+    asserts:
+      - equal:
+          path: spec.template.spec.volumes[0].name
+          value: gateway-config
+      - equal:
+          path: spec.template.spec.volumes[1]
+          value:
+            name: custom-callbacks
+            configMap:
+              name: custom-callbacks
+      - equal:
+          path: spec.template.spec.containers[0].volumeMounts[0].name
+          value: gateway-config
+      - equal:
+          path: spec.template.spec.containers[0].volumeMounts[1]
+          value:
+            name: custom-callbacks
+            mountPath: /app/custom_callbacks.py
+            subPath: custom_callbacks.py
+
+  - it: gateway renders user volumes even when config creation is disabled
+    template: gateway/deployment.yaml
+    set:
+      gateway.config.create: false
+      gateway.volumes:
+        - name: certs
+          secret:
+            secretName: tls-certs
+      gateway.volumeMounts:
+        - name: certs
+          mountPath: /etc/certs
+          readOnly: true
+    asserts:
+      - equal:
+          path: spec.template.spec.volumes
+          value:
+            - name: certs
+              secret:
+                secretName: tls-certs
+      - equal:
+          path: spec.template.spec.containers[0].volumeMounts
+          value:
+            - name: certs
+              mountPath: /etc/certs
+              readOnly: true
+
+  - it: gateway omits volumes when config creation is disabled and no user volumes are set
+    template: gateway/deployment.yaml
+    set:
+      gateway.config.create: false
+    asserts:
+      - isNull:
+          path: spec.template.spec.volumes
+      - isNull:
+          path: spec.template.spec.containers[0].volumeMounts
+
+  - it: backend merges user volumes and volumeMounts with the shared config volume
+    template: backend/deployment.yaml
+    set:
+      backend.volumes:
+        - name: sso-handler
+          configMap:
+            name: sso-handler
+      backend.volumeMounts:
+        - name: sso-handler
+          mountPath: /app/custom_sso.py
+          subPath: custom_sso.py
+    asserts:
+      - equal:
+          path: spec.template.spec.volumes[0].name
+          value: gateway-config
+      - equal:
+          path: spec.template.spec.volumes[1]
+          value:
+            name: sso-handler
+            configMap:
+              name: sso-handler
+      - equal:
+          path: spec.template.spec.containers[0].volumeMounts[1]
+          value:
+            name: sso-handler
+            mountPath: /app/custom_sso.py
+            subPath: custom_sso.py
+
+  - it: backend renders user volumes even when config creation is disabled
+    template: backend/deployment.yaml
+    set:
+      gateway.config.create: false
+      backend.volumes:
+        - name: data
+          emptyDir: {}
+      backend.volumeMounts:
+        - name: data
+          mountPath: /data
+    asserts:
+      - equal:
+          path: spec.template.spec.volumes
+          value:
+            - name: data
+              emptyDir: {}
+      - equal:
+          path: spec.template.spec.containers[0].volumeMounts
+          value:
+            - name: data
+              mountPath: /data
+
+  - it: ui renders no volumes by default
+    template: ui/deployment.yaml
+    asserts:
+      - isNull:
+          path: spec.template.spec.volumes
+      - isNull:
+          path: spec.template.spec.containers[0].volumeMounts
+
+  - it: ui renders user volumes and volumeMounts
+    template: ui/deployment.yaml
+    set:
+      ui.volumes:
+        - name: nginx-config
+          configMap:
+            name: custom-nginx
+      ui.volumeMounts:
+        - name: nginx-config
+          mountPath: /etc/nginx/conf.d
+    asserts:
+      - equal:
+          path: spec.template.spec.volumes
+          value:
+            - name: nginx-config
+              configMap:
+                name: custom-nginx
+      - equal:
+          path: spec.template.spec.containers[0].volumeMounts
+          value:
+            - name: nginx-config
+              mountPath: /etc/nginx/conf.d

helm/litellm/tests/values/required.yaml

@@ -0,0 +1,4 @@
+database:
+  writer:
+    host: postgres.example.com
+    dbname: litellm

helm/litellm/values.yaml

@@ -124,6 +124,11 @@ gateway:
   extraEnv: [] # Add extra environment variables to the gateway
   envConfigMaps: [] # Add extra environment variables to the gateway from config maps
   envSecrets: [] # Add extra environment variables to the gateway from secrets
+  # Additional volumes on the gateway Deployment (e.g. a ConfigMap holding
+  # custom callback / SSO handler code, mounted next to the proxy config).
+  volumes: []
+  # Additional volumeMounts on the gateway container.
+  volumeMounts: []
   config:
     create: true
     proxy_config: {}
@@ -167,6 +172,10 @@ backend:
   extraEnv: []
   envConfigMaps: []
   envSecrets: []
+  # Additional volumes on the backend Deployment.
+  volumes: []
+  # Additional volumeMounts on the backend container.
+  volumeMounts: []
   image:
     repository: ghcr.io/berriai/litellm-backend
     tag: ""
@@ -206,6 +215,10 @@ ui:
   extraEnv: []
   envConfigMaps: []
   envSecrets: []
+  # Additional volumes on the ui Deployment.
+  volumes: []
+  # Additional volumeMounts on the ui container.
+  volumeMounts: []
   image:
     repository: ghcr.io/berriai/litellm-ui
     tag: ""

litellm-proxy-extras/litellm_proxy_extras/migrations/20260630120000_add_token_exchange_to_mcp_servers/migration.sql

@@ -0,0 +1,8 @@
+-- Timestamp sorts before some already-applied migrations; this is safe: the
+-- runner is `prisma migrate deploy`, which applies every pending migration
+-- regardless of name order (utils.py has an informational check for exactly
+-- this), and IF NOT EXISTS keeps a re-apply idempotent.
+-- AlterTable
+ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "token_exchange_endpoint" TEXT;
+ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "audience" TEXT;
+ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "subject_token_type" TEXT;

litellm-proxy-extras/litellm_proxy_extras/migrations/20260703120000_add_token_exchange_profile_to_mcp_servers/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "token_exchange_profile" TEXT;

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -329,6 +329,12 @@ model LiteLLM_MCPServerTable {
   token_url          String?
   registration_url   String?
   oauth2_flow        String?
+  token_exchange_endpoint String?
+  // Named for the RFC 8693 "audience" token-exchange request parameter (that flow only).
+  // RFC 8707 resource indicators are a separate concept, named "resource" in the v2 egress types.
+  audience           String?
+  subject_token_type String?
+  token_exchange_profile String?
   allow_all_keys Boolean @default(false)
   available_on_public_internet Boolean @default(true)
   delegate_auth_to_upstream Boolean @default(false)

litellm/__init__.py

@@ -379,6 +379,7 @@ budget_duration: Optional[str] = (
     None  # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
 )
 default_soft_budget: float = DEFAULT_SOFT_BUDGET  # by default all litellm proxy keys have a soft budget of 50.0
+budget_exceeded_throttle_percentage: Optional[float] = None
 forward_traceparent_to_llm_provider: bool = False
 
 

litellm/a2a_protocol/streaming_iterator.py

@@ -11,7 +11,6 @@ from litellm._logging import verbose_logger
 from litellm.a2a_protocol.cost_calculator import A2ACostCalculator
 from litellm.a2a_protocol.utils import A2ARequestUtils
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
-from litellm.litellm_core_utils.thread_pool_executor import executor
 
 if TYPE_CHECKING:
     from a2a.types import SendStreamingMessageRequest, SendStreamingMessageResponse
@@ -128,22 +127,15 @@ class A2AStreamingIterator:
 
             # Call success handlers - they will build standard_logging_object
             asyncio.create_task(
-                self.logging_obj.async_success_handler(
-                    result=result,
+                self.logging_obj.dispatch_success_handlers(
+                    result,
                     start_time=self.start_time,
                     end_time=end_time,
                     cache_hit=None,
+                    prefer_async_handlers=True,
                 )
             )
 
-            executor.submit(
-                self.logging_obj.success_handler,
-                result=result,
-                cache_hit=None,
-                start_time=self.start_time,
-                end_time=end_time,
-            )
-
             verbose_logger.info(
                 f"A2A streaming completed: prompt_tokens={prompt_tokens}, "
                 f"completion_tokens={completion_tokens}, total_tokens={total_tokens}, "

litellm/batches/batch_utils.py

@@ -3,6 +3,7 @@ from typing import Any, Iterator, List, Literal, Optional, Tuple
 
 import litellm
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.llm_cost_calc.utils import _parse_prompt_tokens_details
 from litellm.types.llms.openai import Batch
 from litellm.types.utils import CallTypes, ModelInfo, Usage
 from litellm.utils import token_counter
@@ -34,7 +35,7 @@ async def calculate_batch_cost_and_usage(
         custom_llm_provider=custom_llm_provider,
         model_name=model_name,
     )
-    batch_models = _get_batch_models_from_file_content(file_content_dictionary, model_name)
+    batch_models = _get_batch_models_from_file_content(file_content_dictionary, model_name, custom_llm_provider)
 
     return batch_cost, batch_usage, batch_models
 
@@ -70,7 +71,7 @@ async def _handle_completed_batch(
         model_name=model_name,
     )
 
-    batch_models = _get_batch_models_from_file_content(file_content_dictionary, model_name)
+    batch_models = _get_batch_models_from_file_content(file_content_dictionary, model_name, custom_llm_provider)
 
     return batch_cost, batch_usage, batch_models
 
@@ -78,6 +79,7 @@ async def _handle_completed_batch(
 def _get_batch_models_from_file_content(
     file_content_dictionary: List[dict],
     model_name: Optional[str] = None,
+    custom_llm_provider: str = "openai",
 ) -> List[str]:
     """
     Get the models from the file content
@@ -86,8 +88,8 @@ def _get_batch_models_from_file_content(
         return [model_name]
     batch_models = []
     for _item in file_content_dictionary:
-        if _batch_response_was_successful(_item):
-            _response_body = _get_response_from_batch_job_output_file(_item)
+        if _batch_response_was_successful(_item, custom_llm_provider):
+            _response_body = _get_response_from_batch_job_output_file(_item, custom_llm_provider)
             _model = _response_body.get("model")
             if _model:
                 batch_models.append(_model)
@@ -373,10 +375,10 @@ def _get_batch_job_cost_from_file_content(
         # parse the file content as json
         verbose_logger.debug("file_content_dictionary=%s", json.dumps(file_content_dictionary, indent=4))
         for _item in file_content_dictionary:
-            if _batch_response_was_successful(_item):
-                _response_body = _get_response_from_batch_job_output_file(_item)
-                if model_info is not None:
-                    usage = _get_batch_job_usage_from_response_body(_response_body)
+            if _batch_response_was_successful(_item, custom_llm_provider):
+                _response_body = _get_response_from_batch_job_output_file(_item, custom_llm_provider)
+                if model_info is not None or custom_llm_provider == "anthropic":
+                    usage = _get_batch_job_usage_from_response_body(_response_body, custom_llm_provider)
                     model = _response_body.get("model", "")
                     prompt_cost, completion_cost = batch_cost_calculator(
                         usage=usage,
@@ -418,17 +420,31 @@ def _get_batch_job_total_usage_from_file_content(
     total_tokens: int = 0
     prompt_tokens: int = 0
     completion_tokens: int = 0
+    cache_read_tokens: int = 0
+    cache_creation_tokens: int = 0
     for _item in file_content_dictionary:
-        if _batch_response_was_successful(_item):
-            _response_body = _get_response_from_batch_job_output_file(_item)
-            usage: Usage = _get_batch_job_usage_from_response_body(_response_body)
+        if _batch_response_was_successful(_item, custom_llm_provider):
+            _response_body = _get_response_from_batch_job_output_file(_item, custom_llm_provider)
+            usage: Usage = _get_batch_job_usage_from_response_body(_response_body, custom_llm_provider)
             total_tokens += usage.total_tokens
             prompt_tokens += usage.prompt_tokens
             completion_tokens += usage.completion_tokens
+            prompt_details = _parse_prompt_tokens_details(usage)
+            cache_read_tokens += prompt_details["cache_hit_tokens"]
+            cache_creation_tokens += prompt_details["cache_creation_tokens"]
+    cache_token_params = {
+        key: tokens
+        for key, tokens in (
+            ("cache_read_input_tokens", cache_read_tokens),
+            ("cache_creation_input_tokens", cache_creation_tokens),
+        )
+        if tokens > 0
+    }
     return Usage(
         total_tokens=total_tokens,
         prompt_tokens=prompt_tokens,
         completion_tokens=completion_tokens,
+        **cache_token_params,
     )
 
 
@@ -465,27 +481,51 @@ def _count_prompt_or_input_tokens(model: str, value: Any) -> int:
     return 0
 
 
-def _get_batch_job_usage_from_response_body(response_body: dict) -> Usage:
+def _get_batch_job_usage_from_response_body(response_body: dict, custom_llm_provider: str = "openai") -> Usage:
     """
     Get the tokens of a batch job from the response body
     """
+    if custom_llm_provider == "anthropic":
+        from litellm.llms.anthropic.chat.transformation import AnthropicConfig
+
+        return AnthropicConfig().calculate_usage(
+            usage_object=response_body.get("usage", None) or {},
+            reasoning_content=None,
+        )
     _usage_dict = response_body.get("usage", None) or {}
     usage: Usage = Usage(**_usage_dict)
     return usage
 
 
-def _get_response_from_batch_job_output_file(batch_job_output_file: dict) -> Any:
+def _get_anthropic_result_from_batch_results_line(batch_results_line: dict) -> dict:
+    """
+    Get the ``result`` object from a line of an Anthropic message batch results JSONL file.
+
+    Anthropic batch results lines look like:
+    ``{"custom_id": ..., "result": {"type": "succeeded", "message": {..., "usage": {...}}}}``
+    """
+    return batch_results_line.get("result", None) or {}
+
+
+def _get_response_from_batch_job_output_file(batch_job_output_file: dict, custom_llm_provider: str = "openai") -> Any:
     """
     Get the response from the batch job output file
     """
+    if custom_llm_provider == "anthropic":
+        return _get_anthropic_result_from_batch_results_line(batch_job_output_file).get("message", None) or {}
     _response: dict = batch_job_output_file.get("response", None) or {}
     _response_body = _response.get("body", None) or {}
     return _response_body
 
 
-def _batch_response_was_successful(batch_job_output_file: dict) -> bool:
+def _batch_response_was_successful(batch_job_output_file: dict, custom_llm_provider: str = "openai") -> bool:
     """
-    Check if the batch job response status == 200
+    Check if the batch job response was successful
+
+    OpenAI-shaped output rows report ``response.status_code == 200``; Anthropic
+    message batch results lines report ``result.type == "succeeded"``.
     """
+    if custom_llm_provider == "anthropic":
+        return _get_anthropic_result_from_batch_results_line(batch_job_output_file).get("type") == "succeeded"
     _response: dict = batch_job_output_file.get("response", None) or {}
     return _response.get("status_code", None) == 200

litellm/caching/disk_cache.py

@@ -59,8 +59,9 @@ class DiskCache(BaseCache):
 
     def increment_cache(self, key, value: int, **kwargs) -> int:
         # get the value
-        init_value = self.get_cache(key=key) or 0
-        value = init_value + value  # type: ignore
+        cached_value = self.get_cache(key=key)
+        init_value = cached_value if isinstance(cached_value, int) else 0
+        value = init_value + value
         self.set_cache(key, value, **kwargs)
         return value
 
@@ -76,8 +77,9 @@ class DiskCache(BaseCache):
 
     async def async_increment(self, key, value: int, **kwargs) -> int:
         # get the value
-        init_value = await self.async_get_cache(key=key) or 0
-        value = init_value + value  # type: ignore
+        cached_value = await self.async_get_cache(key=key)
+        init_value = cached_value if isinstance(cached_value, int) else 0
+        value = init_value + value
         await self.async_set_cache(key, value, **kwargs)
         return value
 

litellm/caching/valkey_semantic_cache.py

@@ -279,7 +279,7 @@ class ValkeySemanticCache(RedisSemanticCache):
                 print_verbose("No prompt provided for semantic caching")
                 return
 
-            embedding = await self._get_async_embedding(prompt, **kwargs)
+            embedding = await self._get_async_embedding(prompt, metadata=kwargs.get("metadata"))
             await self._ensure_index_async(len(embedding))
 
             doc_key = self._doc_key(key)
@@ -298,7 +298,7 @@ class ValkeySemanticCache(RedisSemanticCache):
                 kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
                 return None
 
-            embedding = await self._get_async_embedding(prompt, **kwargs)
+            embedding = await self._get_async_embedding(prompt, metadata=kwargs.get("metadata"))
             await self._ensure_index_async(len(embedding))
 
             search_result = await self.async_client.ft(self.index_name).search(

litellm/constants.py

@@ -1504,6 +1504,7 @@ LITELLM_SETTINGS_SAFE_DB_OVERRIDES = [
     "public_model_groups_links",
     "cost_discount_config",
     "cost_margin_config",
+    "budget_exceeded_throttle_percentage",
 ]
 SPECIAL_LITELLM_AUTH_TOKEN = ["ui-token"]
 DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL = int(os.getenv("DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL", 60))

litellm/cost_calculator.py

@@ -222,7 +222,7 @@ def _cost_per_token_custom_pricing_helper(
         output_cost = completion_tokens * output_cost_per_token
         return input_cost, output_cost
     elif custom_cost_per_second is not None:
-        output_cost = custom_cost_per_second * response_time_ms / 1000  # type: ignore
+        output_cost = custom_cost_per_second * (response_time_ms or 0.0) / 1000
         return 0, output_cost
 
     return None
@@ -662,29 +662,27 @@ def cost_per_token(
                 data_residency=data_residency,
             )
 
-        if model_info.get("input_cost_per_second", None) is not None and response_time_ms is not None:
+        input_cost_per_second = model_info.get("input_cost_per_second")
+        if input_cost_per_second is not None and response_time_ms is not None:
             verbose_logger.debug(
                 "For model=%s - input_cost_per_second: %s; response time: %s",
                 model,
-                model_info.get("input_cost_per_second", None),
+                input_cost_per_second,
                 response_time_ms,
             )
             ## COST PER SECOND ##
-            prompt_tokens_cost_usd_dollar = (
-                model_info["input_cost_per_second"] * response_time_ms / 1000  # type: ignore
-            )
+            prompt_tokens_cost_usd_dollar = input_cost_per_second * response_time_ms / 1000
 
-        if model_info.get("output_cost_per_second", None) is not None and response_time_ms is not None:
+        output_cost_per_second = model_info.get("output_cost_per_second")
+        if output_cost_per_second is not None and response_time_ms is not None:
             verbose_logger.debug(
                 "For model=%s - output_cost_per_second: %s; response time: %s",
                 model,
-                model_info.get("output_cost_per_second", None),
+                output_cost_per_second,
                 response_time_ms,
             )
             ## COST PER SECOND ##
-            completion_tokens_cost_usd_dollar = (
-                model_info["output_cost_per_second"] * response_time_ms / 1000  # type: ignore
-            )
+            completion_tokens_cost_usd_dollar = output_cost_per_second * response_time_ms / 1000
 
         verbose_logger.debug(
             "Returned custom cost for model=%s - prompt_tokens_cost_usd_dollar: %s, completion_tokens_cost_usd_dollar: %s",
@@ -2157,17 +2155,23 @@ def batch_cost_calculator(
     if input_cost_per_token_batches:
         total_prompt_cost = usage.prompt_tokens * input_cost_per_token_batches
     elif input_cost_per_token:
+        details = _parse_prompt_tokens_details(usage)
+        cache_read_tokens = details["cache_hit_tokens"]
+        cache_creation_tokens = details["cache_creation_tokens"]
+
         # Subtract cached tokens from prompt_tokens before calculating cost
         # Fixes issue where cached tokens are being charged again
+        base_input_tokens = get_billable_input_tokens(usage) - cache_creation_tokens
         total_prompt_cost = (
-            get_billable_input_tokens(usage) * (input_cost_per_token) / 2
+            base_input_tokens * (input_cost_per_token) / 2
         )  # batch cost is usually half of the regular token cost
 
         # Add cache read cost if applicable
-        details = _parse_prompt_tokens_details(usage)
-        cache_read_tokens = details["cache_hit_tokens"]
         cache_read_cost_key = _get_service_tier_cost_key("cache_read_input_token_cost", None)
         total_prompt_cost += calculate_cost_component(model_info, cache_read_cost_key, cache_read_tokens) / 2
+
+        cache_creation_cost = model_info.get("cache_creation_input_token_cost") or input_cost_per_token
+        total_prompt_cost += cache_creation_tokens * cache_creation_cost / 2
     if output_cost_per_token_batches:
         total_completion_cost = usage.completion_tokens * output_cost_per_token_batches
     elif output_cost_per_token:

litellm/fine_tuning/main.py

@@ -256,8 +256,6 @@ def create_fine_tuning_job(
             extra_body = optional_params.get("extra_body", {})
             if extra_body is not None:
                 extra_body.pop("azure_ad_token", None)
-            else:
-                get_secret_str("AZURE_AD_TOKEN")  # type: ignore
 
             # Prepare Azure-specific parameters for extra_body
             extra_body = _prepare_azure_extra_body(extra_body, kwargs, azure_specific_hyperparams)
@@ -442,7 +440,7 @@ def cancel_fine_tuning_job(
             )
         # Azure OpenAI
         elif custom_llm_provider == "azure":
-            api_base = optional_params.api_base or litellm.api_base or get_secret("AZURE_API_BASE")  # type: ignore
+            api_base = optional_params.api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
 
             api_version = optional_params.api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION")  # type: ignore
 
@@ -457,8 +455,6 @@ def cancel_fine_tuning_job(
             extra_body = optional_params.get("extra_body", {})
             if extra_body is not None:
                 extra_body.pop("azure_ad_token", None)
-            else:
-                get_secret_str("AZURE_AD_TOKEN")  # type: ignore
 
             response = azure_fine_tuning_apis_instance.cancel_fine_tuning_job(
                 api_base=api_base,
@@ -616,8 +612,6 @@ def list_fine_tuning_jobs(
             extra_body = optional_params.get("extra_body", {})
             if extra_body is not None:
                 extra_body.pop("azure_ad_token", None)
-            else:
-                get_secret("AZURE_AD_TOKEN")  # type: ignore
 
             response = azure_fine_tuning_apis_instance.list_fine_tuning_jobs(
                 api_base=api_base,
@@ -759,8 +753,6 @@ def retrieve_fine_tuning_job(
             extra_body = optional_params.get("extra_body", {})
             if extra_body is not None:
                 extra_body.pop("azure_ad_token", None)
-            else:
-                get_secret_str("AZURE_AD_TOKEN")  # type: ignore
 
             response = azure_fine_tuning_apis_instance.retrieve_fine_tuning_job(
                 api_base=api_base,

litellm/integrations/datadog/datadog.py

@@ -354,14 +354,14 @@ class DataDogLogger(
         Raises:
             Raises a NON Blocking verbose_logger.exception if an error occurs
         """
+        if not self.log_queue:
+            verbose_logger.exception("Datadog: log_queue does not exist")
+            return
+
+        batch_to_send = self.log_queue[:]
+        self.log_queue = []
+
         try:
-            if not self.log_queue:
-                verbose_logger.exception("Datadog: log_queue does not exist")
-                return
-
-            batch_to_send = self.log_queue[:]
-            self.log_queue = []
-
             verbose_logger.debug(
                 "Datadog - about to flush %s events on %s",
                 len(batch_to_send),

litellm/integrations/otel/logger.py

@@ -368,7 +368,11 @@ class OpenTelemetryV2(CustomLogger):
                 # it (named provisionally) so it isn't leaked as an open span.
                 carrier.span.end(end_time=to_ns(end_time))
             return None
-        data = LLMCallSpanData.from_standard_logging_payload(payload, capture_content=self.config.capture_span_content)
+        data = LLMCallSpanData.from_standard_logging_payload(
+            payload,
+            capture_content=self.config.capture_span_content,
+            time_to_first_chunk_seconds=call.time_to_first_chunk_seconds,
+        )
         end_time_ns = to_ns(end_time)
         if carrier.span is not None:
             # Born at the boundary: stamp attributes from the typed payload, set

litellm/integrations/otel/mappers/genai.py

@@ -55,6 +55,7 @@ class GenAIMapper:
         GenAI.RESPONSE_MODEL: lambda d: d.response_model,
         GenAI.RESPONSE_ID: lambda d: d.response_id,
         GenAI.RESPONSE_FINISH_REASONS: lambda d: list(d.finish_reasons) if d.finish_reasons else None,
+        GenAI.RESPONSE_TIME_TO_FIRST_CHUNK: lambda d: d.time_to_first_chunk_seconds,
         GenAI.USAGE_INPUT_TOKENS: lambda d: d.usage.input_tokens,
         GenAI.USAGE_OUTPUT_TOKENS: lambda d: d.usage.output_tokens,
         Error.TYPE: lambda d: d.error.error_type if d.error else None,

litellm/integrations/otel/model/metadata.py

@@ -41,7 +41,7 @@ from typing import TYPE_CHECKING, Any, Mapping, cast
 
 from litellm.constants import LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL
 from litellm.integrations.otel.model.semconv import resolve_operation
-from litellm.integrations.otel.model.utils import as_str
+from litellm.integrations.otel.model.utils import as_str, to_seconds
 
 if TYPE_CHECKING:
     from litellm.types.utils import StandardLoggingPayload
@@ -201,6 +201,7 @@ class LLMCallEvent:
     # span is renamed from the typed payload at close (``finish_span``); this only
     # needs to be reasonable for a span that never gets closed (a leak).
     provisional_span_name: str
+    time_to_first_chunk_seconds: float | None
 
     @classmethod
     def from_dict(cls, kwargs: Mapping[str, Any]) -> "LLMCallEvent":
@@ -214,9 +215,25 @@ class LLMCallEvent:
             dynamic_params=kwargs.get("standard_callback_dynamic_params"),
             is_no_upstream_call=bool(kwargs.get(LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL)),
             provisional_span_name=f"{operation.value} {model}".strip(),
+            time_to_first_chunk_seconds=time_to_first_chunk_seconds(kwargs),
         )
 
 
+def time_to_first_chunk_seconds(kwargs: Mapping[str, Any]) -> float | None:
+    """Seconds from the upstream request being issued (``api_call_start_time``)
+    to the first streamed chunk (``completion_start_time``); ``None`` for
+    non-streaming calls, where ``completion_start_time`` is backfilled with the
+    end time and would not measure first-chunk latency."""
+    optional_params = cast(Mapping[str, Any], kwargs.get("optional_params") or {})
+    if not optional_params.get("stream"):
+        return None
+    api_call_start = to_seconds(kwargs.get("api_call_start_time"))
+    completion_start = to_seconds(kwargs.get("completion_start_time"))
+    if api_call_start is None or completion_start is None:
+        return None
+    return completion_start - api_call_start
+
+
 def _call_id(payload: "StandardLoggingPayload | None", kwargs: Mapping[str, Any]) -> str | None:
     """The call id from the payload (when closed) or the bare kwargs (at pre_call)."""
     if payload is not None:

litellm/integrations/otel/model/payloads.py

@@ -305,10 +305,14 @@ class LLMCallSpanData:
     messages_in: tuple[Mapping[str, object], ...] = ()
     choices_out: tuple[Mapping[str, object], ...] = ()
     system_fingerprint: str | None = None
+    time_to_first_chunk_seconds: float | None = None
 
     @classmethod
     def from_standard_logging_payload(
-        cls, payload: "StandardLoggingPayload", capture_content: bool = False
+        cls,
+        payload: "StandardLoggingPayload",
+        capture_content: bool = False,
+        time_to_first_chunk_seconds: float | None = None,
     ) -> "LLMCallSpanData":
         params = cast(Mapping[str, object], payload.get("model_parameters") or {})
         # The single parse of the request's metadata — the request-vs-provider
@@ -349,6 +353,7 @@ class LLMCallSpanData:
             messages_in=_dicts(payload.get("messages")) if capture_content else (),
             choices_out=choices_out if capture_content else (),
             system_fingerprint=as_str(response.get("system_fingerprint")),
+            time_to_first_chunk_seconds=time_to_first_chunk_seconds,
         )
 
 

litellm/integrations/otel/model/semconv.py

@@ -69,6 +69,7 @@ class GenAI:
     RESPONSE_ID: Final = "gen_ai.response.id"
     RESPONSE_MODEL: Final = "gen_ai.response.model"
     RESPONSE_FINISH_REASONS: Final = "gen_ai.response.finish_reasons"
+    RESPONSE_TIME_TO_FIRST_CHUNK: Final = "gen_ai.response.time_to_first_chunk"
     # usage
     USAGE_INPUT_TOKENS: Final = "gen_ai.usage.input_tokens"
     USAGE_OUTPUT_TOKENS: Final = "gen_ai.usage.output_tokens"

litellm/integrations/otel/plumbing/metrics.py

@@ -21,6 +21,7 @@ from litellm.integrations.opentelemetry import (
     _build_metric_attribute_filter,
     _resolve_metric_attribute_filter,
 )
+from litellm.integrations.otel.model.metadata import time_to_first_chunk_seconds
 from litellm.integrations.otel.model.semconv import Metric, resolve_operation
 from litellm.integrations.otel.model.utils import to_seconds
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
@@ -181,13 +182,10 @@ class GenAIMetricRecorder:
         self._metrics.token_usage.record(usage.get("completion_tokens", 0), attributes=out_attrs)
 
     def _record_time_to_first_token(self, kwargs: Mapping[str, Any], common_attrs: dict) -> None:
-        if not kwargs.get("optional_params", {}).get("stream", False):
+        time_to_first_chunk = time_to_first_chunk_seconds(kwargs)
+        if time_to_first_chunk is None:
             return
-        api_call_start = to_seconds(kwargs.get("api_call_start_time"))
-        completion_start = to_seconds(kwargs.get("completion_start_time"))
-        if api_call_start is None or completion_start is None:
-            return
-        self._metrics.time_to_first_token.record(completion_start - api_call_start, attributes=common_attrs)
+        self._metrics.time_to_first_token.record(time_to_first_chunk, attributes=common_attrs)
 
     def _record_time_per_output_token(
         self,

litellm/integrations/websearch_interception/handler.py

@@ -91,6 +91,7 @@ class WebSearchInterceptionLogger(CustomLogger):
         messages: List[Dict],
         tools: Optional[List[Dict]],
         custom_llm_provider: Optional[str],
+        kwargs: Optional[dict[str, Any]] = None,
     ) -> Optional[Dict[str, Any]]:
         """
         Short-circuit web-search-only requests by executing the search directly.
@@ -176,7 +177,10 @@ class WebSearchInterceptionLogger(CustomLogger):
         # Execute search — keep the structured SearchResponse so the native
         # block can carry per-result url/title/page_age.
         try:
-            search_result_text, structured = await self._execute_search(query)
+            if kwargs is None:
+                search_result_text, structured = await self._execute_search(query)
+            else:
+                search_result_text, structured = await self._execute_search(query, kwargs=kwargs)
         except Exception as e:
             verbose_logger.error(f"WebSearchInterception: Short-circuit search failed: {e}")
             search_result_text, structured = f"Search failed: {e}", None
@@ -936,7 +940,7 @@ class WebSearchInterceptionLogger(CustomLogger):
             query = tool_call["input"].get("query")
             if query:
                 verbose_logger.debug(f"WebSearchInterception: Queuing search for query='{query}'")
-                search_tasks.append(self._execute_search(query))
+                search_tasks.append(self._execute_search(query, kwargs=kwargs))
             else:
                 verbose_logger.debug(f"WebSearchInterception: Tool call {tool_call['id']} has no query")
                 # Add empty result for tools without query
@@ -1009,7 +1013,9 @@ class WebSearchInterceptionLogger(CustomLogger):
         )
         return patch, structured_results
 
-    async def _execute_search(self, query: str) -> Tuple[str, Optional[SearchResponse]]:
+    async def _execute_search(
+        self, query: str, kwargs: Optional[dict[str, Any]] = None
+    ) -> Tuple[str, Optional[SearchResponse]]:
         """
         Execute a single web search using router's search tools.
 
@@ -1031,36 +1037,13 @@ class WebSearchInterceptionLogger(CustomLogger):
                 )
                 llm_router = None
 
-            # Determine search provider from router's search_tools
+            search_tool = self._select_search_tool_from_router(llm_router=llm_router)
             search_provider: Optional[str] = None
-            if llm_router is not None and hasattr(llm_router, "search_tools"):
-                if self.search_tool_name:
-                    # Find specific search tool by name
-                    matching_tools = [
-                        tool
-                        for tool in llm_router.search_tools
-                        if tool.get("search_tool_name") == self.search_tool_name
-                    ]
-                    if matching_tools:
-                        search_tool = matching_tools[0]
-                        search_provider = search_tool.get("litellm_params", {}).get("search_provider")
-                        verbose_logger.debug(
-                            f"WebSearchInterception: Found search tool '{self.search_tool_name}' "
-                            f"with provider '{search_provider}'"
-                        )
-                    else:
-                        verbose_logger.debug(
-                            f"WebSearchInterception: Search tool '{self.search_tool_name}' not found in router, "
-                            "falling back to first available or perplexity"
-                        )
-
-                # If no specific tool or not found, use first available
-                if not search_provider and llm_router.search_tools:
-                    first_tool = llm_router.search_tools[0]
-                    search_provider = first_tool.get("litellm_params", {}).get("search_provider")
-                    verbose_logger.debug(
-                        f"WebSearchInterception: Using first available search tool with provider '{search_provider}'"
-                    )
+            search_litellm_params: dict[str, Any] = {}
+            if search_tool is not None:
+                await self._authorize_search_tool(search_tool=search_tool, kwargs=kwargs)
+                search_litellm_params = dict(search_tool.get("litellm_params", {}) or {})
+                search_provider = search_litellm_params.get("search_provider")
 
             # Fallback to perplexity if no router or no search tools configured
             if not search_provider:
@@ -1073,7 +1056,12 @@ class WebSearchInterceptionLogger(CustomLogger):
             verbose_logger.debug(
                 f"WebSearchInterception: Executing search for '{query}' using provider '{search_provider}'"
             )
-            result = await litellm.asearch(query=query, search_provider=search_provider)
+            search_kwargs = {
+                key: value
+                for key, value in search_litellm_params.items()
+                if key != "search_provider" and value is not None
+            }
+            result = await litellm.asearch(query=query, search_provider=search_provider, **search_kwargs)
 
             # Format using transformation function
             search_result_text = WebSearchTransformation.format_search_response(result)
@@ -1086,6 +1074,107 @@ class WebSearchInterceptionLogger(CustomLogger):
             verbose_logger.error(f"WebSearchInterception: Search failed for '{query}': {str(e)}")
             raise
 
+    async def _authorize_search_tool(
+        self,
+        search_tool: dict[str, Any],
+        kwargs: Optional[dict[str, Any]],
+    ) -> None:
+        search_tool_name = search_tool.get("search_tool_name")
+        if not isinstance(search_tool_name, str) or not search_tool_name:
+            return
+
+        user_api_key_auth = self._get_user_api_key_auth_from_kwargs(kwargs)
+        if user_api_key_auth is None:
+            return
+
+        from litellm.proxy.auth.auth_checks import (
+            can_key_call_search_tool,
+            can_team_call_search_tool,
+            get_team_object,
+        )
+
+        await can_key_call_search_tool(
+            search_tool_name=search_tool_name,
+            valid_token=user_api_key_auth,
+        )
+
+        team_id = getattr(user_api_key_auth, "team_id", None)
+        if team_id:
+            from litellm.proxy.proxy_server import (
+                prisma_client,
+                proxy_logging_obj,
+                user_api_key_cache,
+            )
+
+            team_object = await get_team_object(
+                team_id=team_id,
+                prisma_client=prisma_client,
+                user_api_key_cache=user_api_key_cache,
+                parent_otel_span=getattr(user_api_key_auth, "parent_otel_span", None),
+                proxy_logging_obj=proxy_logging_obj,
+            )
+            await can_team_call_search_tool(
+                search_tool_name=search_tool_name,
+                team_object=team_object,
+            )
+
+    @staticmethod
+    def _get_user_api_key_auth_from_kwargs(kwargs: Optional[dict[str, Any]]) -> Any:
+        if not kwargs:
+            return None
+
+        for metadata_key in ("metadata", "litellm_metadata"):
+            metadata = kwargs.get(metadata_key)
+            if isinstance(metadata, dict) and metadata.get("user_api_key_auth") is not None:
+                return metadata["user_api_key_auth"]
+
+        litellm_params = kwargs.get("litellm_params")
+        if not isinstance(litellm_params, dict):
+            return None
+
+        for metadata_key in ("metadata", "litellm_metadata"):
+            metadata = litellm_params.get(metadata_key)
+            if isinstance(metadata, dict) and metadata.get("user_api_key_auth") is not None:
+                return metadata["user_api_key_auth"]
+
+        return None
+
+    def _select_search_tool_from_router(self, llm_router: Any) -> Optional[dict[str, Any]]:
+        if llm_router is None or not hasattr(llm_router, "search_tools"):
+            return None
+        search_tools = list(getattr(llm_router, "search_tools") or [])
+        return self._select_search_tool_from_list(search_tools=search_tools, source="router")
+
+    def _select_search_tool_from_list(
+        self,
+        search_tools: list[dict[str, Any]],
+        source: str,
+    ) -> Optional[dict[str, Any]]:
+        if self.search_tool_name:
+            matching_tools = [tool for tool in search_tools if tool.get("search_tool_name") == self.search_tool_name]
+            if matching_tools:
+                search_provider = (matching_tools[0].get("litellm_params", {}) or {}).get("search_provider")
+                verbose_logger.debug(
+                    f"WebSearchInterception: Found search tool '{self.search_tool_name}' "
+                    f"from {source} with provider '{search_provider}'"
+                )
+                return matching_tools[0]
+            verbose_logger.debug(
+                f"WebSearchInterception: Search tool '{self.search_tool_name}' not found in {source}, "
+                "falling back to first available or perplexity"
+            )
+
+        if search_tools:
+            first_tool = search_tools[0]
+            search_provider = (first_tool.get("litellm_params", {}) or {}).get("search_provider")
+            verbose_logger.debug(
+                f"WebSearchInterception: Using first available search tool from {source} "
+                f"with provider '{search_provider}'"
+            )
+            return first_tool
+
+        return None
+
     async def _execute_chat_completion_agentic_loop(
         self,
         model: str,
@@ -1145,7 +1234,7 @@ class WebSearchInterceptionLogger(CustomLogger):
 
             if query:
                 verbose_logger.debug(f"WebSearchInterception: Queuing search for query='{query}'")
-                search_tasks.append(self._execute_search(query))
+                search_tasks.append(self._execute_search(query, kwargs=kwargs))
             else:
                 verbose_logger.debug(f"WebSearchInterception: Tool call {tool_call.get('id')} has no query")
                 # Add empty result for tools without query

litellm/interactions/streaming_iterator.py

@@ -174,22 +174,15 @@ class InteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator):
         logging_response = copy.deepcopy(self.completed_response)
 
         asyncio.create_task(
-            self.logging_obj.async_success_handler(
-                result=logging_response,
+            self.logging_obj.dispatch_success_handlers(
+                logging_response,
                 start_time=self.start_time,
                 end_time=datetime.now(),
                 cache_hit=None,
+                prefer_async_handlers=True,
             )
         )
 
-        executor.submit(
-            self.logging_obj.success_handler,
-            result=logging_response,
-            cache_hit=None,
-            start_time=self.start_time,
-            end_time=datetime.now(),
-        )
-
 
 class SyncInteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator):
     """

litellm/litellm_core_utils/audio_utils/utils.py

@@ -123,6 +123,34 @@ def process_audio_file(audio_file: FileTypes) -> ProcessedAudioFile:
     return ProcessedAudioFile(file_content=file_content, filename=filename, content_type=content_type)
 
 
+BARE_ISO_639_1_TO_BCP47 = {
+    "en": "en-US",
+    "es": "es-ES",
+    "de": "de-DE",
+    "fr": "fr-FR",
+    "it": "it-IT",
+    "pt": "pt-BR",
+    "ja": "ja-JP",
+    "ko": "ko-KR",
+    "zh": "zh-CN",
+    "ru": "ru-RU",
+    "hi": "hi-IN",
+    "ar": "ar-SA",
+}
+
+
+def normalize_transcription_language_to_bcp47(language: str) -> str:
+    """
+    OpenAI's transcription `language` param accepts bare ISO-639-1 codes like
+    ``en``; speech APIs such as Google Speech-to-Text and NVIDIA Riva require
+    BCP-47 like ``en-US``. Map the most common bare codes and pass through
+    anything already region-qualified (or unknown, for a clear provider error).
+    """
+    if "-" in language:
+        return language
+    return BARE_ISO_639_1_TO_BCP47.get(language.lower(), language)
+
+
 def get_audio_file_name(file_obj: FileTypes) -> str:
     """
     Safely get the name of a file-like object or return its string representation.

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -1944,7 +1944,7 @@ def _map_azure_exception(
             response=getattr(original_exception, "response", None),
             body=getattr(original_exception, "body", None),
         )
-    elif "invalid_request_error" in error_str:
+    elif "invalid_request_error" in error_str and getattr(original_exception, "status_code", None) in (None, 400):
         raise BadRequestError(
             message=f"AzureException BadRequestError - {message}",
             llm_provider="azure",
@@ -1986,6 +1986,14 @@ def _map_azure_exception(
                 litellm_debug_info=extra_information,
                 response=getattr(original_exception, "response", None),
             )
+        elif original_exception.status_code == 404:
+            raise NotFoundError(
+                message=f"AzureException NotFoundError - {message}",
+                llm_provider="azure",
+                model=model,
+                litellm_debug_info=extra_information,
+                response=getattr(original_exception, "response", None),
+            )
         elif original_exception.status_code == 408:
             raise Timeout(
                 message=f"AzureException Timeout - {message}",
@@ -2173,7 +2181,7 @@ def exception_type(  # type: ignore
     litellm_response_headers = _get_response_headers(original_exception=original_exception)
     try:
         error_str = redact_string(str(original_exception)) if _ENABLE_SECRET_REDACTION else str(original_exception)
-        if model:
+        if model or custom_llm_provider:
             if hasattr(original_exception, "message"):
                 error_str = (
                     redact_string(str(original_exception.message))

litellm/litellm_core_utils/get_litellm_params.py

@@ -36,6 +36,8 @@ OPTIONAL_KWARGS_KEYS = frozenset(
         "aws_bedrock_project_id",
         "tpm",
         "rpm",
+        "itpm",
+        "otpm",
         "use_xai_oauth",
     }
 )
@@ -74,6 +76,7 @@ def get_litellm_params(
     proxy_server_request=None,
     acompletion=None,
     aembedding=None,
+    allm_passthrough_route=None,
     preset_cache_key=None,
     no_log=None,
     input_cost_per_second=None,
@@ -116,6 +119,7 @@ def get_litellm_params(
     # Build base dict with explicit parameters (always included)
     litellm_params = {
         "acompletion": acompletion,
+        "allm_passthrough_route": allm_passthrough_route,
         "api_key": api_key,
         "force_timeout": force_timeout,
         "logger_fn": logger_fn,

litellm/litellm_core_utils/litellm_logging.py

@@ -1530,6 +1530,7 @@ class Logging(LiteLLMLoggingBaseClass):
             and litellm_params.get(CallTypes.aembedding.value, False) is not True
             and litellm_params.get(CallTypes.aimage_generation.value, False) is not True
             and litellm_params.get(CallTypes.atranscription.value, False) is not True
+            and litellm_params.get(CallTypes.allm_passthrough_route.value, False) is not True
         )
 
     def _is_assembled_stream_success(self, result=None) -> bool:

litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py

@@ -656,7 +656,7 @@ def convert_to_model_response_object(
 
                 message: Optional[Message] = None
                 finish_reason: Optional[str] = None
-                if _should_convert_tool_call_to_json_mode(
+                if tool_calls is not None and _should_convert_tool_call_to_json_mode(
                     tool_calls=tool_calls,
                     convert_tool_call_to_json_mode=convert_tool_call_to_json_mode,
                 ):

litellm/litellm_core_utils/realtime_streaming.py

@@ -1,5 +1,4 @@
 import asyncio
-import concurrent.futures
 import json
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Protocol, Union, cast
 
@@ -25,9 +24,6 @@ if TYPE_CHECKING:
 else:
     CLIENT_CONNECTION_CLASS = Any
 
-# Create a thread pool with a maximum of 10 threads
-executor = concurrent.futures.ThreadPoolExecutor(max_workers=10)
-
 
 class RealtimeEventNormalizer(Protocol):
     def should_drop(self, event: object) -> bool: ...
@@ -315,13 +311,12 @@ class RealTimeStreaming:
             if self.session_tools or self.tool_calls:
                 self.logging_obj.model_call_details["realtime_tools"] = self.session_tools
                 self.logging_obj.model_call_details["realtime_tool_calls"] = self.tool_calls
-            ## ASYNC LOGGING
             # Route through the bounded logging worker (per-coroutine timeout +
             # concurrency cap) instead of a bare create_task, so a slow callback
             # can't leave suspended tasks pinning each call's response in memory.
-            GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(self.logging_obj.async_success_handler(self.messages))
-            ## SYNC LOGGING
-            executor.submit(self.logging_obj.success_handler(self.messages))
+            GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
+                self.logging_obj.dispatch_success_handlers(self.messages, prefer_async_handlers=True)
+            )
 
     async def _send_to_backend(self, message: str) -> bool:
         """Send a message to the backend WebSocket.

litellm/litellm_core_utils/streaming_handler.py

@@ -1884,7 +1884,7 @@ class CustomStreamWrapper:
                 await self.fetch_stream()
 
             if is_async_iterable(self.completion_stream):
-                async for chunk in self.completion_stream:  # type: ignore[union-attr]
+                async for chunk in self.completion_stream:  # pyright: ignore[reportOptionalIterable]  # is_async_iterable guard proves __aiter__
                     if chunk == "None" or chunk is None:
                         continue  # skip None chunks
 

litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py

@@ -193,6 +193,14 @@ class AgenticAnthropicStreamingIterator:
 
         raise StopAsyncIteration
 
+    async def aclose(self) -> None:
+        from litellm.llms.anthropic.experimental_pass_through.messages.streaming_iterator import (
+            aclose_if_supported,
+        )
+
+        await aclose_if_supported(self._inner)
+        await aclose_if_supported(self._follow_up_iterator)
+
     async def _process_agentic_hooks(self) -> None:
         """Rebuild the Anthropic response from collected SSE bytes and call hooks."""
         if self._hook_processing_done:

litellm/llms/anthropic/experimental_pass_through/messages/handler.py

@@ -148,6 +148,7 @@ async def _try_websearch_short_circuit(
     tools: Optional[List[Dict]],
     custom_llm_provider: Optional[str],
     stream: Optional[bool],
+    kwargs: Optional[dict] = None,
 ) -> Optional[Union[AnthropicMessagesResponse, AsyncIterator]]:
     """
     Attempt to short-circuit a web-search-only request.
@@ -177,6 +178,7 @@ async def _try_websearch_short_circuit(
             messages=messages,
             tools=tools,
             custom_llm_provider=custom_llm_provider,
+            kwargs=kwargs,
         )
         if response is not None:
             anthropic_response = cast(AnthropicMessagesResponse, response)
@@ -292,6 +294,7 @@ async def anthropic_messages(
         tools=tools,
         custom_llm_provider=custom_llm_provider,
         stream=original_stream,
+        kwargs={**kwargs, "metadata": metadata},
     )
     if short_circuit_response is not None:
         return short_circuit_response

litellm/llms/anthropic/experimental_pass_through/messages/streaming_iterator.py

@@ -1,8 +1,13 @@
 import asyncio
 import json
 from datetime import datetime
-from typing import Any, AsyncIterator, List, Union
+from typing import Any, AsyncIterator, List, Protocol, Union, runtime_checkable
 
+import httpx
+from pydantic import TypeAdapter
+from typing_extensions import TypedDict
+
+from litellm.litellm_core_utils.core_helpers import process_response_headers
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.proxy.pass_through_endpoints.success_handler import (
     PassThroughEndpointLogging,
@@ -12,6 +17,93 @@ from litellm.types.utils import GenericStreamingChunk, ModelResponseStream
 
 GLOBAL_PASS_THROUGH_SUCCESS_HANDLER_OBJ = PassThroughEndpointLogging()
 
+INCOMPLETE_STREAM_ERROR_MESSAGE = (
+    "Provider stream ended before emitting a message_stop event; "
+    "the response is incomplete and any partial content (e.g. tool_use input JSON) may be truncated."
+)
+
+
+def _is_message_stop_chunk(chunk: object) -> bool:
+    if isinstance(chunk, dict):
+        return chunk.get("type") == "message_stop"
+    if isinstance(chunk, (bytes, bytearray)):
+        return any(line == b"event: message_stop" for line in chunk.splitlines())
+    return False
+
+
+def _is_provider_error_chunk(chunk: object) -> bool:
+    if isinstance(chunk, dict):
+        return chunk.get("type") == "error"
+    if isinstance(chunk, (bytes, bytearray)):
+        return any(line == b"event: error" for line in chunk.splitlines())
+    return False
+
+
+def _is_terminal_stream_chunk(chunk: object) -> bool:
+    return _is_message_stop_chunk(chunk) or _is_provider_error_chunk(chunk)
+
+
+def _incomplete_stream_error_sse_event() -> bytes:
+    payload = json.dumps(
+        {
+            "type": "error",
+            "error": {"type": "api_error", "message": INCOMPLETE_STREAM_ERROR_MESSAGE},
+        }
+    )
+    return f"event: error\ndata: {payload}\n\n".encode()
+
+
+class AnthropicMessagesStreamHiddenParams(TypedDict):
+    additional_headers: dict[str, str]
+
+
+@runtime_checkable
+class SupportsAclose(Protocol):
+    async def aclose(self) -> None: ...
+
+
+async def aclose_if_supported(stream: object) -> None:
+    if isinstance(stream, SupportsAclose):
+        await stream.aclose()
+
+
+_RESPONSE_HEADERS_ADAPTER: TypeAdapter[dict[str, str]] = TypeAdapter(dict[str, str])
+
+
+def anthropic_messages_stream_hidden_params(
+    response_headers: httpx.Headers,
+) -> AnthropicMessagesStreamHiddenParams:
+    return AnthropicMessagesStreamHiddenParams(
+        additional_headers=_RESPONSE_HEADERS_ADAPTER.validate_python(process_response_headers(response_headers))
+    )
+
+
+class AnthropicMessagesStreamingResponse:
+    """
+    Wraps the /v1/messages SSE byte stream so upstream provider response
+    headers (e.g. Bedrock's x-amzn-requestid / x-amzn-trace-id) survive as
+    ``_hidden_params["additional_headers"]``, which the proxy forwards to
+    clients as ``llm_provider-*`` response headers. Bare async generators
+    cannot carry attributes, so header context was previously dropped.
+    """
+
+    def __init__(
+        self,
+        completion_stream: AsyncIterator[bytes],
+        hidden_params: AnthropicMessagesStreamHiddenParams,
+    ) -> None:
+        self.completion_stream = completion_stream
+        self._hidden_params = hidden_params
+
+    def __aiter__(self) -> "AnthropicMessagesStreamingResponse":
+        return self
+
+    async def __anext__(self) -> bytes:
+        return await self.completion_stream.__anext__()
+
+    async def aclose(self) -> None:
+        await aclose_if_supported(self.completion_stream)
+
 
 class BaseAnthropicMessagesStreamingIterator:
     """
@@ -102,13 +194,18 @@ class BaseAnthropicMessagesStreamingIterator:
         This method provides the common logic for both Anthropic and Bedrock implementations.
         """
         collected_chunks = []
+        saw_terminal_event = False
 
         async for chunk in completion_stream:
             if self.completion_start_time is None:
                 self.completion_start_time = datetime.now()
+            saw_terminal_event = saw_terminal_event or _is_terminal_stream_chunk(chunk)
             encoded_chunk = self._convert_chunk_to_sse_format(chunk)
             collected_chunks.append(encoded_chunk)
             yield encoded_chunk
 
+        if not saw_terminal_event:
+            yield _incomplete_stream_error_sse_event()
+
         # Handle logging after all chunks are processed
         await self._handle_streaming_logging(collected_chunks)

litellm/llms/anthropic/files/transformation.py

@@ -39,6 +39,7 @@ from ..common_utils import AnthropicError, AnthropicModelInfo
 
 ANTHROPIC_FILES_API_BASE = "https://api.anthropic.com"
 ANTHROPIC_FILES_BETA_HEADER = "files-api-2025-04-14"
+ANTHROPIC_MESSAGE_BATCH_ID_PREFIX = "msgbatch_"
 
 
 class AnthropicFilesConfig(BaseFilesConfig):
@@ -258,6 +259,8 @@ class AnthropicFilesConfig(BaseFilesConfig):
         file_id = file_content_request.get("file_id")
         api_base = AnthropicModelInfo.get_api_base(litellm_params.get("api_base")) or ANTHROPIC_FILES_API_BASE
         encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
+        if file_id.startswith(ANTHROPIC_MESSAGE_BATCH_ID_PREFIX):
+            return f"{api_base.rstrip('/')}/v1/messages/batches/{encoded_file_id}/results", {}
         return f"{api_base.rstrip('/')}/v1/files/{encoded_file_id}/content", {}
 
     def transform_file_content_response(

litellm/llms/azure/responses/transformation.py

@@ -205,7 +205,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
     #########################################################
     ########## DELETE RESPONSE API TRANSFORMATION ##############
     #########################################################
-    def _construct_url_for_response_id_in_path(self, api_base: str, response_id: str) -> str:
+    def _construct_url_for_response_id_in_path(self, api_base: str, response_id: str, path_suffix: str = "") -> str:
         """
         Constructs a URL for the API request with the response_id in the path.
         """
@@ -218,14 +218,14 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
         # Remove trailing slash if present to avoid double slashes
         path = parsed_url.path.rstrip("/")
         encoded_response_id = encode_url_path_segment(response_id, field_name="response_id")
-        new_path = f"{path}/{encoded_response_id}"
+        new_path = f"{path}/{encoded_response_id}{path_suffix}"
 
         # Reconstruct the URL with all original components but with the modified path
         constructed_url = urlunparse(
             (
                 parsed_url.scheme,  # http, https
                 parsed_url.netloc,  # domain name, port
-                new_path,  # path with response_id added
+                new_path,
                 parsed_url.params,  # parameters
                 parsed_url.query,  # query string
                 parsed_url.fragment,  # fragment
@@ -288,7 +288,9 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
         limit: int = 20,
         order: Literal["asc", "desc"] = "desc",
     ) -> Tuple[str, Dict]:
-        url = self._construct_url_for_response_id_in_path(api_base=api_base, response_id=response_id) + "/input_items"
+        url = self._construct_url_for_response_id_in_path(
+            api_base=api_base, response_id=response_id, path_suffix="/input_items"
+        )
         params: Dict[str, Any] = {}
         if after is not None:
             params["after"] = after
@@ -322,27 +324,8 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
         This function handles URLs with query parameters by inserting the response_id
         at the correct location (before any query parameters).
         """
-        from urllib.parse import urlparse, urlunparse
-
-        # Parse the URL to separate its components
-        parsed_url = urlparse(api_base)
-
-        # Insert the response_id and /cancel at the end of the path component
-        # Remove trailing slash if present to avoid double slashes
-        path = parsed_url.path.rstrip("/")
-        encoded_response_id = encode_url_path_segment(response_id, field_name="response_id")
-        new_path = f"{path}/{encoded_response_id}/cancel"
-
-        # Reconstruct the URL with all original components but with the modified path
-        cancel_url = urlunparse(
-            (
-                parsed_url.scheme,  # http, https
-                parsed_url.netloc,  # domain name, port
-                new_path,  # path with response_id and /cancel added
-                parsed_url.params,  # parameters
-                parsed_url.query,  # query string
-                parsed_url.fragment,  # fragment
-            )
+        cancel_url = self._construct_url_for_response_id_in_path(
+            api_base=api_base, response_id=response_id, path_suffix="/cancel"
         )
 
         data: Dict = {}

litellm/llms/bedrock/base_aws_llm.py

@@ -50,6 +50,8 @@ _STS_REGION_FROM_ENDPOINT_PATTERN = re.compile(
     r"(?:^|\.)sts(?:-fips)?\.([a-z0-9-]+)\.(?:amazonaws\.com(?:\.cn)?|vpce\.amazonaws\.com)"
 )
 
+SIGV4_COMPUTED_HEADERS = frozenset({"authorization", "x-amz-date", "x-amz-security-token", "date"})
+
 
 class Boto3CredentialsInfo(BaseModel):
     credentials: Credentials
@@ -1400,11 +1402,13 @@ class BaseAWSLLM:
 
             # Add back all original headers (including forwarded ones) after signature calculation
             for header_name, header_value in headers.items():
-                if header_value is not None:
+                if header_value is not None and header_name.lower() not in SIGV4_COMPUTED_HEADERS:
                     request.headers[header_name] = header_value
 
             if (
-                extra_headers is not None and "Authorization" in extra_headers
+                extra_headers is not None
+                and "Authorization" in extra_headers
+                and not extra_headers["Authorization"].startswith("AWS4-HMAC-SHA256")
             ):  # prevent sigv4 from overwriting the auth header
                 request.headers["Authorization"] = extra_headers["Authorization"]
         prepped = request.prepare()
@@ -1527,9 +1531,15 @@ class BaseAWSLLM:
         # Add back original headers after signing. Only headers in SignedHeaders
         # are integrity-protected; forwarded headers (x-forwarded-*) must remain unsigned.
         for header_name, header_value in headers.items():
-            if header_value is not None:
+            if header_value is not None and header_name.lower() not in SIGV4_COMPUTED_HEADERS:
                 request_headers_dict[header_name] = header_value
-        if headers is not None and "Authorization" in headers:  # prevent sigv4 from overwriting the auth header
-            request_headers_dict["Authorization"] = headers["Authorization"]
+        incoming_authorization = next(
+            (value for name, value in headers.items() if name.lower() == "authorization" and value is not None),
+            None,
+        )
+        if incoming_authorization is not None and not incoming_authorization.startswith(
+            "AWS4-HMAC-SHA256"
+        ):  # prevent sigv4 from overwriting the auth header
+            request_headers_dict["Authorization"] = incoming_authorization
 
         return request_headers_dict, request.body

litellm/llms/bedrock/chat/invoke_handler.py

@@ -1558,7 +1558,7 @@ class AWSEventStreamDecoder:
             text = chunk_data["outputText"]
         # ai21 mapping
         elif "ai21" in self.model:  # fake ai21 streaming
-            text = chunk_data.get("completions")[0].get("data").get("text")  # type: ignore
+            text = chunk_data["completions"][0]["data"]["text"]
             is_finished = True
             finish_reason = "stop"
         ######## /bedrock/converse mappings ###############

litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py

@@ -51,10 +51,7 @@ class AmazonQwen2Config(AmazonQwen3Config):
         Qwen2 uses "text" field, but we also support "generation" field for compatibility.
         """
         try:
-            if hasattr(raw_response, "json"):
-                response_data = raw_response.json()
-            else:
-                response_data = raw_response
+            response_data = raw_response.json()
 
             # Extract the generated text - Qwen2 uses "text" field, but also support "generation" for compatibility
             generated_text = response_data.get("generation", "") or response_data.get("text", "")