Merge branch 'litellm_internal_staging' into litellm_ban_frozen_instance_bypass

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

.circleci/config.yml

@@ -1025,7 +1025,7 @@ jobs:
           name: Run tests
           command: |
             mkdir -p test-results
-            TEST_FILES=$(circleci tests glob "tests/agent_tests/**/test_*.py" | grep -v "^tests/agent_tests/local_only_agent_tests/")
+            TEST_FILES=$(circleci tests glob "tests/agent_tests/test_*.py")
             echo "$TEST_FILES" | circleci tests run \
               --verbose \
               --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
@@ -2744,84 +2744,6 @@ jobs:
           file: ./coverage.xml
           flags: circleci
 
-  ui_build:
-    docker:
-      - image: cimg/node:24.19@sha256:8966565f07189a67d64d6808a2b127f31dafae566508e3547f55640e1070bfad
-        auth:
-          username: ${DOCKERHUB_USERNAME}
-          password: ${DOCKERHUB_PASSWORD}
-    resource_class: medium+
-    working_directory: ~/project
-    steps:
-      - checkout
-      - skip_if_unrelated_changes:
-          category: client
-      - setup_google_dns
-      - restore_cache:
-          keys:
-            - ui-build-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
-            - ui-build-deps-v1-
-      - restore_cache:
-          keys:
-            - ui-nextjs-cache-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
-            - ui-nextjs-cache-v1-
-      - run:
-          name: Install dependencies
-          command: |
-            cd ui/litellm-dashboard
-            npm ci
-      - save_cache:
-          key: ui-build-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
-          paths:
-            - ui/litellm-dashboard/node_modules
-      - run:
-          name: Build UI
-          command: |
-            cd ui/litellm-dashboard
-            source ./build_ui.sh
-      - save_cache:
-          key: ui-nextjs-cache-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
-          paths:
-            - ui/litellm-dashboard/.next/cache
-      - persist_to_workspace:
-          root: .
-          paths:
-            - litellm/proxy/_experimental/out
-
-  ui_unit_tests:
-    docker:
-      - image: cimg/node:24.19@sha256:8966565f07189a67d64d6808a2b127f31dafae566508e3547f55640e1070bfad
-        auth:
-          username: ${DOCKERHUB_USERNAME}
-          password: ${DOCKERHUB_PASSWORD}
-    resource_class: xlarge
-    working_directory: ~/project
-    steps:
-      - checkout
-      - skip_if_unrelated_changes:
-          category: client
-      - setup_google_dns
-      - restore_cache:
-          keys:
-            - ui-unit-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
-            - ui-unit-deps-v1-
-      - run:
-          name: Install dependencies
-          command: |
-            cd ui/litellm-dashboard
-            npm ci
-      - save_cache:
-          key: ui-unit-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
-          paths:
-            - ui/litellm-dashboard/node_modules
-      - run:
-          name: Run UI unit tests (Vitest)
-          command: |
-            cd ui/litellm-dashboard
-
-            CI=true npm run test -- --run \
-              --pool forks --poolOptions.forks.maxForks=6
-
   e2e_ui_testing:
     docker:
       - image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2
@@ -3181,12 +3103,6 @@ workflows:
           filters: *main_branches
       - litellm_router_unit_testing:
           filters: *main_branches
-      - ui_build:
-          filters: *main_branches
-      - ui_unit_tests:
-          requires:
-            - ui_build
-          filters: *main_branches
       - auth_ui_unit_tests:
           filters: *main_branches
       - proxy_behavior_tests:

.circleci/scripts/classify_changes.sh

@@ -1,15 +1,17 @@
 #!/usr/bin/env bash
 set -uo pipefail
 
-category="${1:?usage: classify_changes.sh <backend|client>}"
+category="${1:?usage: classify_changes.sh <backend|client|ui>}"
 
 has_client=false
 has_backend=false
+has_ci=false
 while IFS= read -r file || [ -n "$file" ]; do
   [ -n "$file" ] || continue
   case "$file" in
     ui/* | tests/e2e/ui/*) has_client=true ;;
     docs/* | *.md | *.mdx) : ;;
+    .github/* | .circleci/*) has_ci=true; has_backend=true ;;
     *) has_backend=true ;;
   esac
 done
@@ -21,6 +23,9 @@ case "$category" in
   client)
     { [ "$has_client" = true ] || [ "$has_backend" = true ]; } && echo run || echo skip
     ;;
+  ui)
+    { [ "$has_client" = true ] || [ "$has_ci" = true ]; } && echo run || echo skip
+    ;;
   *)
     echo run
     ;;

.github/CODEOWNERS

@@ -1,3 +1,5 @@
 /ui/ @yuneng-jiang @ryan-crabbe-berri
 /litellm/proxy/_experimental/out/ @yuneng-jiang @ryan-crabbe-berri
 /ui/litellm-dashboard/src/lib/http/schema.d.ts
+/model_prices_and_context_window.json @mateo-berri
+/litellm/model_prices_and_context_window_backup.json @mateo-berri

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -23,30 +23,56 @@ body:
       label: What happened?
       description: Also tell us, what did you expect to happen?
       placeholder: Tell us what you see!
-      value: "A bug happened!"
     validations:
       required: true
   - type: textarea
-    id: steps-to-reproduce
+    id: user-flow
     attributes:
-      label: Steps to Reproduce
-      description: Please provide a numbered list of the exact steps to reproduce this bug (include a curl/python snippet to reproduce it). Number each step (1., 2., 3., ...) in the order you performed them.
+      label: User Flow
+      description: |
+        Two ordered lists, "Before a (hypothetical) fix" and "After a (hypothetical) fix", walking the same end user through the same task, written strictly from that user's seat. Every rule below applies.
+
+        - Describe the real application and the routes its users actually hit, not a generic scenario
+        - Lead each list with one plain sentence saying where the flow fails (before) or would succeed (after), then number the steps
+        - Every step is something the user does or observes: the HTTP method and full URL they hit, what they sent, and what visibly came back (status code, error text, the shape of an ID). UI steps name the page URL and what is on screen
+        - No LiteLLM internals: never name functions, files, DB tables, config classes, hooks, callbacks, or code paths. "The upload hands back an ID that looks like OpenAI's own `file-abc123` instead of the scrambled one the gateway returned" is right, "no managed-file row was registered" is wrong
+        - Keep the two lists step-for-step identical until they diverge, so the broken step is obvious
+        - If the bug has a security or authorization consequence, end each list with what another user can do that they shouldn't be able to, and what they could no longer do after a fix
       placeholder: |
-        1. config.yaml file/ .env file/ etc.
-        2. Run the following code...
-        3. Observe the error...
-      value: |
-        1. 
-        2. 
-        3. 
+        Before a (hypothetical) fix: a developer whose app streams chat completions gets no token counts back, so their cost dashboard reads zero
+
+        1. They send POST https://litellm-domain/v1/chat/completions with "stream": true and no stream_options
+        2. The last SSE chunk arrives with "usage": null, so their app records 0 prompt and 0 completion tokens
+        3. They open https://litellm-domain/ui/?page=logs and see the request logged at $0 spend
+
+        After a (hypothetical) fix: the same request comes back with real token counts, so the dashboard shows real spend
+
+        1. The proxy admin sets always_include_stream_usage: true and restarts the proxy
+        2. The developer sends the same POST https://litellm-domain/v1/chat/completions with "stream": true and no stream_options
+        3. The last SSE chunk now carries a usage object with real prompt and completion token counts
+        4. https://litellm-domain/ui/?page=logs shows that request at non-zero spend
     validations:
       required: true
   - type: textarea
-    id: logs
+    id: proof-of-bug
     attributes:
-      label: Relevant log output
-      description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
-      render: shell
+      label: Proof the bug occurs
+      description: |
+        The commands (e.g., curl) and their full output, screenshots, or a screen recording demonstrating that the bug happens. Every rule below applies.
+
+        - The proof must be completely e2e with no mocks, against a live proxy you ran yourself (e.g., `litellm --config config.yaml --detailed_debug` on localhost:4000), hitting real LLM provider APIs, costing real $ if needed, where the bug involves a provider call. `pytest` commands are not enough
+        - Show exactly what the end user sees or does, matching the User Flow above step for step
+        - Start with the config.yaml (or SDK setup) and any env vars the proxy ran with, then the exact version or commit hash the proof was captured at, so a maintainer can stand up the same proxy before running your commands. Keep the real values for env vars that aren't sensitive, they are often the reason the bug happens, and redact only the secrets: never paste a real API key, virtual key, database URL, or other credential, here or anywhere else in the issue
+        - If the bug applies to more than one of the LLM endpoints (/v1/responses, /v1/chat/completions, /v1/messages), include proof for every one of them, not just one
+        - For UI bugs: include screenshots and the page URLs you were on. Scrub keys and tokens out of screenshots too (for example, the virtual key is briefly shown in the panel right after you create a virtual key)
+      placeholder: |
+        Config / setup the proxy ran with:
+
+        Version or commit:
+
+        Commands and their full output:
+    validations:
+      required: true
   - type: dropdown
     id: component
     attributes:

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -24,10 +24,53 @@ body:
     validations:
       required: true
   - type: textarea
-    id: motivation
+    id: user-flow
     attributes:
-      label: Motivation, pitch
-      description: Please outline the motivation for the proposal. Is your feature request related to a specific problem? e.g., "I'm working on X and would like Y to be possible". If this is related to another GitHub issue, please link here too.
+      label: User Flow
+      description: |
+        Two ordered lists, "Before this feature (today)" and "After this feature (ideal user flow)", walking the same end user through the same task, written strictly from that user's seat. Every rule below applies.
+
+        - Describe the real application and the routes its users actually hit, not a generic scenario. Link any related GitHub issue or provider API docs
+        - Lead each list with one plain sentence saying where the flow dead-ends today and what it would let them do instead, then number the steps
+        - Every step is something the user does or observes: the HTTP method and full URL they hit, what they sent, and what visibly came back (status code, error text, the shape of an ID). UI steps name the page URL and what is on screen
+        - No LiteLLM internals: never name functions, files, DB tables, config classes, hooks, callbacks, or code paths. Ask for the behavior you need, not the implementation you imagine
+        - Keep the two lists step-for-step identical until they diverge, so the missing capability is obvious
+        - "Before this feature" is also where you show the workaround you're living with, which is what tells us how badly this is needed
+      placeholder: |
+        Before this feature (today): a developer batching nightly summaries has no way to mark those calls as low priority, so they compete with live traffic for the same rate limit
+
+        1. They send POST https://litellm-domain/v1/chat/completions for 500 documents in a loop
+        2. Around document 120 they start getting 429s naming the rpm limit, and their user-facing chat app starts getting them too
+        3. Their workaround is a hand-rolled sleep between calls, which stretches the batch to 3 hours and still collides at peak
+
+        After this feature (ideal user flow): the same batch runs as background work that yields to live traffic
+
+        1. The developer sends the same POST with "service_tier": "flex"
+        2. Batch calls queue behind interactive ones instead of 429ing, and the response comes back with the tier it was served at
+        3. The live chat app keeps returning 200s throughout the batch
+        4. https://litellm-domain/ui/?page=logs shows the batch requests tagged with that tier
+    validations:
+      required: true
+  - type: textarea
+    id: how-far-you-got
+    attributes:
+      label: How far you got
+      description: |
+        Run as many steps of the "After this feature (ideal user flow)" list as you can against a live proxy you ran yourself (e.g., `litellm --config config.yaml --detailed_debug` on localhost:4000), then paste the commands (e.g., curl) and their full output, ending at the step that dead-ends. Every rule below applies.
+
+        - Say plainly what stopped you there, in user terms: the option you passed came back ignored, the response 400'd naming an unsupported field, there is no button on the page for it. This is what proves the feature is genuinely missing rather than undocumented
+        - No mocks. Where the flow involves a provider call, hit the real provider API, even if it costs real $. `pytest` commands are not enough
+        - Include the config.yaml (or SDK setup) and env vars the proxy ran with, plus the version or commit you were on. Keep the real values for env vars that aren't sensitive, and redact only the secrets: never paste a real API key, virtual key, database URL, or other credential, here or anywhere else in the issue
+        - If the provider already supports this, link their API docs and paste a direct call to them succeeding, so we can see the shape LiteLLM should be sending
+        - For UI asks: include screenshots of the page you got stuck on and its URL. Scrub keys and tokens out of screenshots too (for example, the virtual key is briefly shown in the panel right after you create a virtual key)
+      placeholder: |
+        Config / setup the proxy ran with:
+
+        Version or commit:
+
+        Commands and their full output, up to the step that dead-ends:
+
+        What stopped me there:
     validations:
       required: true
   - type: dropdown

.github/actions/detect-backend-changes/action.yml

@@ -1,48 +0,0 @@
-name: "Detect backend-relevant changes"
-description: >-
-  Classify the pull request's changed files with .circleci/scripts/classify_changes.sh
-  and expose decision=run|skip. decision=skip means only ui/**, **.md or **.mdx files
-  changed, so callers can short-circuit expensive steps while the job still completes
-  successfully and satisfies its required status check. The decision defaults to run for
-  any non pull_request event or whenever the changed set cannot be resolved, so tests are
-  never skipped when the classification is uncertain.
-
-outputs:
-  decision:
-    description: "run when backend-relevant files changed, otherwise skip"
-    value: ${{ steps.classify.outputs.decision }}
-
-runs:
-  using: composite
-  steps:
-    - id: classify
-      shell: bash
-      env:
-        BASE_SHA: ${{ github.event.pull_request.base.sha }}
-      run: |
-        set -uo pipefail
-        if [ -z "${BASE_SHA:-}" ]; then
-          echo "detect-backend-changes: not a pull_request event; running job"
-          echo "decision=run" >> "${GITHUB_OUTPUT}"
-          exit 0
-        fi
-        if ! git fetch --no-tags --depth=1 origin "${BASE_SHA}" >/dev/null 2>&1; then
-          echo "detect-backend-changes: could not fetch base ${BASE_SHA}; running job"
-          echo "decision=run" >> "${GITHUB_OUTPUT}"
-          exit 0
-        fi
-        changed="$(git diff --name-only "${BASE_SHA}" HEAD 2>/dev/null)" || {
-          echo "detect-backend-changes: git diff failed; running job"
-          echo "decision=run" >> "${GITHUB_OUTPUT}"
-          exit 0
-        }
-        if [ -z "${changed}" ]; then
-          echo "detect-backend-changes: no changed files vs ${BASE_SHA}; skipping job"
-          echo "decision=skip" >> "${GITHUB_OUTPUT}"
-          exit 0
-        fi
-        echo "detect-backend-changes: changed files vs ${BASE_SHA}:"
-        printf '%s\n' "${changed}" | sed 's/^/  /'
-        decision="$(printf '%s\n' "${changed}" | bash .circleci/scripts/classify_changes.sh backend)" || decision="run"
-        echo "detect-backend-changes: decision=${decision}"
-        echo "decision=${decision}" >> "${GITHUB_OUTPUT}"

.github/actions/detect-changes/action.yml

@@ -0,0 +1,41 @@
+name: "Detect relevant changes"
+description: >-
+  Classify the pull request's changed files with .circleci/scripts/classify_changes.sh
+  and expose decision=run|skip for one category. backend means anything outside ui/,
+  docs/ and markdown; ui means the dashboard sources alone. decision=skip lets callers
+  short-circuit expensive steps while the job still completes successfully and satisfies
+  its required status check, which a paths: filter cannot do because a workflow that
+  never starts never reports. The file list comes from the pull request itself rather
+  than from a git diff, because the checked-out merge ref is recomputed as the base
+  branch advances and would otherwise attribute the base branch's own commits to the
+  pull request. The decision defaults to run for any non pull_request event or whenever
+  the changed set cannot be resolved, so jobs are never skipped when the classification
+  is uncertain.
+
+inputs:
+  category:
+    description: "Which classification to apply: backend, client or ui"
+    required: false
+    default: backend
+  github-token:
+    description: "Token used to list the pull request's files; needs pull-requests: read"
+    required: false
+    default: ${{ github.token }}
+
+outputs:
+  decision:
+    description: "run when category-relevant files changed, otherwise skip"
+    value: ${{ steps.classify.outputs.decision }}
+
+runs:
+  using: composite
+  steps:
+    - id: classify
+      shell: bash
+      env:
+        GH_TOKEN: ${{ inputs.github-token }}
+        CATEGORY: ${{ inputs.category }}
+        REPO: ${{ github.repository }}
+        PR_NUMBER: ${{ github.event.pull_request.number }}
+        CHANGED_FILE_COUNT: ${{ github.event.pull_request.changed_files }}
+      run: bash "${GITHUB_ACTION_PATH}/../../scripts/detect_changes.sh"

.github/ci-coverage-allowlist.yml

@@ -63,30 +63,6 @@ test_paths:
       - tests/litellm/test_router_retry_backoff_headers.py
       - tests/litellm/test_sambanova_model_metadata.py
       - tests/litellm/test_stream_chunk_builder_images.py
-  - reason: >-
-      Legacy proxy suite superseded by the proxy shards; no job invokes it and whether it still
-      describes supported behaviour is unresolved
-    paths:
-      - tests/old_proxy_tests/tests/test_anthropic_context_caching.py
-      - tests/old_proxy_tests/tests/test_anthropic_sdk.py
-      - tests/old_proxy_tests/tests/test_async.py
-      - tests/old_proxy_tests/tests/test_gemini_context_caching.py
-      - tests/old_proxy_tests/tests/test_langchain_embedding.py
-      - tests/old_proxy_tests/tests/test_langchain_request.py
-      - tests/old_proxy_tests/tests/test_llamaindex.py
-      - tests/old_proxy_tests/tests/test_mistral_sdk.py
-      - tests/old_proxy_tests/tests/test_openai_embedding.py
-      - tests/old_proxy_tests/tests/test_openai_exception_request.py
-      - tests/old_proxy_tests/tests/test_openai_request.py
-      - tests/old_proxy_tests/tests/test_openai_request_with_traceparent.py
-      - tests/old_proxy_tests/tests/test_openai_simple_embedding.py
-      - tests/old_proxy_tests/tests/test_openai_tts_request.py
-      - tests/old_proxy_tests/tests/test_pass_through_langfuse.py
-      - tests/old_proxy_tests/tests/test_q.py
-      - tests/old_proxy_tests/tests/test_simple_traceparent_openai.py
-      - tests/old_proxy_tests/tests/test_vertex_sdk_forward_headers.py
-      - tests/old_proxy_tests/tests/test_vtx_embedding.py
-      - tests/old_proxy_tests/tests/test_vtx_sdk_embedding.py
   - reason: >-
       No job invokes this suite and its files mix pure transformation tests with ones driving live
       vendor vector stores, so assigning them needs a per-file decision
@@ -116,6 +92,14 @@ test_paths:
       - tests/load_tests/test_otel_load_test.py
       - tests/load_tests/test_vertex_embeddings_load_test.py
       - tests/load_tests/test_vertex_load_tests.py
+  - reason: >-
+      A local-only agent rig: test_a2a_completion_bridge.py needs a LangGraph server on
+      localhost:2024 and test_a2a.py drives a live A2A endpoint, so neither can run in a
+      pull request job. Until 2026-08-20 the CircleCI agent job hid them behind a grep -v
+      that this census could not see; the glob now excludes them structurally and this entry
+      is the decision on the record. Revisit when the A2A bridge gets a recorded-wire fixture
+    paths:
+      - tests/agent_tests/local_only_agent_tests
   - reason: >-
       Third-party integration tests that skip themselves without OCI configuration or sandbox
       credentials, neither of which a pull request job holds

.github/pull_request_template.md

@@ -53,7 +53,8 @@ After: the same request comes back with real token counts, so the dashboard show
 **Please complete all items before asking a LiteLLM maintainer to review your PR**
 
 - [ ] I have added meaningful tests
-- [ ] My PR passes all CI/CD checks (e.g., lint, format, unit tests)
+- [ ] The handful of test files covering my change pass locally, e.g. `uv run pytest tests/test_litellm/<your_test_file>.py -v`. Leave the suites (`make test-unit-*`, `make test-unit`) to CI: it finishes in ~15 minutes where a laptop takes an hour or more
+- [ ] My PR passes all required CI/CD checks (e.g., lint, schema.d.ts sync check, etc.)
 - [ ] My PR's scope is as isolated as possible; it only solves 1 specific problem
 - [ ] I have received a Greptile **Confidence Score of at least 4/5** before requesting a maintainer review (Greptile reviews automatically once the PR is opened; only comment `@greptileai` to re-request a review after pushing changes)
 
@@ -64,12 +65,36 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
 ## Screenshots / Proof of Fix
 
 <!-- Include screenshots, screen recordings, or command (e.g., curl) + output demonstrating that your changes work as expected
-     The proof must be completely e2e with no mocks, using, for example, actual LLM calls costing real $. `pytest` commands are not enough
-     For bug fixes: show reproduction before the fix and passing behavior after
-     Include the commit hash each proof was captured at, for both the before and the after runs
-     If the change applies to all three LLM endpoints (/v1/responses, /v1/chat/completions, /v1/messages), include proof for every single one of them, not just one
-     For new features: show the feature working end-to-end
-     For UI changes: include before/after screenshots -->
+     The proof must be completely e2e with no mocks, using actual LLM calls costing real $$$ if applicable. `pytest` commands are not enough
+     Show ONLY the latest run: capture Before at the merge base and After at the PR's current tip, and when new commits change behavior, replace this whole section with the fresh run instead of stacking it on top of older ones. The run must be up to date. As soon as a new commit is made and it makes this PR description's after sha stale (it's no longer tip of PR), you must re-run the QA
+     Structure the section exactly as below: Before and After one heading level below this section, each naming the commit hash it was captured at, one lower-level heading per case inside each, the same case names in the same order on both sides, and numbered steps (command, observed output) under every case, never loose prose; shared setup (config, payloads) goes above Before, and with a single case, drop the case headings and number the steps directly
+
+### Before (<hash>)
+
+#### <case 1>
+
+1. ...
+2. ...
+
+#### <case 2>
+
+1. ...
+
+### After (<hash>)
+
+#### <case 1>
+
+1. ...
+2. ...
+
+#### <case 2>
+
+1. ...
+
+     For bug fixes: Before shows the reproduction, After shows the same steps passing
+     For new features: Before shows the capability missing, After shows it working end-to-end
+     If the change applies to all three LLM endpoints (/v1/responses, /v1/chat/completions, /v1/messages), make each endpoint its own case, not just one
+     For UI changes: before/after screenshots under the same headings -->
 
 ## Type
 

.github/scripts/detect_changes.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+set -uo pipefail
+
+readonly API_FILE_CEILING=3000
+readonly CATEGORY="${CATEGORY:-backend}"
+
+decide() {
+  echo "detect-changes[${CATEGORY}]: decision=$1"
+  [ -z "${GITHUB_OUTPUT:-}" ] || echo "decision=$1" >>"${GITHUB_OUTPUT}"
+  exit 0
+}
+
+run_full() {
+  echo "detect-changes[${CATEGORY}]: $1; running job"
+  decide run
+}
+
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+classify="${here}/../../.circleci/scripts/classify_changes.sh"
+
+[ -n "${PR_NUMBER:-}" ] || run_full "not a pull_request event"
+[ -n "${REPO:-}" ] || run_full "no repository in the environment"
+
+case "${CHANGED_FILE_COUNT:-}" in
+'' | *[!0-9]*) run_full "the event payload carries no changed_files count" ;;
+esac
+[ "${CHANGED_FILE_COUNT}" -le "${API_FILE_CEILING}" ] ||
+  run_full "PR #${PR_NUMBER} changes ${CHANGED_FILE_COUNT} files, past the ${API_FILE_CEILING}-file listing ceiling"
+
+changed="$(gh api "repos/${REPO}/pulls/${PR_NUMBER}/files" --paginate --jq '.[].filename')" ||
+  run_full "could not list the files on PR #${PR_NUMBER}"
+[ -n "${changed}" ] || run_full "the API listed no files on PR #${PR_NUMBER}"
+
+echo "detect-changes[${CATEGORY}]: files changed by PR #${PR_NUMBER}:"
+printf '%s\n' "${changed}" | sed 's/^/  /'
+
+decision="$(printf '%s\n' "${changed}" | bash "${classify}" "${CATEGORY}")" ||
+  run_full "classify_changes.sh failed"
+case "${decision}" in
+run | skip) decide "${decision}" ;;
+*) run_full "classify_changes.sh printed an unexpected decision: ${decision}" ;;
+esac

.github/scripts/select_ui_test_scope.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -uo pipefail
+
+has_file=false
+has_file_outside_src=false
+while IFS= read -r file || [ -n "$file" ]; do
+  [ -n "$file" ] || continue
+  has_file=true
+  case "$file" in
+    src/*) ;;
+    *) has_file_outside_src=true ;;
+  esac
+done
+
+{ [ "$has_file" = true ] && [ "$has_file_outside_src" = false ]; } && echo related || echo full

.github/scripts/triage_rollout_heads_up.py

@@ -1,557 +0,0 @@
-#!/usr/bin/env python3
-"""One-shot 7-day heads-up sweep for the Agent Shin rollout.
-
-Posts a friendly "the OSS triage bot kicks in next Monday" comment on every
-open external PR/issue that currently *would* fail the new rubric — i.e.,
-every PR/issue Agent Shin would close once the rollout completes. The point
-is to give contributors a full week to fix their description before the bot
-ever takes a destructive action, so nobody is surprised by an auto-close.
-
-The script is designed to run **exactly once** at rollout, fired by a manual
-``workflow_dispatch`` (``dry_run=false``) on the heads-up workflow. Re-runs
-are safe: every comment is stamped with the hidden ``HEADS_UP_MARKER`` and
-PRs/issues that already carry the marker are skipped.
-
-Dry-run vs. real run
---------------------
-Defaults to dry-run. Passing ``--close`` flips into real mode. Every GitHub
-mutation goes through ``_agent_shin_actions``, which has a one-line
-``if dry_run: log else: do_it`` per call, so the only difference between a
-dry-run preview and the real run is the call site that actually hits the
-GitHub API.
-
-Local preview::
-
-    python3 .github/scripts/triage_rollout_heads_up.py --repo BerriAI/litellm
-
-Real run (the manual rollout dispatch uses this)::
-
-    python3 .github/scripts/triage_rollout_heads_up.py --repo BerriAI/litellm --close
-"""
-
-from __future__ import annotations
-
-import argparse
-import datetime as dt
-import json
-import os
-import sys
-from pathlib import Path
-from typing import Any
-
-# Make the sibling triage_with_llm + _agent_shin_actions importable when this
-# script is invoked directly (the GitHub workflow does `python3 .github/scripts/...`).
-_SCRIPTS_DIR = Path(__file__).resolve().parent
-if str(_SCRIPTS_DIR) not in sys.path:
-    sys.path.insert(0, str(_SCRIPTS_DIR))
-
-from _agent_shin_actions import maybe_post_comment  # noqa: E402
-from agent_shin_shared import (  # noqa: E402
-    AGENT_SHIN_DEFAULT_BOT_LOGIN,
-    ALLOWLIST_LOGINS,
-    list_open_items,
-)
-from triage_with_llm import (  # noqa: E402
-    DEFAULT_MODEL,
-    call_llm_judge,
-    fetch_issue,
-    fetch_pr,
-    gh,
-    is_internal_contributor,
-    review_gate,
-    triage,
-)
-
-# Hidden marker so re-runs skip PRs/issues we've already notified. Distinct from
-# the within-grace / ready / regressed markers so it can't be confused with the
-# steady-state lifecycle comments.
-HEADS_UP_MARKER = "<!-- agent-shin:rollout-heads-up -->"
-
-# Placeholder until the litellm-docs PR ships. The rollout blog post explains
-# the new rubric, the 7-day grace, and how to recover after an auto-close.
-# TODO(docs): replace with the canonical URL once the litellm-docs PR merges.
-ROLLOUT_BLOG_URL = "https://docs.litellm.ai/docs/agent_shin_triage_rollout"
-
-# Default cutoff is one week from "now". Computed at runtime so the wording
-# stays correct even if the rollout is merged later than planned. The user can
-# override with --close-on YYYY-MM-DD when running the script manually.
-DEFAULT_GRACE_DAYS = 7
-
-# The daily auto-close sweeps (close_low_quality_prs.yml at 09:00 UTC and
-# review_gate.yml at 09:30 UTC) are what actually close a still-failing item,
-# so the deadline we promise contributors has to name that wall-clock moment.
-ACTIVATION_TIME_UTC = "09:00 UTC"
-
-
-def _format_cutoff(cutoff: dt.date) -> str:
-    """Human-readable, timezone-explicit cutoff, e.g. ``Monday, June 1, 2026
-    (09:00 UTC)`` — the moment a still-failing PR/issue gets closed."""
-    return (
-        f"{cutoff.strftime('%A, %B')} {cutoff.day}, {cutoff.year} "
-        f"({ACTIVATION_TIME_UTC})"
-    )
-
-
-def _rubric_section_pr() -> str:
-    return (
-        "**Going forward, every external PR needs ONE of:**\n"
-        "\n"
-        "- A linked GitHub issue using a closing keyword: "
-        "`Fixes #1234`, `Closes #1234`, or `Resolves #1234`, OR\n"
-        "- All three of: a clear **problem description**, **expected vs. "
-        "actual behavior**, and **end-to-end QA proof** (at least one of a "
-        "short screen recording / video, before/after screenshots, or the "
-        "exact commands you ran with their real output; mocked or stubbed "
-        "runs don't count).\n"
-        "\n"
-        "PRs also need a **Greptile confidence score of 4/5 or higher** before "
-        "the bot will tag them `ready for review`. You can `@greptileai` to "
-        "request a fresh review at any time, including after the PR is closed."
-    )
-
-
-def _rubric_section_issue() -> str:
-    return (
-        "**Going forward, every external issue needs:**\n"
-        "\n"
-        "- For **bug reports**: end-to-end evidence of the bug (at least one "
-        "of a screen recording / video, a screenshot, or the exact commands "
-        "you ran with their real output / traceback) plus expected vs. actual "
-        "behavior. Written steps with no run output don't count, and mocked "
-        "or stubbed runs don't count.\n"
-        "- For **feature requests**: a clear description of the proposed "
-        "feature plus a use case + concrete example (config, API call, UI "
-        "flow, or scenario showing what's blocked today)."
-    )
-
-
-def _description_only_note(kind: str) -> str:
-    noun = "PR" if kind == "pr" else "issue"
-    return (
-        f"⚠️ **The requirements must live in the {noun} *description*, not in "
-        "comments.** Some PRs/issues collect 100+ comments from humans and "
-        "bots; reading the entire thread on every triage run would balloon "
-        "GitHub API usage (we'd start getting 429'd) and blow out the LLM "
-        "judge's context. The bot only reads the description, so anything "
-        "you add as a comment will be invisible to it."
-    )
-
-
-def _missing_section(verdict: dict, greptile_score: int | None) -> str:
-    """Bullet list of what's currently missing on this PR/issue.
-
-    Combines the LLM judge's `missing` list (rubric items) with a Greptile
-    shortfall (for PRs) so the contributor sees one list of things to fix.
-    """
-    missing = list(verdict.get("missing") or [])
-    if greptile_score is not None and greptile_score < 4:
-        missing.insert(
-            0,
-            f"Greptile's most recent review scored this PR {greptile_score}/5 "
-            "(below the 4/5 bar Agent Shin will require).",
-        )
-    if not missing:
-        return (
-            "_The bot couldn't articulate a specific missing piece; see the "
-            "rubric link above and double-check the description includes all "
-            "of it before the rollout._"
-        )
-    bullets = "\n".join(f"- {m}" for m in missing)
-    return f"**What this one is currently missing:**\n\n{bullets}"
-
-
-def _recovery_section(kind: str) -> str:
-    if kind == "pr":
-        return (
-            "**If the bot closes this PR after the rollout:** update the "
-            "description with the missing pieces, then either open a fresh "
-            "PR or comment `@agent-shin reconsider` on the closed PR. If "
-            "Greptile re-scores you at 4/5 or higher I'll reopen and tag "
-            "the PR `ready for review`. (`@greptileai` works on closed PRs "
-            "too; a fresh review is one of the signals that lifts you back "
-            "into the queue.) This is **not** us losing interest in your "
-            "change; far from it. We just need open PRs to be a list of "
-            "things a maintainer can act on, so we can get to yours faster."
-        )
-    return (
-        "**If the bot closes this issue after the rollout:** edit the issue "
-        "description to add the missing pieces, then comment `@agent-shin "
-        "reconsider` on the closed issue. I'll re-evaluate and, if the rubric "
-        "is met, reopen it. (GitHub doesn't let external authors reopen an "
-        "issue a maintainer or bot closed, so the comment is the reliable "
-        "path.) This is **not** us saying the bug isn't real or the request "
-        "isn't useful; it's so the remaining open issues are a list of things "
-        "a maintainer can act on."
-    )
-
-
-def format_heads_up_comment(
-    *, kind: str, verdict: dict, greptile_score: int | None, cutoff: dt.date
-) -> str:
-    """Compose the friendly 7-day heads-up comment posted on a failing PR/issue."""
-    noun = "PR" if kind == "pr" else "issue"
-    rubric = _rubric_section_pr() if kind == "pr" else _rubric_section_issue()
-    cutoff_str = _format_cutoff(cutoff)
-    explanation = (verdict.get("explanation") or "").strip()
-    explanation_block = (
-        f"> _(The judge's note for this one: {explanation})_\n\n" if explanation else ""
-    )
-
-    return (
-        "🚅 **Heads-up: we're turning on the OSS triage bot in "
-        f"{DEFAULT_GRACE_DAYS} days, on {cutoff_str}.**\n"
-        "\n"
-        "We're rolling out **Agent Shin**, an LLM-as-judge triage bot for "
-        f"external {noun}s. Once it's live, the bot reads each open "
-        f"{noun}'s description, scores it against a small rubric, and "
-        f"auto-closes any {noun} that's missing the basics, with a single "
-        f"comment explaining what's missing and how to recover. Full "
-        f"context: [Agent Shin rollout blog post]({ROLLOUT_BLOG_URL}).\n"
-        "\n"
-        f"{rubric}\n"
-        "\n"
-        f"{_description_only_note(kind)}\n"
-        "\n"
-        f"{_missing_section(verdict, greptile_score)}\n"
-        "\n"
-        f"{explanation_block}"
-        "**Timeline (you have a week):**\n"
-        "\n"
-        f"- We turn the bot on in {DEFAULT_GRACE_DAYS} days, on "
-        f"**{cutoff_str}**. You have until then to update this {noun}'s "
-        "description with the missing pieces above.\n"
-        f"- If this {noun} still fails the rubric at **{cutoff_str}**, "
-        "we'll close it.\n"
-        f"- From then on the bot runs daily, and every {noun} that fails "
-        "the rubric gets a **2-hour lifetime**: one warning comment, then "
-        "auto-close 2 hours later.\n"
-        "\n"
-        f"{_recovery_section(kind)}\n"
-        "\n"
-        f"{HEADS_UP_MARKER}"
-    )
-
-
-def _list_open_numbers(repo: str, kind: str) -> list[int]:
-    """Return every open PR or issue number in ``repo``.
-
-    Delegates to ``list_open_items`` so the full backlog is fetched (no cap)
-    and the `gh {pr,issue} list` invocation stays in one shared place. ``gh
-    issue list`` would include PRs, but ``list_open_items`` uses the dedicated
-    command per kind, so the two never mix.
-    """
-    return [
-        item["number"] for item in list_open_items(kind, repo=repo, fields="number")
-    ]
-
-
-def _has_heads_up_marker(item: dict) -> bool:
-    """Cheap fast-path: check the PR/issue body itself for the marker.
-
-    The marker is appended to the *comment* we post, not the body, so this
-    will only fire if the body literally contains the marker text. We still
-    do the comment-marker check separately below; this body check just lets
-    us short-circuit for PRs/issues that quote the marker for any reason.
-    """
-    body = item.get("body") or ""
-    return HEADS_UP_MARKER in body
-
-
-def _comments_have_marker(repo: str, number: int) -> bool:
-    """True if the bot already posted a comment carrying the marker.
-
-    Used for idempotency: a re-run skips items the previous run notified.
-    Filters by author (matching the sibling marker-checks in
-    ``triage_with_llm._has_marker`` and
-    ``agent_shin_shared.seconds_since_latest_marker_comment``) so a
-    contributor who quotes the heads-up via GitHub's "Quote reply" — which
-    preserves HTML comments in the raw markdown — can't trick the
-    idempotency check into silently skipping a real heads-up.
-
-    Comments live on the unified issues endpoint regardless of whether the
-    item is a PR or an issue, so no ``kind`` argument is required here.
-    """
-    expected_login = (
-        os.environ.get("AGENT_SHIN_BOT_LOGIN") or AGENT_SHIN_DEFAULT_BOT_LOGIN
-    ).lower()
-    raw = gh(
-        "api",
-        "--paginate",
-        f"repos/{repo}/issues/{number}/comments?per_page=100",
-    )
-    for line in raw.splitlines():
-        line = line.strip()
-        if not line:
-            continue
-        try:
-            payload = json.loads(line)
-        except json.JSONDecodeError:
-            continue
-        comments = payload if isinstance(payload, list) else [payload]
-        for comment in comments:
-            author = ((comment.get("user") or {}).get("login") or "").lower()
-            if author != expected_login:
-                continue
-            if HEADS_UP_MARKER in (comment.get("body") or ""):
-                return True
-    return False
-
-
-def _evaluate_pr(*, repo: str, number: int, model: str, judge: Any = None) -> dict:
-    """Run the future PR rubric (review_gate) in dry-run and return the result."""
-    return review_gate(
-        repo=repo,
-        number=number,
-        close=False,  # we only want the verdict, never act here
-        model=model,
-        judge=judge,
-    )
-
-
-def _evaluate_issue(*, repo: str, number: int, model: str, judge: Any = None) -> dict:
-    """Run the future issue rubric (triage kind='issue') in dry-run."""
-    return triage(
-        repo=repo,
-        kind="issue",
-        number=number,
-        close=False,
-        model=model,
-        judge=judge,
-    )
-
-
-def _would_be_closed(kind: str, result: dict) -> bool:
-    """True if the future triage would auto-close this PR/issue based on the
-    rubric (regardless of grace-period gating).
-
-    For PRs we trust ``review_gate``'s ``passing`` field — it combines the LLM
-    verdict and the Greptile score. For issues we read the LLM verdict
-    directly. Both fields are ``None``/missing on skip paths
-    (skip-internal-author, skip-llm-error, etc.) where the future bot would
-    NOT close the item — those return False.
-    """
-    if kind == "pr":
-        passing = result.get("passing")
-        if passing is None:
-            return False  # skipped — nothing for the heads-up to warn about
-        return passing is False
-    verdict = result.get("verdict") or {}
-    return (verdict.get("verdict") or "").lower() == "fail"
-
-
-def _process_one(
-    *,
-    repo: str,
-    kind: str,
-    number: int,
-    model: str,
-    cutoff: dt.date,
-    dry_run: bool,
-    judge: Any = None,
-    skip_marker_check: bool = False,
-    allowlist: frozenset[str] = ALLOWLIST_LOGINS,
-) -> dict:
-    """Evaluate one PR/issue and post a heads-up if it would be auto-closed.
-
-    Returns a per-item dict for the summary table.
-    """
-    base = {"kind": kind, "number": number}
-    fetcher = fetch_pr if kind == "pr" else fetch_issue
-    item = fetcher(repo, number)
-
-    if (item.get("state") or "") != "open":
-        return {**base, "action": "skip-not-open"}
-    if allowlist:
-        login = (item.get("user") or {}).get("login") or ""
-        if login.lower() not in allowlist:
-            return {**base, "action": "skip-not-allowlisted"}
-    elif is_internal_contributor(item):
-        return {**base, "action": "skip-internal-author"}
-    if not skip_marker_check and _has_heads_up_marker(item):
-        return {**base, "action": "skip-already-marked-in-body"}
-    if not skip_marker_check and _comments_have_marker(repo, number):
-        return {**base, "action": "skip-already-notified"}
-
-    if kind == "pr":
-        result = _evaluate_pr(repo=repo, number=number, model=model, judge=judge)
-    else:
-        result = _evaluate_issue(repo=repo, number=number, model=model, judge=judge)
-
-    if not _would_be_closed(kind, result):
-        return {**base, "action": "skip-passing", "evaluator": result.get("action")}
-
-    verdict = result.get("verdict") or {}
-    greptile_score = result.get("greptile_score") if kind == "pr" else None
-    comment = format_heads_up_comment(
-        kind=kind, verdict=verdict, greptile_score=greptile_score, cutoff=cutoff
-    )
-    maybe_post_comment(repo, number, comment, dry_run=dry_run)
-    return {
-        **base,
-        "action": "heads-up-posted" if not dry_run else "would-post-heads-up",
-        "verdict": (verdict.get("verdict") or "").lower(),
-        "greptile_score": greptile_score,
-    }
-
-
-def _print_summary(results: list[dict]) -> None:
-    """Tally per-action counts so a dry-run preview tells you at a glance how
-    many comments the real run would post."""
-    counts: dict[str, int] = {}
-    for r in results:
-        counts[r["action"]] = counts.get(r["action"], 0) + 1
-    print("\n=== rollout heads-up summary ===")
-    for action in sorted(counts):
-        print(f"  {action:35s} {counts[action]}")
-    print(f"  total                                {len(results)}")
-
-
-def run(
-    *,
-    repo: str,
-    close: bool,
-    cutoff: dt.date,
-    model: str,
-    kinds: tuple[str, ...] = ("pr", "issue"),
-    judge: Any = None,
-    only_numbers: dict[str, list[int]] | None = None,
-    skip_marker_check: bool = False,
-) -> list[dict]:
-    """Sweep ``repo`` and post heads-up comments. Returns the per-item results."""
-    dry_run = not close
-    if dry_run:
-        print(
-            f"[DRY RUN] sweeping {repo}; --close not passed, no comments will be posted."
-        )
-    else:
-        print(f"[REAL RUN] sweeping {repo}; comments WILL be posted.")
-    print(f"Cutoff date in comment body: {cutoff.isoformat()}")
-
-    results: list[dict] = []
-    for kind in kinds:
-        if only_numbers and kind in only_numbers:
-            numbers = list(only_numbers[kind])
-        else:
-            numbers = _list_open_numbers(repo, kind)
-        print(f"\n--- {kind}s: {len(numbers)} open ---")
-        for n in numbers:
-            try:
-                result = _process_one(
-                    repo=repo,
-                    kind=kind,
-                    number=n,
-                    model=model,
-                    cutoff=cutoff,
-                    dry_run=dry_run,
-                    judge=judge,
-                    skip_marker_check=skip_marker_check,
-                )
-            except (
-                Exception
-            ) as exc:  # noqa: BLE001 - per-item errors don't abort the sweep
-                result = {
-                    "kind": kind,
-                    "number": n,
-                    "action": "error",
-                    "error": str(exc),
-                }
-                print(f"!! {kind}#{n}: {exc}", file=sys.stderr)
-            print(f"  {kind}#{n}: {result['action']}")
-            results.append(result)
-    _print_summary(results)
-    return results
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--repo", required=True, help="owner/repo")
-    parser.add_argument(
-        "--close",
-        action="store_true",
-        help=(
-            "Actually post comments. Without this flag the script is in "
-            "dry-run mode and only logs what it would do."
-        ),
-    )
-    parser.add_argument(
-        "--close-on",
-        type=dt.date.fromisoformat,
-        default=None,
-        help=(
-            "Cutoff date shown in the heads-up comment as the rollout date "
-            f"(default: today + {DEFAULT_GRACE_DAYS} days)."
-        ),
-    )
-    parser.add_argument(
-        "--model",
-        default=os.environ.get("TRIAGE_MODEL") or DEFAULT_MODEL,
-        help=f"Model for the rubric LLM judge (default: {DEFAULT_MODEL}).",
-    )
-    parser.add_argument(
-        "--kind",
-        choices=("pr", "issue", "both"),
-        default="both",
-        help="Restrict the sweep to PRs or issues only (default: both).",
-    )
-    parser.add_argument(
-        "--only-pr",
-        type=int,
-        action="append",
-        default=[],
-        help="Limit the PR sweep to these PR numbers (repeat for several).",
-    )
-    parser.add_argument(
-        "--only-issue",
-        type=int,
-        action="append",
-        default=[],
-        help="Limit the issue sweep to these issue numbers (repeat for several).",
-    )
-    parser.add_argument(
-        "--ignore-existing-marker",
-        action="store_true",
-        help=(
-            "Re-post on PRs/issues that already carry the heads-up marker. "
-            "Useful for testing the comment wording on a known PR."
-        ),
-    )
-    args = parser.parse_args()
-
-    cutoff = args.close_on or (
-        dt.datetime.now(dt.timezone.utc).date() + dt.timedelta(days=DEFAULT_GRACE_DAYS)
-    )
-
-    kinds: tuple[str, ...]
-    if args.kind == "pr":
-        kinds = ("pr",)
-    elif args.kind == "issue":
-        kinds = ("issue",)
-    else:
-        kinds = ("pr", "issue")
-
-    only: dict[str, list[int]] = {}
-    if args.only_pr:
-        only["pr"] = args.only_pr
-    if args.only_issue:
-        only["issue"] = args.only_issue
-
-    # The script must NOT hit the LLM in dry-run if no key is set — we still
-    # want a useful preview that says "skip-no-llm-key" for items that would
-    # have been judged. Production runs require OPENAI_API_KEY.
-    if args.close and not os.environ.get("OPENAI_API_KEY"):
-        parser.error("OPENAI_API_KEY must be set for --close (real-run) mode.")
-
-    run(
-        repo=args.repo,
-        close=args.close,
-        cutoff=cutoff,
-        model=args.model,
-        kinds=kinds,
-        only_numbers=only or None,
-        skip_marker_check=args.ignore_existing_marker,
-    )
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

.github/scripts/triage_with_llm.py

@@ -582,7 +582,9 @@ def build_issue_prompt(*, title: str, body: str) -> str:
                   Commands whose external dependencies (LLM provider, DB,
                   network) are mocked or stubbed do NOT count.
               Prose-only "steps to reproduce" with no run output, video, or
-              screenshot do NOT satisfy (1).
+              screenshot do NOT satisfy (1). An unfilled template scaffold
+              (bare headings such as "Version or commit:" with nothing under
+              them, empty numbered lists) counts as absent, not as evidence.
           (2) Expected vs. actual behavior (`has_expected_vs_actual`).
 
         FAIL the bug report if either (1) or (2) is missing. Do not bias
@@ -595,6 +597,13 @@ def build_issue_prompt(*, title: str, body: str) -> str:
             that it does not today).
           - Motivation / use case with a concrete example (config, API call,
             UI flow, or scenario showing what's blocked today).
+          - END-TO-END EVIDENCE OF THE DEAD-END (set
+            `has_dead_end_evidence=true` only when this is present): a video,
+            a screenshot, or the exact command(s) actually run paired with
+            their real output, showing the point where the flow stops today.
+            Mocked or stubbed dependencies do NOT count, and an unfilled
+            template scaffold (bare headings, empty numbered lists) counts as
+            absent.
 
         For an issue that is neither a bug report nor a feature request (a
         question, support request, or discussion), PASS as long as it has a
@@ -608,6 +617,7 @@ def build_issue_prompt(*, title: str, body: str) -> str:
           "has_repro": boolean,
           "has_expected_vs_actual": boolean,
           "has_motivation_example": boolean,
+          "has_dead_end_evidence": boolean,
           "missing": ["plain-english strings naming what is missing"],
           "explanation": "1-2 sentence reasoning for the team to skim"
         }}
@@ -705,6 +715,10 @@ _ISSUE_BUG_LABELS: tuple[tuple[str, str], ...] = (
 )
 _ISSUE_FEATURE_LABELS: tuple[tuple[str, str], ...] = (
     ("has_motivation_example", "Motivation and concrete example"),
+    (
+        "has_dead_end_evidence",
+        "End-to-end evidence of the dead-end (video, screenshot, or command + real output)",
+    ),
 )
 
 
@@ -836,8 +850,11 @@ def format_issue_close_comment(verdict: dict) -> str:
         "video, a screenshot, or the exact commands you ran with their real output / "
         "traceback) plus expected vs. actual behavior. Written steps with no run output, "
         "video, or screenshot don't count, and mocked or stubbed runs don't count.\n"
-        "   - For **feature requests**: a concrete description of what should change, plus a "
-        "use case and example (config / API call / UI flow).\n"
+        "   - For **feature requests**: a concrete description of what should change, a "
+        "use case and example (config / API call / UI flow), plus end-to-end evidence of "
+        "the dead-end (a video, a screenshot, or the exact commands you ran with their "
+        "real output showing where the flow stops today). Mocked or stubbed runs don't "
+        "count.\n"
         "2. Comment `@agent-shin reconsider`. I'll re-run triage and reopen the issue if it "
         "now meets the bar. (GitHub doesn't let external authors reopen an issue a maintainer "
         "or bot closed, so the comment-based reconsider is the reliable path.)\n"
@@ -943,8 +960,10 @@ def format_grace_warning_issue_comment(verdict: dict) -> str:
         "screenshot, or the exact commands you ran with their real output / traceback) plus "
         "expected vs. actual behavior. Written steps with no run output don't count, and "
         "mocked or stubbed runs don't count.\n"
-        "- For **feature requests**: a concrete description of what should change, plus a use "
-        "case and example (config / API call / UI flow).\n"
+        "- For **feature requests**: a concrete description of what should change, a use "
+        "case and example (config / API call / UI flow), plus end-to-end evidence of the "
+        "dead-end (a video, a screenshot, or the exact commands you ran with their real "
+        "output showing where the flow stops today). Mocked or stubbed runs don't count.\n"
         "\n"
         "**If the issue does get auto-closed in 2 hours**, comment `@agent-shin reconsider` "
         "and I'll re-evaluate. If it now meets the bar, I'll reopen the issue.\n"

.github/workflows/_test-unit-base.yml

@@ -60,6 +60,9 @@ jobs:
     name: Run tests
     runs-on: ubuntu-latest
     timeout-minutes: ${{ inputs.job-timeout-minutes }}
+    permissions:
+      contents: read
+      pull-requests: read
     outputs:
       decision: ${{ steps.changes.outputs.decision }}
 
@@ -69,24 +72,27 @@ jobs:
         with:
           persist-credentials: false
 
-      - name: Detect backend-relevant changes
+      - name: Detect relevant changes
         id: changes
         timeout-minutes: 2
-        uses: ./.github/actions/detect-backend-changes
+        uses: ./.github/actions/detect-changes
 
       - name: Set up Python
+        if: steps.changes.outputs.decision != 'skip'
         timeout-minutes: 3
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
           python-version: "3.12"
 
       - name: Set up uv
+        if: steps.changes.outputs.decision != 'skip'
         timeout-minutes: 3
         uses: ./.github/actions/setup-uv-with-retries
         with:
           version: "0.10.9"
 
       - name: Cache uv dependencies
+        if: steps.changes.outputs.decision != 'skip'
         timeout-minutes: 5
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:

.github/workflows/auto_update_price_and_context_window.yml

@@ -23,7 +23,7 @@ jobs:
           version: "0.10.9"
       - name: Update JSON Data
         run: |
-          uv run --frozen --with 'aiohttp==3.13.3' python ".github/workflows/auto_update_price_and_context_window_file.py"
+          uv run --frozen --with 'aiohttp==3.13.3' python ".github/scripts/auto_update_price_and_context_window_file.py"
       - name: Regenerate JSON Schema
         run: |
           uv run --frozen python ci_cd/generate_model_prices_schema.py

.github/workflows/test-linting.yml

@@ -24,6 +24,7 @@ jobs:
     # re-running basedpyright over the merge-base tree.
     permissions:
       contents: read
+      pull-requests: read
       actions: read
 
     steps:
@@ -37,51 +38,65 @@ jobs:
           clean: true
           persist-credentials: false
 
+      - name: Detect relevant changes
+        id: changes
+        uses: ./.github/actions/detect-changes
+
       - name: Fetch gate base (merge-base with target branch)
+        if: steps.changes.outputs.decision != 'skip'
         env:
           GH_TOKEN: ${{ github.token }}
           BASE_SHA: ${{ github.event.pull_request.base.sha }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: |
-          MERGE_BASE=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
+          retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
+          MERGE_BASE=$(retry gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
           test -n "$MERGE_BASE"
-          git fetch --no-tags --depth=1 origin "$MERGE_BASE"
+          retry git fetch --no-tags --depth=1 origin "$MERGE_BASE"
           echo "GATE_BASE_SHA=$MERGE_BASE" >> "$GITHUB_ENV"
 
       - name: Set up Python
+        if: steps.changes.outputs.decision != 'skip'
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
           python-version: "3.12"
 
       - name: Set up uv
+        if: steps.changes.outputs.decision != 'skip'
         uses: ./.github/actions/setup-uv-with-retries
         with:
           version: "0.10.9"
 
       - name: Clean Python cache
+        if: steps.changes.outputs.decision != 'skip'
         run: |
           find . -type d -name "__pycache__" -exec rm -rf {} + || true
           find . -name "*.pyc" -delete || true
 
       - name: Check uv.lock is up to date
+        if: steps.changes.outputs.decision != 'skip'
         run: |
           uv lock --check || (echo "❌ uv.lock is out of sync with pyproject.toml. Run 'uv lock' locally and commit the result." && exit 1)
 
       - name: Install dependencies
+        if: steps.changes.outputs.decision != 'skip'
         run: |
           uv sync --frozen --group proxy-dev --group e2e-dev
 
       - name: Cache Prisma binaries
+        if: steps.changes.outputs.decision != 'skip'
         uses: ./.github/actions/cache-prisma-binaries
 
       # basedpyright resolves Prisma's generated client (litellm/proxy/schema.prisma)
       # only after `prisma generate` writes prisma/client.py et al. Without this the
       # DB wrappers typed against the generated client would degrade to Unknown.
       - name: Generate Prisma client
+        if: steps.changes.outputs.decision != 'skip'
         run: |
           uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 
       - name: Check ruff format
+        if: steps.changes.outputs.decision != 'skip'
         run: |
           git diff --name-only --diff-filter=ACMR "$GATE_BASE_SHA" HEAD -- 'litellm/**/*.py' | grep -v '^litellm/enterprise/' > "$RUNNER_TEMP/ruff_format_files.txt" || true
           if [ ! -s "$RUNNER_TEMP/ruff_format_files.txt" ]; then
@@ -91,6 +106,7 @@ jobs:
           xargs uv run --no-sync ruff format --check --exclude '/enterprise/' < "$RUNNER_TEMP/ruff_format_files.txt"
 
       - name: Debug - Check file state
+        if: steps.changes.outputs.decision != 'skip'
         run: |
           echo "Current branch:"
           git branch --show-current
@@ -100,30 +116,41 @@ jobs:
           head -50 litellm/litellm_core_utils/custom_logger_registry.py | tail -10
 
       - name: Run Ruff linting
+        if: steps.changes.outputs.decision != 'skip'
         run: |
           cd litellm
           uv run --no-sync ruff check .
           cd ..
 
       - name: Check strict-rule budget (delta vs base)
+        if: steps.changes.outputs.decision != 'skip'
         run: |
           uv run --no-sync python scripts/ruff_strict_gate.py --base "$GATE_BASE_SHA"
 
       - name: Check type-discipline budget (mutable collections / casts / type guards / kwargs / unexplained suppressions, delta vs base)
+        if: steps.changes.outputs.decision != 'skip'
         run: |
           uv run --no-sync python scripts/type_discipline_gate.py --base "$GATE_BASE_SHA"
 
+      - name: Check test-quality budget (zero-assert / mock-echo tests, sys.path.insert, raw env writes, litellm global mutation, delta vs base)
+        if: steps.changes.outputs.decision != 'skip'
+        run: |
+          uv run --no-sync python scripts/test_quality_gate.py --base "$GATE_BASE_SHA"
+
       - name: Print OpenAI version
+        if: steps.changes.outputs.decision != 'skip'
         run: |
           uv run --no-sync python -c "import openai; print(f'OpenAI version: {openai.__version__}')"
 
       - name: Check basedpyright budget (delta vs base)
+        if: steps.changes.outputs.decision != 'skip'
         env:
           GH_TOKEN: ${{ github.token }}
         run: |
           uv run --no-sync python scripts/type_check_gate.py --base "$GATE_BASE_SHA"
 
       - name: Check tests/e2e basedpyright (zero errors)
+        if: steps.changes.outputs.decision != 'skip'
         run: |
           if git diff --name-only --diff-filter=ACMRD "$GATE_BASE_SHA" HEAD -- 'tests/e2e/**/*.py' | grep -q .; then
             uv run --no-sync basedpyright tests/e2e
@@ -132,12 +159,14 @@ jobs:
           fi
 
       - name: Check for circular imports
+        if: steps.changes.outputs.decision != 'skip'
         run: |
           cd litellm
           uv run --no-sync python ../tests/documentation_tests/test_circular_imports.py
           cd ..
 
       - name: Check import safety
+        if: steps.changes.outputs.decision != 'skip'
         run: |
           uv run --no-sync python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1)
 
@@ -161,7 +190,8 @@ jobs:
         env:
           BASE_SHA: ${{ github.event.pull_request.base.sha }}
         run: |
-          git fetch --no-tags --depth=1 origin "$BASE_SHA"
+          retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
+          retry git fetch --no-tags --depth=1 origin "$BASE_SHA"
 
       - name: Set up Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
@@ -205,7 +235,8 @@ jobs:
           GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }}
         run: |
           if [ -n "$GITGUARDIAN_API_KEY" ]; then
-            git fetch --no-tags --unshallow origin
+            retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
+            retry git fetch --no-tags --unshallow origin
             uv tool run --from 'ggshield==1.48.0' ggshield secret scan repo .
           else
             echo "GITGUARDIAN_API_KEY not set, skipping ggshield scan"

.github/workflows/test-litellm-ui-build.yml

@@ -1,6 +1,7 @@
 name: UI Build Check
 permissions:
   contents: read
+  pull-requests: read
 
 on:
   pull_request:
@@ -28,7 +29,14 @@ jobs:
         with:
           persist-credentials: false
 
+      - name: Detect relevant changes
+        id: changes
+        uses: ./.github/actions/detect-changes
+        with:
+          category: ui
+
       - name: Setup Node.js
+        if: steps.changes.outputs.decision != 'skip'
         uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
         with:
           node-version-file: ui/litellm-dashboard/.nvmrc
@@ -36,7 +44,9 @@ jobs:
           cache-dependency-path: ui/litellm-dashboard/package-lock.json
 
       - name: Install dependencies
+        if: steps.changes.outputs.decision != 'skip'
         run: npm ci
 
       - name: Build
+        if: steps.changes.outputs.decision != 'skip'
         run: npm run build

.github/workflows/test-litellm-ui-unit.yml

@@ -1,6 +1,7 @@
 name: UI Unit Tests
 permissions:
   contents: read
+  pull-requests: read
 
 on:
   pull_request:
@@ -32,7 +33,14 @@ jobs:
           fetch-depth: 1
           persist-credentials: false
 
+      - name: Detect relevant changes
+        id: changes
+        uses: ./.github/actions/detect-changes
+        with:
+          category: ui
+
       - name: Setup Node.js
+        if: steps.changes.outputs.decision != 'skip'
         uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
         with:
           node-version-file: ui/litellm-dashboard/.nvmrc
@@ -40,31 +48,50 @@ jobs:
           cache-dependency-path: ui/litellm-dashboard/package-lock.json
 
       - name: Install dependencies
+        if: steps.changes.outputs.decision != 'skip'
         run: npm ci
 
+      - name: Run UI type tests (Vitest)
+        if: steps.changes.outputs.decision != 'skip'
+        env:
+          CI: "true"
+        run: npm run test:types
+
       - name: Run UI unit tests (Vitest)
+        if: steps.changes.outputs.decision != 'skip'
         env:
           CI: "true"
           GH_TOKEN: ${{ github.token }}
           BASE_SHA: ${{ github.event.pull_request.base.sha }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: |
-          if [ -n "$BASE_SHA" ]; then
-            merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
-            test -n "$merge_base"
-            git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
-            changed_files=()
-            while IFS= read -r f; do
-              changed_files+=("$f")
-            done < <(git diff --name-only --relative "$merge_base" "$HEAD_SHA" -- .)
-            if [ ${#changed_files[@]} -eq 0 ]; then
-              echo "No UI files changed in this PR; skipping unit tests."
-              exit 0
-            fi
-            echo "Pull request: running tests related to ${#changed_files[@]} changed UI files"
-            npm run test -- related "${changed_files[@]}" --run --passWithNoTests \
-              --pool forks --poolOptions.forks.maxForks=14
-          else
+          full_suite() { npm run test -- --run --pool forks --poolOptions.forks.maxForks=14; }
+
+          if [ -z "$BASE_SHA" ]; then
             echo "Push to $GITHUB_REF_NAME: running the full suite"
-            npm run test -- --run --pool forks --poolOptions.forks.maxForks=14
+            full_suite
+            exit 0
           fi
+
+          merge_base=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
+          test -n "$merge_base"
+          git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
+          changed_files=()
+          while IFS= read -r f; do
+            changed_files+=("$f")
+          done < <(git diff --name-only --relative "$merge_base" "$HEAD_SHA" -- .)
+          if [ ${#changed_files[@]} -eq 0 ]; then
+            echo "No UI files changed in this PR; skipping unit tests."
+            exit 0
+          fi
+
+          scope=$(printf '%s\n' "${changed_files[@]}" | bash "$GITHUB_WORKSPACE/.github/scripts/select_ui_test_scope.sh")
+          if [ "$scope" != related ]; then
+            echo "Pull request: ${#changed_files[@]} changed UI files reach outside src/, so related would miss their dependents; running the full suite"
+            full_suite
+            exit 0
+          fi
+
+          echo "Pull request: running tests related to ${#changed_files[@]} changed UI files"
+          npm run test -- related "${changed_files[@]}" --run --passWithNoTests \
+            --pool forks --poolOptions.forks.maxForks=14

.github/workflows/test-mcp.yml

@@ -10,6 +10,7 @@ on:
 
 permissions:
   contents: read
+  pull-requests: read
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
@@ -25,26 +26,34 @@ jobs:
         with:
           persist-credentials: false
 
+      - name: Detect relevant changes
+        id: changes
+        uses: ./.github/actions/detect-changes
+
       - name: Thank You Message
         run: |
           echo "### 🙏 Thank you for contributing to LiteLLM!" >> $GITHUB_STEP_SUMMARY
           echo "Your PR is being tested now. We appreciate your help in making LiteLLM better!" >> $GITHUB_STEP_SUMMARY
 
       - name: Set up Python
+        if: steps.changes.outputs.decision != 'skip'
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
           python-version: "3.12"
 
       - name: Set up uv
+        if: steps.changes.outputs.decision != 'skip'
         uses: ./.github/actions/setup-uv-with-retries
         with:
           version: "0.10.9"
 
       - name: Install dependencies
+        if: steps.changes.outputs.decision != 'skip'
         run: |
           uv lock --check
           .github/scripts/uv_sync_with_retries.sh --frozen --group proxy-dev --extra proxy --extra semantic-router
 
       - name: Run MCP tests
+        if: steps.changes.outputs.decision != 'skip'
         run: |
           uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov-report=xml --durations=5

.github/workflows/test-terraform-modules.yml

@@ -0,0 +1,54 @@
+name: Terraform Modules
+
+on:
+  push:
+    paths:
+      - "terraform/litellm/aws/**"
+      - ".github/workflows/test-terraform-modules.yml"
+  pull_request:
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_staging
+      - "litellm_**"
+    paths:
+      - "terraform/litellm/aws/**"
+      - ".github/workflows/test-terraform-modules.yml"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  aws-module:
+    name: fmt, validate, test (aws)
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    defaults:
+      run:
+        working-directory: terraform/litellm/aws
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2
+        with:
+          terraform_version: 1.13.3
+          terraform_wrapper: false
+
+      - name: fmt
+        run: terraform fmt -recursive -check -diff
+
+      - name: init
+        run: terraform init -backend=false -input=false
+
+      - name: validate
+        run: terraform validate
+
+      # Plan-only, mock_provider-backed: no AWS credentials, no API calls.
+      - name: test
+        run: terraform test

.github/workflows/test-unit-core-utils.yml

@@ -1,31 +0,0 @@
-name: "Unit Tests: Core Utilities"
-
-on:
-  pull_request:
-    branches:
-      - main
-      - litellm_internal_staging
-      - litellm_oss_staging
-      - "litellm_**"
-  push:
-    branches:
-      - main
-      - litellm_internal_staging
-
-permissions:
-  contents: read
-  id-token: write
-  pull-requests: write
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
-
-jobs:
-  core-utils:
-    uses: ./.github/workflows/_test-unit-base.yml
-    with:
-      test-path: "tests/test_litellm/litellm_core_utils"
-      workers: 2
-      reruns: 1
-      artifact-name: core-utils

.github/workflows/test-unit-documentation.yml

@@ -23,34 +23,41 @@ jobs:
   documentation:
     runs-on: ubuntu-latest
     timeout-minutes: 10
+    permissions:
+      contents: read
+      pull-requests: read
 
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
           persist-credentials: false
 
+      - name: Detect relevant changes
+        id: changes
+        uses: ./.github/actions/detect-changes
+
       - name: Checkout litellm-docs into docs/my-website (for documentation_tests)
+        if: steps.changes.outputs.decision != 'skip'
         uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
           repository: BerriAI/litellm-docs
           path: docs/my-website
           persist-credentials: false
 
-      - name: Detect backend-relevant changes
-        id: changes
-        uses: ./.github/actions/detect-backend-changes
-
       - name: Set up Python
+        if: steps.changes.outputs.decision != 'skip'
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
           python-version: "3.12"
 
       - name: Set up uv
+        if: steps.changes.outputs.decision != 'skip'
         uses: ./.github/actions/setup-uv-with-retries
         with:
           version: "0.10.9"
 
       - name: Cache uv dependencies
+        if: steps.changes.outputs.decision != 'skip'
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:
           path: |

.github/workflows/test-unit-enterprise-routing.yml

@@ -1,35 +0,0 @@
-name: "Unit Tests: Enterprise, Google GenAI & Routing"
-
-on:
-  pull_request:
-    branches:
-      - main
-      - litellm_internal_staging
-      - litellm_oss_staging
-      - "litellm_**"
-  push:
-    branches:
-      - main
-      - litellm_internal_staging
-
-permissions:
-  contents: read
-  id-token: write
-  pull-requests: write
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
-
-jobs:
-  enterprise-routing:
-    uses: ./.github/workflows/_test-unit-base.yml
-    with:
-      test-path: >-
-        tests/test_litellm/enterprise
-        tests/test_litellm/google_genai
-        tests/test_litellm/router_utils
-        tests/test_litellm/router_strategy
-      workers: 2
-      reruns: 2
-      artifact-name: enterprise-routing

.github/workflows/test-unit-integrations.yml

@@ -1,31 +0,0 @@
-name: "Unit Tests: Integrations (Callbacks & Logging)"
-
-on:
-  pull_request:
-    branches:
-      - main
-      - litellm_internal_staging
-      - litellm_oss_staging
-      - "litellm_**"
-  push:
-    branches:
-      - main
-      - litellm_internal_staging
-
-permissions:
-  contents: read
-  id-token: write
-  pull-requests: write
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
-
-jobs:
-  integrations:
-    uses: ./.github/workflows/_test-unit-base.yml
-    with:
-      test-path: "tests/test_litellm/integrations"
-      workers: 2
-      reruns: 3
-      artifact-name: integrations

.github/workflows/test-unit-llm-providers.yml

@@ -1,47 +0,0 @@
-name: "Unit Tests: LLM Provider Transformations"
-
-on:
-  pull_request:
-    branches:
-      - main
-      - litellm_internal_staging
-      - litellm_oss_staging
-      - "litellm_**"
-  push:
-    branches:
-      - main
-      - litellm_internal_staging
-
-permissions:
-  contents: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
-
-jobs:
-  vertex-ai:
-    name: Vertex AI
-    permissions:
-      contents: read
-      id-token: write
-      pull-requests: write
-    uses: ./.github/workflows/_test-unit-base.yml
-    with:
-      test-path: "tests/test_litellm/llms/vertex_ai"
-      workers: 1
-      reruns: 2
-      artifact-name: llm-vertex-ai
-
-  other-providers:
-    name: All Other Providers
-    permissions:
-      contents: read
-      id-token: write
-      pull-requests: write
-    uses: ./.github/workflows/_test-unit-base.yml
-    with:
-      test-path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai"
-      workers: 2
-      reruns: 2
-      artifact-name: llm-other-providers

.github/workflows/test-unit-misc.yml

@@ -1,53 +0,0 @@
-name: "Unit Tests: MCP, Secrets, Containers & Misc"
-
-on:
-  pull_request:
-    branches:
-      - main
-      - litellm_internal_staging
-      - litellm_oss_staging
-      - "litellm_**"
-  push:
-    branches:
-      - main
-      - litellm_internal_staging
-
-permissions:
-  contents: read
-  id-token: write
-  pull-requests: write
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
-
-jobs:
-  misc:
-    uses: ./.github/workflows/_test-unit-base.yml
-    with:
-      test-path: >-
-        tests/test_litellm/batches
-        tests/test_litellm/secret_managers
-        tests/test_litellm/a2a_protocol
-        tests/test_litellm/anthropic_interface
-        tests/test_litellm/completion_extras
-        tests/test_litellm/compression
-        tests/test_litellm/containers
-        tests/test_litellm/experimental_mcp_client
-        tests/test_litellm/models
-        tests/test_litellm/repositories
-        tests/test_litellm/images
-        tests/test_litellm/interactions
-        tests/test_litellm/ocr
-        tests/test_litellm/passthrough
-        tests/test_litellm/rag
-        tests/test_litellm/realtime_api
-        tests/test_litellm/rerank_api
-        tests/test_litellm/sandbox
-        tests/test_litellm/test_router
-        tests/test_litellm/vector_stores
-        tests/test_litellm/videos
-        tests/test_litellm/test_*.py
-      workers: 2
-      reruns: 2
-      artifact-name: misc

.github/workflows/test-unit-proxy-auth.yml

@@ -1,31 +0,0 @@
-name: "Unit Tests: Proxy Auth & Key Management"
-
-on:
-  pull_request:
-    branches:
-      - main
-      - litellm_internal_staging
-      - litellm_oss_staging
-      - "litellm_**"
-  push:
-    branches:
-      - main
-      - litellm_internal_staging
-
-permissions:
-  contents: read
-  id-token: write
-  pull-requests: write
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
-
-jobs:
-  proxy-auth:
-    uses: ./.github/workflows/_test-unit-base.yml
-    with:
-      test-path: "tests/test_litellm/proxy/auth tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine tests/test_litellm/proxy/client"
-      workers: 2
-      reruns: 2
-      artifact-name: proxy-auth

.github/workflows/test-unit-proxy-db.yml

@@ -135,8 +135,6 @@ jobs:
             test-path: >-
               tests/proxy_unit_tests/test_proxy_server.py
               tests/proxy_unit_tests/test_proxy_server_keys.py
-              tests/proxy_unit_tests/test_proxy_server_caching.py
-              tests/proxy_unit_tests/test_proxy_server_langfuse.py
               tests/proxy_unit_tests/test_proxy_server_spend.py
               tests/proxy_unit_tests/test_aproxy_startup.py
             workers: 4

.github/workflows/test-unit-proxy-endpoints.yml

@@ -1,80 +0,0 @@
-name: "Unit Tests: Proxy API Endpoints"
-
-on:
-  pull_request:
-    branches:
-      - main
-      - litellm_internal_staging
-      - litellm_oss_staging
-      - "litellm_**"
-  push:
-    branches:
-      - main
-      - litellm_internal_staging
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
-
-jobs:
-  proxy-endpoints:
-    permissions:
-      contents: read
-      id-token: write
-      pull-requests: write
-    uses: ./.github/workflows/_test-unit-base.yml
-    with:
-      test-path: >-
-        tests/test_litellm/proxy/analytics_endpoints
-        tests/test_litellm/proxy/management_endpoints
-        tests/test_litellm/proxy/memory
-        tests/test_litellm/proxy/guardrails
-        tests/test_litellm/proxy/management_helpers
-        tests/test_litellm/proxy/anthropic_endpoints
-        tests/test_litellm/proxy/google_endpoints
-        tests/test_litellm/proxy/openai_files_endpoint
-        tests/test_litellm/proxy/batches_endpoints
-        tests/test_litellm/proxy/fine_tuning_endpoints
-        tests/test_litellm/proxy/vector_store_files_endpoints
-        tests/test_litellm/proxy/video_endpoints
-        tests/test_litellm/proxy/response_api_endpoints
-        tests/test_litellm/proxy/image_endpoints
-        tests/test_litellm/proxy/vector_store_endpoints
-        tests/test_litellm/proxy/agent_endpoints
-        tests/test_litellm/proxy/a2a
-        tests/test_litellm/proxy/credential_endpoints
-        tests/test_litellm/proxy/discovery_endpoints
-        tests/test_litellm/proxy/health_endpoints
-        tests/test_litellm/proxy/shutdown
-        tests/test_litellm/proxy/public_endpoints
-        tests/test_litellm/proxy/prompts
-        tests/test_litellm/proxy/rag_endpoints
-        tests/test_litellm/proxy/realtime_endpoints
-        tests/test_litellm/proxy/ui_crud_endpoints
-        tests/test_litellm/proxy/config_resolvers
-        tests/test_litellm/proxy/utils
-      workers: 2
-      reruns: 2
-      artifact-name: proxy-endpoints
-
-  # Behavior-pinning tests for litellm/proxy/proxy_server.py. Owns its
-  # own job (not a path on the proxy-endpoints job above) so its budget
-  # is independent and its coverage artifact is uploaded separately.
-  # See: https://www.notion.so/36c43b8acdab81ee845fd5365128a2fc
-  proxy-server:
-    permissions:
-      contents: read
-      id-token: write
-      pull-requests: write
-    uses: ./.github/workflows/_test-unit-base.yml
-    with:
-      test-path: tests/test_litellm/proxy/proxy_server
-      workers: 4
-      reruns: 2
-      timeout-minutes: 60
-      job-timeout-minutes: 95
-      artifact-name: proxy-server

.github/workflows/test-unit-proxy-infra.yml

@@ -1,42 +0,0 @@
-name: "Unit Tests: Proxy Infrastructure"
-
-on:
-  pull_request:
-    branches:
-      - main
-      - litellm_internal_staging
-      - litellm_oss_staging
-      - "litellm_**"
-  push:
-    branches:
-      - main
-      - litellm_internal_staging
-
-permissions:
-  contents: read
-  id-token: write
-  pull-requests: write
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
-
-jobs:
-  proxy-infra:
-    uses: ./.github/workflows/_test-unit-base.yml
-    with:
-      test-path: >-
-        tests/test_litellm/proxy/db
-        tests/test_litellm/proxy/middleware
-        tests/test_litellm/proxy/spend_tracking
-        tests/test_litellm/proxy/pass_through_endpoints
-        tests/test_litellm/proxy/_experimental
-        tests/test_litellm/proxy/experimental
-        tests/test_litellm/proxy/common_utils
-        tests/test_litellm/proxy/enterprise_billing
-        tests/test_litellm/proxy/types_utils
-        tests/test_litellm/proxy/logging_endpoints
-        tests/test_litellm/proxy/test_*.py
-      workers: 2
-      reruns: 2
-      artifact-name: proxy-infra

.github/workflows/test-unit-proxy-legacy.yml

@@ -1,106 +0,0 @@
-name: "Unit Tests: Proxy Legacy Tests"
-
-on:
-  pull_request:
-    branches:
-      - main
-      - litellm_internal_staging
-      - litellm_oss_staging
-      - "litellm_**"
-  push:
-    branches:
-      - main
-      - litellm_internal_staging
-
-permissions:
-  contents: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
-    strategy:
-      fail-fast: false
-      matrix:
-        test-group:
-          - name: "auth-and-jwt"
-            path: "tests/proxy_unit_tests/test_[a-j]*.py"
-          - name: "key-generation"
-            path: "tests/proxy_unit_tests/test_[k-o]*.py"
-          - name: "proxy-config"
-            path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py"
-          - name: "proxy-server"
-            path: "tests/proxy_unit_tests/test_proxy_server.py"
-          - name: "proxy-server-extras"
-            path: "tests/proxy_unit_tests/test_proxy_server_*.py tests/proxy_unit_tests/test_proxy_setting_guardrails.py"
-          - name: "proxy-utils"
-            path: "tests/proxy_unit_tests/test_proxy_utils.py"
-          - name: "proxy-token-counter"
-            path: "tests/proxy_unit_tests/test_proxy_token_counter.py"
-          - name: "proxy-response-and-misc"
-            path: "tests/proxy_unit_tests/test_[r-t]*.py"
-          - name: "proxy-user-auth-and-spend"
-            path: "tests/proxy_unit_tests/test_[u-z]*.py"
-
-    name: ${{ matrix.test-group.name }}
-
-    steps:
-      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-
-      - name: Detect backend-relevant changes
-        id: changes
-        uses: ./.github/actions/detect-backend-changes
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: "3.12"
-
-      - name: Set up uv
-        uses: ./.github/actions/setup-uv-with-retries
-        with:
-          version: "0.10.9"
-
-      - name: Cache uv dependencies
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
-        with:
-          path: |
-            ~/.cache/uv
-            .venv
-          key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-uv-
-
-      - name: Install dependencies
-        if: steps.changes.outputs.decision != 'skip'
-        run: |
-          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
-
-      - name: Cache Prisma binaries
-        if: steps.changes.outputs.decision != 'skip'
-        uses: ./.github/actions/cache-prisma-binaries
-
-      - name: Generate Prisma client
-        if: steps.changes.outputs.decision != 'skip'
-        run: |
-          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
-
-      - name: Run tests - ${{ matrix.test-group.name }}
-        if: steps.changes.outputs.decision != 'skip'
-        env:
-          TEST_PATH: ${{ matrix.test-group.path }}
-        run: |
-          uv run --no-sync pytest ${TEST_PATH} \
-            --tb=short -vv \
-            --maxfail=10 \
-            -n 2 \
-            --reruns 1 \
-            --reruns-delay 1 \
-            --dist=loadscope \
-            --durations=20

.github/workflows/test-unit-responses-caching-types.yml

@@ -1,31 +0,0 @@
-name: "Unit Tests: Responses, Caching & Types"
-
-on:
-  pull_request:
-    branches:
-      - main
-      - litellm_internal_staging
-      - litellm_oss_staging
-      - "litellm_**"
-  push:
-    branches:
-      - main
-      - litellm_internal_staging
-
-permissions:
-  contents: read
-  id-token: write
-  pull-requests: write
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
-
-jobs:
-  responses-caching-types:
-    uses: ./.github/workflows/_test-unit-base.yml
-    with:
-      test-path: "tests/test_litellm/responses tests/test_litellm/caching tests/test_litellm/types"
-      workers: 2
-      reruns: 2
-      artifact-name: responses-caching-types

.github/workflows/test-unit.yml

@@ -0,0 +1,219 @@
+name: "Unit Tests"
+
+on:
+  pull_request:
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_staging
+      - "litellm_**"
+  push:
+    branches:
+      - main
+      - litellm_internal_staging
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+# One caller for every tests/test_litellm shard, replacing the nine thin workflow
+# files that each wrapped a single call to _test-unit-base.yml. Adding a shard is
+# now one matrix entry rather than a new file.
+#
+# `name` is the shard id and nothing else, so each check reports as
+# "<shard> / Run tests" exactly as it did when the shard had its own file. Those
+# strings are the branch ruleset's required contexts, so they are load-bearing:
+# renaming an entry renames a required check and the ruleset stops matching it.
+#
+# Every entry states its timeouts even when they equal the base workflow's
+# defaults. An absent matrix key renders as an empty string, which is not a
+# number, so a partially-specified entry would fail the call rather than fall
+# back to the default.
+#
+# tests/proxy_unit_tests keeps its own caller (test-unit-proxy-db.yml): it is
+# already a matrix and carries a shard-coverage guard that reads that file by
+# name. Folding it in here is a follow-up, together with generalising that guard
+# into assert_ci_coverage.py.
+jobs:
+  unit:
+    name: ${{ matrix.shard }}
+    permissions:
+      contents: read
+      id-token: write
+      pull-requests: write
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - shard: core-utils
+            artifact-name: core-utils
+            test-path: "tests/test_litellm/litellm_core_utils"
+            workers: 2
+            reruns: 1
+            timeout-minutes: 20
+            job-timeout-minutes: 55
+
+          - shard: enterprise-routing
+            artifact-name: enterprise-routing
+            test-path: >-
+              tests/test_litellm/enterprise
+              tests/test_litellm/google_genai
+              tests/test_litellm/router_utils
+              tests/test_litellm/router_strategy
+            workers: 2
+            reruns: 2
+            timeout-minutes: 20
+            job-timeout-minutes: 55
+
+          - shard: integrations
+            artifact-name: integrations
+            test-path: "tests/test_litellm/integrations"
+            workers: 2
+            reruns: 3
+            timeout-minutes: 20
+            job-timeout-minutes: 55
+
+          - shard: Vertex AI
+            artifact-name: llm-vertex-ai
+            test-path: "tests/test_litellm/llms/vertex_ai"
+            workers: 1
+            reruns: 2
+            timeout-minutes: 20
+            job-timeout-minutes: 55
+
+          - shard: All Other Providers
+            artifact-name: llm-other-providers
+            test-path: "tests/test_litellm/llms --ignore=tests/test_litellm/llms/vertex_ai"
+            workers: 2
+            reruns: 2
+            timeout-minutes: 20
+            job-timeout-minutes: 55
+
+          - shard: misc
+            artifact-name: misc
+            test-path: >-
+              tests/test_litellm/batches
+              tests/test_litellm/secret_managers
+              tests/test_litellm/a2a_protocol
+              tests/test_litellm/anthropic_interface
+              tests/test_litellm/completion_extras
+              tests/test_litellm/compression
+              tests/test_litellm/containers
+              tests/test_litellm/experimental_mcp_client
+              tests/test_litellm/models
+              tests/test_litellm/repositories
+              tests/test_litellm/images
+              tests/test_litellm/interactions
+              tests/test_litellm/ocr
+              tests/test_litellm/passthrough
+              tests/test_litellm/rag
+              tests/test_litellm/realtime_api
+              tests/test_litellm/rerank_api
+              tests/test_litellm/sandbox
+              tests/test_litellm/test_router
+              tests/test_litellm/vector_stores
+              tests/test_litellm/videos
+              tests/test_litellm/test_*.py
+            workers: 2
+            reruns: 2
+            timeout-minutes: 20
+            job-timeout-minutes: 55
+
+          - shard: proxy-auth
+            artifact-name: proxy-auth
+            test-path: >-
+              tests/test_litellm/proxy/auth
+              tests/test_litellm/proxy/hooks
+              tests/test_litellm/proxy/policy_engine
+              tests/test_litellm/proxy/client
+            workers: 2
+            reruns: 2
+            timeout-minutes: 20
+            job-timeout-minutes: 55
+
+          - shard: proxy-endpoints
+            artifact-name: proxy-endpoints
+            test-path: >-
+              tests/test_litellm/proxy/analytics_endpoints
+              tests/test_litellm/proxy/management_endpoints
+              tests/test_litellm/proxy/memory
+              tests/test_litellm/proxy/guardrails
+              tests/test_litellm/proxy/management_helpers
+              tests/test_litellm/proxy/anthropic_endpoints
+              tests/test_litellm/proxy/google_endpoints
+              tests/test_litellm/proxy/openai_files_endpoint
+              tests/test_litellm/proxy/batches_endpoints
+              tests/test_litellm/proxy/fine_tuning_endpoints
+              tests/test_litellm/proxy/vector_store_files_endpoints
+              tests/test_litellm/proxy/video_endpoints
+              tests/test_litellm/proxy/response_api_endpoints
+              tests/test_litellm/proxy/image_endpoints
+              tests/test_litellm/proxy/ocr_endpoints
+              tests/test_litellm/proxy/vector_store_endpoints
+              tests/test_litellm/proxy/agent_endpoints
+              tests/test_litellm/proxy/a2a
+              tests/test_litellm/proxy/credential_endpoints
+              tests/test_litellm/proxy/discovery_endpoints
+              tests/test_litellm/proxy/health_endpoints
+              tests/test_litellm/proxy/shutdown
+              tests/test_litellm/proxy/public_endpoints
+              tests/test_litellm/proxy/prompts
+              tests/test_litellm/proxy/rag_endpoints
+              tests/test_litellm/proxy/realtime_endpoints
+              tests/test_litellm/proxy/ui_crud_endpoints
+              tests/test_litellm/proxy/config_resolvers
+              tests/test_litellm/proxy/utils
+            workers: 2
+            reruns: 2
+            timeout-minutes: 20
+            job-timeout-minutes: 55
+
+          - shard: proxy-server
+            artifact-name: proxy-server
+            test-path: "tests/test_litellm/proxy/proxy_server"
+            workers: 4
+            reruns: 2
+            timeout-minutes: 60
+            job-timeout-minutes: 95
+
+          - shard: proxy-infra
+            artifact-name: proxy-infra
+            test-path: >-
+              tests/test_litellm/proxy/db
+              tests/test_litellm/proxy/middleware
+              tests/test_litellm/proxy/spend_tracking
+              tests/test_litellm/proxy/pass_through_endpoints
+              tests/test_litellm/proxy/_experimental
+              tests/test_litellm/proxy/experimental
+              tests/test_litellm/proxy/common_utils
+              tests/test_litellm/proxy/enterprise_billing
+              tests/test_litellm/proxy/types_utils
+              tests/test_litellm/proxy/logging_endpoints
+              tests/test_litellm/proxy/test_*.py
+            workers: 2
+            reruns: 2
+            timeout-minutes: 20
+            job-timeout-minutes: 55
+
+          - shard: responses-caching-types
+            artifact-name: responses-caching-types
+            test-path: >-
+              tests/test_litellm/responses
+              tests/test_litellm/caching
+              tests/test_litellm/types
+            workers: 2
+            reruns: 2
+            timeout-minutes: 20
+            job-timeout-minutes: 55
+    uses: ./.github/workflows/_test-unit-base.yml
+    with:
+      test-path: ${{ matrix.test-path }}
+      workers: ${{ matrix.workers }}
+      reruns: ${{ matrix.reruns }}
+      timeout-minutes: ${{ matrix.timeout-minutes }}
+      job-timeout-minutes: ${{ matrix.job-timeout-minutes }}
+      artifact-name: ${{ matrix.artifact-name }}

.github/workflows/triage_rollout_heads_up.yml

@@ -1,92 +0,0 @@
-name: Agent Shin — rollout heads-up (one-shot)
-
-# Fires the 7-day heads-up comment on every open external PR/issue that the
-# new triage bot would auto-close. The real sweep is a deliberate one-shot:
-# trigger it at rollout via a manual `workflow_dispatch` with `dry_run=false`.
-# The script is idempotent (skips items that already carry the
-# `<!-- agent-shin:rollout-heads-up -->` marker), so a re-run is harmless.
-#
-# The automatic push trigger runs DRY-RUN only, so merging the script to
-# `litellm_internal_staging` never posts a comment; it just confirms the
-# workflow is wired up. Posting real comments requires the manual dispatch,
-# which is also the only trigger that exposes `OPENAI_API_KEY`. The heads-up
-# is intentionally NOT gated on `AGENT_SHIN_ENABLED`: it has to warn
-# contributors while that flag is still off, ahead of the flip that turns on
-# auto-closing.
-#
-# The workflow is a thin shell over `.github/scripts/triage_rollout_heads_up.py`.
-# Dry-run vs. real run differ in EXACTLY one CLI flag (`--close`), added only
-# on a manual dispatch with `dry_run=false`.
-
-on:
-  push:
-    branches:
-      - litellm_internal_staging
-    paths:
-      # The presence of this script on staging IS the rollout merge marker.
-      # Editing the file later would re-fire the workflow; that's safe because
-      # the script skips PRs/issues that already have the heads-up marker.
-      - ".github/scripts/triage_rollout_heads_up.py"
-  workflow_dispatch:
-    inputs:
-      dry_run:
-        description: "Dry run (true = preview only, false = actually post comments)."
-        required: false
-        default: "true"
-        type: choice
-        options:
-          - "true"
-          - "false"
-
-permissions:
-  contents: read
-  issues: write
-  pull-requests: write
-
-jobs:
-  heads-up:
-    if: github.repository == 'BerriAI/litellm'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout triage scripts
-        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          sparse-checkout: .github/scripts
-          persist-credentials: false
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: "3.12"
-
-      - name: Install LLM client
-        run: pip install --no-cache-dir --require-hashes -r .github/scripts/triage-requirements.txt
-
-      - name: Run heads-up sweep
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          # Only the manual dispatch (the real-run trigger) needs the LLM key.
-          # The automatic push trigger runs dry-run and never posts, so it gets
-          # no key. Mirrors the sibling triage workflows, which expose the key
-          # only on an enabled/dispatched run rather than unconditionally.
-          OPENAI_API_KEY: ${{ github.event_name == 'workflow_dispatch' && secrets.OPENAI_API_KEY || '' }}
-          OPENAI_BASE_URL: ${{ vars.OPENAI_BASE_URL }}
-          TRIAGE_MODEL: ${{ vars.TRIAGE_MODEL }}
-          # The real run is a deliberate manual dispatch with dry_run=false.
-          # Use the EXACT "false" comparison so any unexpected input value
-          # fail-closes to dry-run (mirrors the AGENT_SHIN_ENABLED pattern in
-          # the sibling workflows). The automatic push trigger always stays
-          # dry-run, so merging the script never posts.
-          DRY_RUN_INPUT: ${{ github.event.inputs.dry_run }}
-        run: |
-          set -euo pipefail
-          ARGS=(--repo "${{ github.repository }}")
-          if [ "${GITHUB_EVENT_NAME:-}" = "workflow_dispatch" ] && [ "${DRY_RUN_INPUT:-true}" = "false" ]; then
-            ARGS+=(--close)
-            echo "::notice::Manual rollout dispatch with dry_run=false -> heads-up comments WILL be posted."
-          elif [ "${GITHUB_EVENT_NAME:-}" = "workflow_dispatch" ]; then
-            echo "::notice::Manual dispatch in dry-run mode -> previewing only, no comments will be posted."
-          else
-            echo "::notice::Automatic push trigger -> dry-run preview only. Fire the real rollout sweep with a manual workflow_dispatch (dry_run=false)."
-          fi
-          python3 .github/scripts/triage_rollout_heads_up.py "${ARGS[@]}"

.gitignore

@@ -1,9 +1,11 @@
 .python-version
 .venv
+tests/e2e/.fixtures/
 .venv-typecheck
 .venv_policy_test
 .env
 .claude
+CLAUDE.local.md
 .newenv
 newenv/*
 litellm/proxy/myenv/*

CLAUDE.md

@@ -29,7 +29,9 @@ End-to-end tests belong in `tests/e2e/` and must follow the harness conventions
 
 When creating PRs, don't set base to `main`. `litellm_internal_staging` is the default base branch and serves that purpose for both internal and external / OSS contributions
 
-When writing a PR body, treat the comments and imperative instructions inside @.github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule
+When writing a PR body, treat the comments and imperative instructions inside .github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule
+
+Same applies for filing bug reports and feature requests, with .github/ISSUE_TEMPLATE/bug_report.yml and .github/ISSUE_TEMPLATE/feature_request.yml, respectively
 
 If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just leave the section blank
 
@@ -51,6 +53,8 @@ When you fix violations gated by `ruff-strict-budget.json`, `type-discipline-bud
 
 `make check` (f.k.a. `make pre-commit`, which still works identically as an alias) saves its complete output to a log file in .git (overwriting previous logs) and prints that path as its first and last output lines. To inspect a run, read or grep that log instead of re-running the multi-minute checks just to see a different slice
 
+`make check`, `make lint`, `scripts/pre_commit_lint.sh`, and the standalone budget gates (`scripts/ruff_strict_gate.py`, `scripts/type_discipline_gate.py`, `scripts/type_check_gate.py`) each hold one of 2 machine-wide slots, so when other sessions or worktrees on the same box are already running heavy work, yours prints "all N machine-wide slots are busy; queueing" and then stays quiet until a slot frees. Give the command a long timeout and let it wait rather than killing it, retrying it, or assuming it hung. Don't change the # of machine-wide slots or make it unlimited by setting `LITELLM_GATE_SLOTS=0`
+
 If you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and pushing `reportAny` / `reportExplicitAny` closer to their basedpyright ceilings, just validate it in the caller with Pydantic (a model or `TypeAdapter` that returns the typed thing or raises will do) and then pass the now typed variable in
 
 If you get an LIT001 or LIT002 fail, refactor the code to follow functional programming best practices rather than introducing mutable data structures. For example, build values in one shot with comprehensions or generators wrapped in `tuple()` / `MappingProxyType()` / `frozenset()` instead of seeding an empty `list`/`dict`/`set` and mutating it over time. Ideally, `# mutable-ok` is never used; reach for it only as a genuine last resort when an immutable rewrite is truly impossible, and always pair it with a real reason
@@ -81,7 +85,8 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega
 - Never-nester: early returns over deep nesting
 - Don't throw; model failures as values (One function (e.g., raise_public) maps error union to existing public exception contracts via exhaustive match + assert_never)
 - No mutation; don't reassign variables, global or local. Instead of mutable lists and dicts, prefer tuples, frozen dataclasses (with slots=True), `MappingProxyType`, etc.
-  - Annotate every variable with `: Final` (LIT010). Unpacking and walrus targets cannot carry the annotation, so they are implicitly final. Don't rebind them. Never rebind or mutate function parameters (LIT011); `self`/`cls` attribute stores are the exception. If rebinding or in-place mutation is truly unavoidable, suppress with `# rebind-ok: <reason>` explaining why
+  - Annotate every variable with `: Final` (LIT010). Unpacking and walrus targets cannot carry the annotation, so they are implicitly final. Don't rebind them. Never rebind or mutate function parameters (LIT011); `self`/`cls` attribute stores are the exception. If rebinding or in-place mutation is truly unavoidable, suppress with `# rebind-ok: <reason>`
+  - Qualify every TypedDict field with `ReadOnly[...]` (LIT012), which nests freely with `Required` / `NotRequired` / `Annotated` in any order. If making the key writable is truly unavoidable, suppress with `# writable-ok: <reason>`
 - Use dependency injection
 - Fully typed; no `Any` or coarse types like `dict[str, Any]` or just `dict`. Every function parameter must be strongly typed
 - Use tagged unions + match

CONTRIBUTING.md

@@ -13,8 +13,8 @@ Here are the core requirements for any PR submitted to LiteLLM:
 
 - [ ] **Add testing** - Adding at least 1 test is a hard requirement - [see details](#adding-testing)
 - [ ] **Ensure your PR passes all checks**:
-  - [ ] [Unit Tests](#running-unit-tests) - `make test-unit`
   - [ ] [Linting / Formatting](#running-linting-and-formatting-checks) - `make lint`
+  - [ ] [The tests covering your change](#running-unit-tests) pass, e.g. `uv run pytest tests/test_litellm/<your_test_file>.py -v`. CI runs the full unit test matrix, so you don't need to run the whole suite locally
 
 #### UI PRs
 
@@ -71,8 +71,8 @@ make format
 # Run all linting checks (matches CI exactly)
 make lint
 
-# Run unit tests to ensure nothing is broken
-make test-unit
+# Run the tests covering your change (CI runs the full suite)
+uv run pytest tests/test_litellm/<your_test_file>.py -v
 
 # Commit your changes (must follow Conventional Commits — see above)
 git add .
@@ -123,12 +123,13 @@ def test_your_feature():
 
 ### Running Unit Tests
 
-Run all unit tests (uses parallel execution for speed):
-
+Run the tests covering your change:
 ```bash
-make test-unit
+uv run pytest tests/test_litellm/test_your_file.py -v
 ```
 
+`tests/test_litellm` holds thousands of tests, so running all of it locally takes a long time. CI runs it as a parallel matrix (`make test-unit-llms`, `make test-unit-proxy-core`, and the other `test-unit-*` targets) on beefier boxes, so if, for whatever reason, you must run the whole suite, it's better to rely on CI to do that.
+
 If you're running broader test suites, proxy tests, or anything that touches PostgreSQL-backed fixtures/plugins, install the full local test environment first:
 
 ```bash
@@ -137,11 +138,6 @@ make install-test-deps
 
 This syncs the locked test environment used across the repo, including `psycopg` v3 plus `psycopg-binary` (used by `pytest-postgresql`), `psycopg2-binary` (used by some proxy E2E tests), and a generated Prisma client for DB-backed proxy tests, so pytest startup matches CI without manual package installs.
 
-Run specific test files:
-```bash
-uv run pytest tests/test_litellm/test_your_file.py -v
-```
-
 ### Running Linting and Formatting Checks
 
 Run all linting checks (matches CI exactly):

Makefile

@@ -4,11 +4,12 @@
 .PHONY: help test test-unit test-unit-llms test-unit-proxy-guardrails test-unit-proxy-core test-unit-proxy-misc \
 	test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \
 	test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \
-	info lint lint-dev lint-checks format \
+	info lint lint-inner lint-dev lint-checks format \
 	lint-basedpyright lint-e2e-basedpyright lint-basedpyright-budget-update lint-type-discipline lint-type-discipline-budget-update \
 	lint-ruff-budget lint-ruff-budget-update lint-budget-update lint-gate \
+	lint-test-quality lint-test-quality-budget-update \
 	install-dev install-proxy-dev install-test-deps install-hooks \
-	install-helm-unittest check-circular-imports check-import-safety check pre-commit \
+	install-helm-unittest check-circular-imports check-import-safety check check-inner pre-commit \
 	lint-install lint-fetch-base bootstrap
 
 # Default target
@@ -35,7 +36,8 @@ help:
 	@echo "  make lint-ruff-budget - Gate the codebase total of each strict ruff rule against its limit"
 	@echo "  make lint-gate        - Strict ruff gate in CI-parity mode (fetches staging, simulates the merge)"
 	@echo "  make lint-ruff-budget-update - Ratchet ruff-strict-budget.json limits down by what this branch fixed"
-	@echo "  make lint-budget-update - Ratchet all budgets down (ruff + type-discipline + basedpyright)"
+	@echo "  make lint-test-quality  - Gate the test suite against test-quality-budget.json"
+	@echo "  make lint-budget-update - Ratchet all budgets down (ruff + type-discipline + test quality + basedpyright)"
 	@echo "  make check-circular-imports - Check for circular imports"
 	@echo "  make check-import-safety - Check import safety"
 	@echo "  make test               - Run all tests"
@@ -52,10 +54,17 @@ help:
 	@echo "  make test-proxy-unit-b  - Run proxy_unit_tests (p-z, ~28 files)"
 	@echo "  make test-integration   - Run integration tests"
 	@echo "  make test-unit-helm     - Run helm unit tests"
+	@echo ""
+	@echo "Heavy targets (check, lint) queue for LITELLM_GATE_SLOTS machine-wide"
+	@echo "slots (default 2; 0 disables) so parallel sessions don't thrash one machine."
 
 UV := uv
 UV_RUN := $(UV) run --no-sync
 
+# Machine-wide slot queue for the heavy targets below; python3 + stdlib only, so
+# it runs before any venv exists. See scripts/gate_slot_lock.py.
+GATE_SLOT_LOCK := python3 scripts/gate_slot_lock.py
+
 LINT_DEP_INSTALL ?= install-dev
 LINT_E2E_DEP_INSTALL ?= lint-install
 LINT_DEP_BASE ?= lint-fetch-base
@@ -73,6 +82,8 @@ info:
 install-dev:
 	$(UV) sync --inexact --frozen
 
+# Deliberately unqueued: provisioning is I/O bound, so it doesn't need one of the
+# machine-wide slots the CPU-bound gates below share.
 bootstrap:
 	$(UV) sync --inexact --frozen --extra proxy --group proxy-dev --group e2e-dev
 	$(UV_RUN) python scripts/prisma_generate_if_needed.py
@@ -191,6 +202,11 @@ lint-e2e-basedpyright: $(LINT_E2E_DEP_INSTALL)
 lint-type-discipline: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
 	$(UV_RUN) python scripts/type_discipline_gate.py --base origin/litellm_internal_staging
 
+# Test-quality budget (zero-assert / mock-echo tests, sys.path.insert, raw env writes,
+# litellm module-global mutation), counted across tests/ the same delta-vs-base way.
+lint-test-quality: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE)
+	$(UV_RUN) python scripts/test_quality_gate.py --base origin/litellm_internal_staging
+
 # --update lowers each limit by what this branch fixed since its branch point, so
 # it needs the base ref fetched to resolve the merge-base.
 lint-basedpyright-budget-update: install-dev lint-fetch-base
@@ -212,8 +228,11 @@ lint-ruff-budget-update: install-dev lint-fetch-base
 lint-type-discipline-budget-update: install-dev lint-fetch-base
 	$(UV_RUN) python scripts/type_discipline_gate.py --update
 
-# Ratchet all budgets in one shot (ruff strict + type-discipline + basedpyright)
-lint-budget-update: lint-ruff-budget-update lint-type-discipline-budget-update lint-basedpyright-budget-update
+lint-test-quality-budget-update: install-dev lint-fetch-base
+	$(UV_RUN) python scripts/test_quality_gate.py --update
+
+# Ratchet all budgets in one shot (ruff strict + type-discipline + test quality + basedpyright)
+lint-budget-update: lint-ruff-budget-update lint-type-discipline-budget-update lint-test-quality-budget-update lint-basedpyright-budget-update
 
 check-circular-imports: $(LINT_DEP_INSTALL)
 	cd litellm && $(UV_RUN) python ../tests/documentation_tests/test_circular_imports.py && cd ..
@@ -229,10 +248,13 @@ check-import-safety: $(LINT_DEP_INSTALL)
 # does (merge-base with origin/litellm_internal_staging). Setup (env sync, Prisma client,
 # base fetch) runs once up front; the checks themselves are independent, so a sub-make
 # fans them out with -j and the fast ones finish under basedpyright's shadow.
-lint: lint-install lint-fetch-base
+lint:
+	@$(GATE_SLOT_LOCK) $(MAKE) lint-inner
+
+lint-inner: lint-install lint-fetch-base
 	$(MAKE) -j $(LINT_JOBS) $(LINT_OUTPUT_SYNC) LINT_DEP_INSTALL= LINT_E2E_DEP_INSTALL= LINT_DEP_BASE= lint-checks
 
-lint-checks: lint-format-check-changed lint-ruff lint-gate lint-type-discipline lint-basedpyright lint-e2e-basedpyright check-circular-imports check-import-safety
+lint-checks: lint-format-check-changed lint-ruff lint-gate lint-type-discipline lint-test-quality lint-basedpyright lint-e2e-basedpyright check-circular-imports check-import-safety
 
 # Faster linting for local development (only checks changed code)
 lint-dev: lint-format-changed check-circular-imports check-import-safety
@@ -244,7 +266,10 @@ lint-dev: lint-format-changed check-circular-imports check-import-safety
 # test-linting.yml (Python), test-litellm-ui-build.yml's frontend-lint (dashboard), and
 # check-ui-api-types.yml (API-type drift), skipping any whose files aren't in scope.
 # Not auto-installed as a git hook so it never slows an unrelated human commit.
-check: bootstrap
+check:
+	@$(GATE_SLOT_LOCK) $(MAKE) check-inner
+
+check-inner: bootstrap
 	./scripts/pre_commit_lint.sh
 
 pre-commit:
@@ -299,7 +324,7 @@ test-unit-helm: install-helm-unittest
 # LLM Translation testing targets
 test-llm-translation: install-test-deps
 	@echo "Running LLM translation tests..."
-	@python .github/workflows/run_llm_translation_tests.py
+	@python .github/scripts/run_llm_translation_tests.py
 
 test-llm-translation-single: install-test-deps
 	@echo "Running single LLM translation test file..."

backend/routes/allowlist.py

@@ -35,6 +35,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
     # Models & routing config
     "/model/",
     "/v1/model/info",
+    "/v1/model/deprecations",
     "/v2/model/",
     "/model_group",
     "/model_access_group/",
@@ -146,11 +147,13 @@ BACKEND_EXACT_PATHS: frozenset[str] = frozenset(
         "/docs/oauth2-redirect",
         "/redoc",
         "/fallback/login",
+        "/mcp",  # bare spelling of the aggregate MCP endpoint; /mcp/ prefix covers the rest
     }
 )
 
 BACKEND_MOUNT_PATHS: frozenset[str] = frozenset(
     {
         "/swagger",  # API documentation static assets belong to the backend
+        "/mcp",  # lazily-mounted MCP sub-app serves on the backend component
     }
 )

basedpyright-code-budget.json

@@ -1,15 +1,15 @@
 {
   "reportAny": {
-    "limit": 26391
+    "limit": 19955
   },
   "reportArgumentType": {
-    "limit": 2614
+    "limit": 2566
   },
   "reportAssignmentType": {
-    "limit": 327
+    "limit": 320
   },
   "reportAttributeAccessIssue": {
-    "limit": 514
+    "limit": 488
   },
   "reportCallIssue": {
     "limit": 114
@@ -18,19 +18,19 @@
     "limit": 40
   },
   "reportDeprecated": {
-    "limit": 215
+    "limit": 213
   },
   "reportDuplicateImport": {
     "limit": 19
   },
   "reportExplicitAny": {
-    "limit": 8319
+    "limit": 6049
   },
   "reportFunctionMemberAccess": {
     "limit": 7
   },
   "reportGeneralTypeIssues": {
-    "limit": 157
+    "limit": 154
   },
   "reportIncompatibleMethodOverride": {
     "limit": 56
@@ -54,10 +54,10 @@
     "limit": 0
   },
   "reportMissingParameterType": {
-    "limit": 5825
+    "limit": 5663
   },
   "reportMissingTypeArgument": {
-    "limit": 15695
+    "limit": 15555
   },
   "reportMissingTypeStubs": {
     "limit": 40
@@ -72,7 +72,7 @@
     "limit": 0
   },
   "reportOptionalMemberAccess": {
-    "limit": 1077
+    "limit": 1061
   },
   "reportOptionalOperand": {
     "limit": 0
@@ -84,7 +84,7 @@
     "limit": 56
   },
   "reportPrivateUsage": {
-    "limit": 1824
+    "limit": 1823
   },
   "reportRedeclaration": {
     "limit": 8
@@ -99,46 +99,46 @@
     "limit": 0
   },
   "reportUnknownArgumentType": {
-    "limit": 44996
+    "limit": 44655
   },
   "reportUnknownLambdaType": {
-    "limit": 113
+    "limit": 109
   },
   "reportUnknownMemberType": {
-    "limit": 39643
+    "limit": 39017
   },
   "reportUnknownParameterType": {
-    "limit": 20132
+    "limit": 19885
   },
   "reportUnknownVariableType": {
-    "limit": 31153
+    "limit": 30572
   },
   "reportUnnecessaryCast": {
-    "limit": 118
+    "limit": 117
   },
   "reportUnnecessaryComparison": {
-    "limit": 701
+    "limit": 699
   },
   "reportUnnecessaryContains": {
     "limit": 5
   },
   "reportUnnecessaryIsInstance": {
-    "limit": 857
+    "limit": 836
   },
   "reportUntypedBaseClass": {
     "limit": 0
   },
   "reportUntypedFunctionDecorator": {
-    "limit": 33
+    "limit": 27
   },
   "reportUnusedClass": {
-    "limit": 23
+    "limit": 21
   },
   "reportUnusedFunction": {
     "limit": 139
   },
   "reportUnusedImport": {
-    "limit": 555
+    "limit": 545
   },
   "reportUnusedVariable": {
     "limit": 146

ci_cd/generate_model_prices_schema.py

@@ -41,6 +41,11 @@ OBJECT_KEYS: dict[str, JsonSchema] = {
         },
         "additionalProperties": False,
     },
+    "guardrail_cost_per_unit": {
+        "type": "object",
+        "description": "USD cost per billable guardrail unit, keyed by the provider's usage counter name (e.g. Bedrock's contentPolicyUnits).",
+        "additionalProperties": NONNEG_NUMBER,
+    },
     "metadata": {
         "type": "object",
         "description": "Free-form notes about the entry (e.g. pricing derivation).",
@@ -96,6 +101,7 @@ ARRAY_KEYS: dict[str, JsonSchema] = {
                 "output_cost_per_token": NONNEG_NUMBER,
                 "output_cost_per_reasoning_token": NONNEG_NUMBER,
                 "cache_read_input_token_cost": NONNEG_NUMBER,
+                "cache_creation_input_token_cost": NONNEG_NUMBER,
                 "input_cost_per_query": NONNEG_NUMBER,
             },
             "additionalProperties": False,
@@ -139,6 +145,11 @@ NUMBER_KEYS: dict[str, JsonSchema] = {
         "minimum": 1,
         "description": "Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%).",
     },
+    "regional_endpoint_uplift_multiplier": {
+        "type": "number",
+        "minimum": 1,
+        "description": "Multiplier applied to all token costs when served from a non-global Vertex AI endpoint (e.g. 1.10 = +10%).",
+    },
 }
 
 COST_DESCRIPTIONS: dict[str, str] = {

enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py

@@ -99,6 +99,7 @@ class BaseEmailLogger(CustomLogger):
         email_html_content = USER_INVITATION_EMAIL_TEMPLATE.format(
             email_logo_url=email_params.logo_url,
             recipient_email=email_params.recipient_email,
+            invitation_link=email_params.base_url,
             base_url=email_params.base_url,
             email_support_contact=email_params.support_contact,
             email_footer=email_params.signature,
@@ -826,10 +827,15 @@ class BaseEmailLogger(CustomLogger):
         """
         # Early validation
         if not user_id:
-            verbose_proxy_logger.debug("No user_id provided for invitation link")
+            verbose_proxy_logger.warning(
+                "No user_id provided for invitation link. Email will link to base URL instead of onboarding page"
+            )
             return base_url
 
         if not await self._is_prisma_client_available():
+            verbose_proxy_logger.warning(
+                "Prisma client not available. Email will link to base URL instead of onboarding page"
+            )
             return base_url
 
         # Wait for any concurrent invitation creation to complete
@@ -839,11 +845,15 @@ class BaseEmailLogger(CustomLogger):
         invitation = await self._get_or_create_invitation(user_id)
         if not invitation:
             verbose_proxy_logger.warning(
-                f"Failed to get/create invitation for user_id: {user_id}"
+                f"Failed to get/create invitation for user_id: {user_id}. Email will link to base URL instead of onboarding page"
             )
             return base_url
 
-        return self._construct_invitation_link(invitation.id, base_url)
+        invitation_link = self._construct_invitation_link(invitation.id, base_url)
+        verbose_proxy_logger.info(
+            f"Successfully created invitation link for user_id: {user_id}"
+        )
+        return invitation_link
 
     async def _is_prisma_client_available(self) -> bool:
         """Check if Prisma client is available"""
@@ -921,7 +931,9 @@ class BaseEmailLogger(CustomLogger):
 
         # http://localhost:4000/ui/onboarding?invitation_id=7a096b3a-37c6-440f-9dd1-ba22e8043f6b
         """
-        return f"{base_url}/ui/onboarding?invitation_id={invitation_id}"
+        base_url = base_url.rstrip("/")
+        invitation_link = f"{base_url}/ui/onboarding?invitation_id={invitation_id}"
+        return invitation_link
 
     async def send_email(
         self,

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -3,7 +3,8 @@ Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if t
 """
 
 from datetime import datetime, timedelta, timezone
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Any, Dict, Final, List, Optional, Tuple
 
 from litellm._logging import verbose_proxy_logger
 from litellm._uuid import uuid
@@ -23,6 +24,19 @@ if TYPE_CHECKING:
 
 CHECK_BATCH_COST_USER_AGENT = "LiteLLM Proxy/CheckBatchCost"
 
+PROVIDER_TERMINAL_BATCH_STATUSES: Final[Tuple[str, ...]] = (
+    "completed",
+    "complete",
+    "failed",
+    "expired",
+    "cancelled",
+)
+
+TERMINAL_MANAGED_OBJECT_STATUSES: Final[Tuple[str, ...]] = (
+    *PROVIDER_TERMINAL_BATCH_STATUSES,
+    "stale_expired",
+)
+
 
 class CheckBatchCost:
     def __init__(
@@ -42,6 +56,33 @@ class CheckBatchCost:
         # Cached after the first poll cycle. Once we know the column is absent we skip
         # the guaranteed-failing primary query on every subsequent cycle.
         self._has_batch_processed_column: bool = True
+        self.batch_processed_support_confirmed: bool = False
+
+    @staticmethod
+    def _is_missing_batch_processed_column_error(err: Exception) -> bool:
+        message: Final = str(err).lower()
+        return "batch_processed" in message or "unknown column" in message or "does not exist" in message
+
+    async def confirm_batch_processed_support(self) -> None:
+        """
+        Probe the batch_processed column before the proxy serves traffic, so the retrieve
+        path never sees an unconfirmed poller on a schema that has the column and accounts
+        inline for a batch the first poll cycle then accounts again.
+        """
+        try:
+            await self.prisma_client.db.litellm_managedobjecttable.find_first(
+                where={"file_purpose": "batch", "batch_processed": False}
+            )
+        except Exception as probe_err:
+            if not self._is_missing_batch_processed_column_error(probe_err):
+                verbose_proxy_logger.debug(
+                    f"CheckBatchCost: batch_processed probe failed, the poll cycle will confirm support: {probe_err}"
+                )
+                return
+            self._has_batch_processed_column = False
+            verbose_proxy_logger.warning("CheckBatchCost: batch_processed column not found, querying without it")
+            return
+        self.batch_processed_support_confirmed = True
 
     async def _get_user_info(self, batch_id: str, user_id: Optional[str]) -> Dict[str, Any]:
         """
@@ -132,11 +173,11 @@ class CheckBatchCost:
         in non-terminal states as 'stale_expired'. These will never complete and
         should not be polled.
         """
-        cutoff = datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS)
-        result = await self.prisma_client.db.litellm_managedobjecttable.update_many(
+        cutoff: Final = datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS)
+        result: Final = await self.prisma_client.db.litellm_managedobjecttable.update_many(
             where={
                 "file_purpose": "batch",
-                "status": {"not_in": ["completed", "complete", "failed", "expired", "cancelled", "stale_expired"]},
+                "status": {"not_in": list(TERMINAL_MANAGED_OBJECT_STATUSES)},
                 "created_at": {"lt": cutoff},
             },
             data={"status": "stale_expired"},
@@ -147,6 +188,26 @@ class CheckBatchCost:
                 f"(older than {MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days) as stale_expired"
             )
 
+        if not self._has_batch_processed_column:
+            return
+
+        # A row already in a terminal status is never rewritten by the sweep above, so
+        # without this it keeps a poll-page slot forever and starves newer batches.
+        retired: Final = await self.prisma_client.db.litellm_managedobjecttable.update_many(
+            where={
+                "file_purpose": "batch",
+                "batch_processed": False,
+                "status": {"in": ["complete", "completed"]},
+                "created_at": {"lt": cutoff},
+            },
+            data={"batch_processed": True},
+        )
+        if retired > 0:
+            verbose_proxy_logger.warning(
+                f"CheckBatchCost: gave up on {retired} completed managed objects older than "
+                f"{MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days that were never costed"
+            )
+
     async def _fallback_find_jobs(self) -> list:
         """Query batch jobs without the batch_processed filter (for older schemas)."""
         return await self.prisma_client.db.litellm_managedobjecttable.find_many(
@@ -167,6 +228,119 @@ class CheckBatchCost:
             order={"created_at": "asc"},
         )
 
+    async def _retire_job(self, job: "LiteLLM_ManagedObjectTable", reason: str) -> None:
+        """
+        Take a row that can never be costed out of the poll page. Leaving it selectable
+        would burn one of the MAX_OBJECTS_PER_POLL_CYCLE slots on every future cycle, and
+        once enough such rows accumulate no newer batch is ever reached. Older schemas
+        without batch_processed can only be excluded through the status filter.
+        """
+        data: Final = (
+            {"batch_processed": True}
+            if self._has_batch_processed_column
+            else {"status": "stale_expired"}
+        )
+        try:
+            await self.prisma_client.db.litellm_managedobjecttable.update(
+                where={"id": job.id},
+                data=data,
+            )
+        except Exception as db_err:
+            verbose_proxy_logger.error(
+                f"CheckBatchCost: failed to retire uncostable job {job.id} ({reason}): {db_err}"
+            )
+            return
+        verbose_proxy_logger.warning(
+            f"CheckBatchCost: job {job.id} can never be costed ({reason}), "
+            "so it will no longer be polled"
+        )
+
+    @staticmethod
+    def _has_unified_id_without_model(job: "LiteLLM_ManagedObjectTable") -> bool:
+        """A unified id that decodes but carries no model_id can never be routed."""
+        from litellm.proxy.openai_files_endpoints.common_utils import (
+            convert_b64_uid_to_unified_uid,
+            get_model_id_from_unified_batch_id,
+        )
+
+        decoded: Final = convert_b64_uid_to_unified_uid(job.unified_object_id)
+        return (
+            decoded != job.unified_object_id
+            and get_model_id_from_unified_batch_id(decoded) is None
+        )
+
+    @staticmethod
+    def _is_batch_gone_at_provider(error: Exception, batch_id: str) -> bool:
+        """
+        A 404 naming the batch means the provider dropped its record of it, so no later
+        retrieve can ever succeed. A 404 about anything else, a renamed Azure deployment
+        or a fallback deployment that never saw this batch, is still fixable in config, so
+        it keeps retrying.
+        """
+        import openai
+
+        from litellm.exceptions import NotFoundError
+
+        return isinstance(error, (NotFoundError, openai.NotFoundError)) and batch_id in str(error)
+
+    def _batch_deployment_exists(self, model_id: str) -> bool:
+        """A 404 only proves the batch is gone when it came from the batch's own
+        deployment. Once that deployment leaves the router, default fallbacks can
+        silently send the retrieve to a provider that never saw the batch, so its
+        404 must not retire the row; the staleness sweep bounds it instead."""
+        return self.llm_router.get_deployment(model_id=model_id) is not None
+
+    @staticmethod
+    def _is_output_file_gone_at_provider(error: Exception, output_file_id: Optional[str]) -> bool:
+        """A 404 naming the output file means there is nothing to fetch on this or any
+        later poll: providers like Vertex AI advertise an output path for every batch,
+        including terminal ones that never wrote it. Any other failure may be
+        transient, so it keeps retrying until the staleness sweep bounds it."""
+        import openai
+
+        from litellm.exceptions import NotFoundError
+
+        if not output_file_id:
+            return False
+        return isinstance(error, (NotFoundError, openai.NotFoundError)) and output_file_id in str(error)
+
+    async def _finalize_unbilled_terminal_job(
+        self, job: "LiteLLM_ManagedObjectTable", response: "LiteLLMBatch"
+    ) -> None:
+        """Persist a terminal batch that has nothing billable, converting any raw
+        provider file ids to managed ids, and take it out of the poll page."""
+        try:
+            from litellm.proxy.openai_files_endpoints.common_utils import (
+                _is_base64_encoded_unified_file_id,
+                ensure_batch_response_managed_file_ids,
+            )
+
+            response.id = job.unified_object_id
+            await ensure_batch_response_managed_file_ids(
+                response=response,
+                managed_files_obj=self.proxy_logging_obj.get_proxy_hook("managed_files"),
+                prisma_client=self.prisma_client,
+                verbose_proxy_logger=verbose_proxy_logger,
+                db_batch_object=job,
+                unified_batch_id=_is_base64_encoded_unified_file_id(job.unified_object_id),
+            )
+            update_data: Final[dict] = {
+                "status": response.status,
+                "file_object": response.model_dump_json(),
+                **({"batch_processed": True} if self._has_batch_processed_column else {}),
+            }
+            await self.prisma_client.db.litellm_managedobjecttable.update(
+                where={"id": job.id},
+                data=update_data,
+            )
+            verbose_proxy_logger.info(
+                f"CheckBatchCost: marked job {job.id} as {response.status} in DB"
+            )
+        except Exception as db_err:
+            verbose_proxy_logger.error(
+                f"CheckBatchCost: failed to mark job {job.id} as {response.status} in DB: {db_err}"
+            )
+
     @staticmethod
     def _record_error(
         prom_logger: Optional["PrometheusLogger"], error_type: str
@@ -409,6 +583,7 @@ class CheckBatchCost:
         from litellm.files.main import afile_content
         from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
         from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
+        from litellm.litellm_core_utils.litellm_logging import deployment_pricing_model_info
         from litellm.proxy.openai_files_endpoints.common_utils import (
             _is_base64_encoded_unified_file_id,
         )
@@ -446,6 +621,7 @@ class CheckBatchCost:
         credentials = self.llm_router.get_deployment_credentials_with_provider(model_id) or {}
         _file_content = await afile_content(
             file_id=raw_output_file_id,
+            _litellm_internal_model_credentials=MappingProxyType(dict(credentials)),
             **credentials,
         )
 
@@ -528,15 +704,20 @@ class CheckBatchCost:
                             f"{_file_attr}={_raw_file_id!r}: {_e}"
                         )
 
-        # Pass deployment model_info so custom batch pricing
-        # (input_cost_per_token_batches etc.) is used for cost calc
-        deployment_model_info = deployment_info.model_info.model_dump() if deployment_info.model_info else {}
+        # Pass the deployment's router-registered pricing (litellm_params custom
+        # rates merged with the model's published rates) so custom batch pricing
+        # (input_cost_per_token_batches etc.) is used for cost calc, exactly as
+        # the inline retrieve path does.
+        deployment_model_info = deployment_pricing_model_info(
+            model_id=model_id,
+            deployment_model=litellm_model_name,
+        )
         batch_cost, batch_usage, batch_models = (
             await calculate_batch_cost_and_usage(
                 file_content_dictionary=file_content_as_dict,
                 custom_llm_provider=llm_provider,  # type: ignore
                 model_name=model_name,
-                model_info=deployment_model_info,  # type: ignore[arg-type]
+                model_info=deployment_model_info,
             )
         )
         logging_obj = LiteLLMLogging(
@@ -631,8 +812,9 @@ class CheckBatchCost:
                     take=MAX_OBJECTS_PER_POLL_CYCLE,
                     order={"created_at": "asc"},
                 )
+                self.batch_processed_support_confirmed = True
             except Exception as query_err:
-                if "batch_processed" not in str(query_err).lower() and "unknown column" not in str(query_err).lower() and "does not exist" not in str(query_err).lower():
+                if not self._is_missing_batch_processed_column_error(query_err):
                     raise
                 # Permanent schema gap — cache the result so future cycles skip straight to fallback
                 self._has_batch_processed_column = False
@@ -645,6 +827,8 @@ class CheckBatchCost:
         for job in jobs:
             routing = self._resolve_job_routing(job, prom_logger)
             if routing is None:
+                if self._has_unified_id_without_model(job):
+                    await self._retire_job(job, "unified object id has no model id")
                 continue
             model_id, batch_id = routing
 
@@ -667,11 +851,13 @@ class CheckBatchCost:
                 )
                 if prom_logger:
                     prom_logger.record_check_batch_cost_error("provider_retrieval_error")
+                if self._is_batch_gone_at_provider(e, batch_id) and self._batch_deployment_exists(model_id):
+                    await self._retire_job(job, f"batch {batch_id} no longer exists at the provider")
                 continue
 
             ## RETRIEVE THE BATCH JOB OUTPUT FILE
             if (
-                response.status == "completed"
+                response.status in PROVIDER_TERMINAL_BATCH_STATUSES
                 and response.output_file_id is not None
             ):
                 try:
@@ -683,6 +869,15 @@ class CheckBatchCost:
                         prom_logger=prom_logger,
                     )
                 except Exception as tracking_err:
+                    if self._is_output_file_gone_at_provider(
+                        tracking_err, response.output_file_id
+                    ) and self._batch_deployment_exists(model_id):
+                        verbose_proxy_logger.warning(
+                            f"CheckBatchCost: output file {response.output_file_id} of batch {batch_id} "
+                            f"does not exist at the provider; retiring job {job.id} unbilled"
+                        )
+                        await self._finalize_unbilled_terminal_job(job, response)
+                        continue
                     verbose_proxy_logger.error(
                         f"CheckBatchCost: failed to track cost for batch {batch_id} "
                         f"(job {job.id}); leaving it unprocessed so the next poll retries: {tracking_err}"
@@ -698,7 +893,7 @@ class CheckBatchCost:
                 # mark the job as complete
                 try:
                     update_data: dict = {
-                        "status": "complete",
+                        "status": response.status if response.status != "completed" else "complete",
                         "file_object": response.model_dump_json(),
                     }
                     if self._has_batch_processed_column:
@@ -712,39 +907,8 @@ class CheckBatchCost:
                         f"CheckBatchCost: failed to mark job {job.id} complete in DB: {db_err}"
                     )
 
-            elif response.status in ("failed", "expired", "cancelled"):
-                try:
-                    from litellm.proxy.openai_files_endpoints.common_utils import (
-                        _is_base64_encoded_unified_file_id,
-                        ensure_batch_response_managed_file_ids,
-                    )
-
-                    response.id = job.unified_object_id
-                    await ensure_batch_response_managed_file_ids(
-                        response=response,
-                        managed_files_obj=self.proxy_logging_obj.get_proxy_hook("managed_files"),
-                        prisma_client=self.prisma_client,
-                        verbose_proxy_logger=verbose_proxy_logger,
-                        db_batch_object=job,
-                        unified_batch_id=_is_base64_encoded_unified_file_id(job.unified_object_id),
-                    )
-                    update_data = {
-                        "status": response.status,
-                        "file_object": response.model_dump_json(),
-                    }
-                    if self._has_batch_processed_column:
-                        update_data["batch_processed"] = True
-                    await self.prisma_client.db.litellm_managedobjecttable.update(
-                        where={"id": job.id},
-                        data=update_data,
-                    )
-                    verbose_proxy_logger.info(
-                        f"CheckBatchCost: marked job {job.id} as {response.status} in DB"
-                    )
-                except Exception as db_err:
-                    verbose_proxy_logger.error(
-                        f"CheckBatchCost: failed to mark job {job.id} as {response.status} in DB: {db_err}"
-                    )
+            elif response.status in PROVIDER_TERMINAL_BATCH_STATUSES:
+                await self._finalize_unbilled_terminal_job(job, response)
 
         # Record polling run metrics (always, even if nothing was processed)
         if prom_logger:

enterprise/litellm_enterprise/proxy/hooks/managed_files.py

@@ -41,6 +41,7 @@ from litellm.proxy._types import (
     CallTypes,
     LiteLLM_ManagedFileTable,
     LiteLLM_ManagedObjectTable,
+    ProxyException,
     UserAPIKeyAuth,
 )
 from litellm.proxy.openai_files_endpoints.common_utils import (
@@ -54,6 +55,9 @@ from litellm.proxy.openai_files_endpoints.common_utils import (
     normalize_mime_type_for_provider,
     resolve_managed_output_file_model_name,
 )
+from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attribution import (
+    request_tags_from_metadata,
+)
 from litellm.types.llms.openai import (  # pyright: ignore[reportAttributeAccessIssue]
     AllMessageValues,
     AsyncCursorPage,
@@ -420,13 +424,26 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
         # This is because the encoded object ids stored in the managed objects table do not contain the provider information
         # To support provider filtering, we would need to store the provider information in the encoded object ids
         if provider:
-            raise Exception("Filtering by 'provider' is not supported when using managed batches.")
+            raise ProxyException(
+                message="Filtering by 'provider' is not supported when using managed batches.",
+                type="invalid_request_error",
+                param="provider",
+                code=400,
+            )
 
         # Model name filtering is not supported for managed batches
         # This is because the encoded object ids stored in the managed objects table do not contain the model name
         # A hash of the model name + litellm_params for the model name is encoded as the model id. This is not sufficient to reliably map the target model names to the model ids.
         if target_model_names:
-            raise Exception("Filtering by 'target_model_names' is not supported when using managed batches.")
+            raise ProxyException(
+                message="Filtering by 'target_model_names' is not supported when using managed batches.",
+                type="invalid_request_error",
+                param="target_model_names",
+                code=400,
+            )
+
+        if limit == 0:
+            return build_list_page([])
 
         owner_filter = build_owner_filter(user_api_key_dict)
         if owner_filter is None:
@@ -1146,6 +1163,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
             ## Check if unified_file_id is in the response
             unified_file_id = response._hidden_params.get("unified_file_id")  # managed file id
             unified_batch_id = response._hidden_params.get("unified_batch_id")  # managed batch id
+            is_batch_create: Final = unified_file_id is not None
             model_id = cast(Optional[str], response._hidden_params.get("model_id"))
             model_name = cast(Optional[str], response._hidden_params.get("model_name"))
 
@@ -1216,6 +1234,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                             model_mappings={model_id: provider_file_id},
                             user_api_key_dict=user_api_key_dict,
                         )
+            request_metadata: Final = data.get("litellm_metadata")
             await self.store_unified_object_id(
                 unified_object_id=response.id,
                 file_object=response,
@@ -1223,6 +1242,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
                 model_object_id=original_response_id,
                 file_purpose="batch",
                 user_api_key_dict=user_api_key_dict,
+                request_tags=request_tags_from_metadata(request_metadata if isinstance(request_metadata, dict) else {}),
+                persist_attribution=is_batch_create,
             )
 
             # Only record batch creation metric on actual create (not retrieve/cancel).

enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py

@@ -11,7 +11,7 @@ Endpoints for /project operations
 #### PROJECT MANAGEMENT ####
 
 import json
-from collections.abc import Mapping, Sequence
+from collections.abc import Sequence
 from typing import TYPE_CHECKING
 
 from fastapi import APIRouter, Depends, HTTPException, Request
@@ -29,7 +29,11 @@ from litellm.proxy.utils import PrismaClient, handle_exception_on_proxy
 
 if TYPE_CHECKING:
     from prisma import models as prisma_models
-    from prisma.actions import LiteLLM_TeamTableActions
+    from prisma.actions import (
+        LiteLLM_ProjectTableActions,
+        LiteLLM_TeamTableActions,
+        LiteLLM_VerificationTokenActions,
+    )
 
 router = APIRouter()
 
@@ -39,6 +43,27 @@ def _team_table(prisma_client: PrismaClient) -> "LiteLLM_TeamTableActions[prisma
     return team_table
 
 
+def _project_table(prisma_client: PrismaClient) -> "LiteLLM_ProjectTableActions[prisma_models.LiteLLM_ProjectTable]":
+    project_table: LiteLLM_ProjectTableActions[prisma_models.LiteLLM_ProjectTable] = (
+        prisma_client.db.litellm_projecttable
+    )
+    return project_table
+
+
+def _verification_token_table(
+    prisma_client: PrismaClient,
+) -> "LiteLLM_VerificationTokenActions[prisma_models.LiteLLM_VerificationToken]":
+    verification_token_table: LiteLLM_VerificationTokenActions[prisma_models.LiteLLM_VerificationToken] = (
+        prisma_client.db.litellm_verificationtoken
+    )
+    return verification_token_table
+
+
+def _jsonified(prisma_client: PrismaClient, payload: dict[str, object]) -> dict[str, object]:
+    jsonified: dict[str, object] = prisma_client.jsonify_object(payload)
+    return jsonified
+
+
 async def _check_user_permission_for_project(
     user_api_key_dict: UserAPIKeyAuth,
     team_id: str | None,
@@ -137,7 +162,7 @@ def _check_team_project_limits(
 
     # --- Validate project models are a subset of team models ---
     project_models = data.models
-    team_models = team_object.models or []
+    team_models: list[str] = team_object.models or []
     if project_models and len(team_models) > 0:
         # If team has 'all-proxy-models', skip validation as it allows all models
         if SpecialModelNames.all_proxy_models.value not in team_models:
@@ -188,11 +213,11 @@ async def _create_budget_for_project(
 ) -> str:
     """Create a budget for the project and return budget_id."""
     budget_params = LiteLLM_BudgetTable.model_fields.keys()
-    _json_data: Mapping[str, object] = data.json(exclude_none=True)
+    _json_data: dict[str, object] = data.model_dump(exclude_none=True)
     _budget_data = {k: v for k, v in _json_data.items() if k in budget_params}
     budget_row = LiteLLM_BudgetTable.model_validate(_budget_data)
 
-    new_budget = prisma_client.jsonify_object(budget_row.json(exclude_none=True))
+    new_budget = _jsonified(prisma_client, budget_row.model_dump(exclude_none=True))
 
     _budget: prisma_models.LiteLLM_BudgetTable = await prisma_client.db.litellm_budgettable.create(
         data={
@@ -227,7 +252,7 @@ async def _set_project_object_permission(
     return None
 
 
-def _remove_budget_fields_from_project_data(project_data: dict) -> dict:
+def _remove_budget_fields_from_project_data(project_data: dict[str, object]) -> dict[str, object]:
     """
     Remove budget fields from project data.
     Budget fields belong to LiteLLM_BudgetTable, not LiteLLM_ProjectTable.
@@ -396,9 +421,7 @@ async def new_project(
             data.project_id = str(uuid.uuid4())
         else:
             # Check if project_id already exists
-            existing_project = await prisma_client.db.litellm_projecttable.find_unique(
-                where={"project_id": data.project_id}
-            )
+            existing_project = await _project_table(prisma_client).find_unique(where={"project_id": data.project_id})
             if existing_project is not None:
                 raise ProxyException(
                     message=f"Project id = {data.project_id} already exists. Please use a different project id.",
@@ -423,11 +446,14 @@ async def new_project(
         )
 
         # Create project row (following organization_endpoints.py pattern)
-        project_row = LiteLLM_ProjectTable(
-            **data.json(exclude_none=True),
-            object_permission_id=object_permission_id,
-            created_by=user_api_key_dict.user_id or litellm_proxy_admin_name,
-            updated_by=user_api_key_dict.user_id or litellm_proxy_admin_name,
+        project_row_payload: dict[str, object] = data.model_dump(exclude_none=True)
+        project_row = LiteLLM_ProjectTable.model_validate(
+            {
+                **project_row_payload,
+                "object_permission_id": object_permission_id,
+                "created_by": user_api_key_dict.user_id or litellm_proxy_admin_name,
+                "updated_by": user_api_key_dict.user_id or litellm_proxy_admin_name,
+            }
         )
 
         for field in LiteLLM_ManagementEndpoint_MetadataFields:
@@ -438,7 +464,7 @@ async def new_project(
                     value=getattr(data, field),
                 )
 
-        new_project_row = prisma_client.jsonify_object(project_row.json(exclude_none=True))
+        new_project_row = _jsonified(prisma_client, project_row.model_dump(exclude_none=True))
 
         # Remove budget fields (following organization_endpoints.py pattern)
         new_project_row = _remove_budget_fields_from_project_data(new_project_row)
@@ -560,7 +586,7 @@ async def update_project(
         # Fetch existing project
         existing_project: (
             prisma_models.LiteLLM_ProjectTable | None
-        ) = await prisma_client.db.litellm_projecttable.find_unique(where={"project_id": data.project_id})
+        ) = await _project_table(prisma_client).find_unique(where={"project_id": data.project_id})
 
         if existing_project is None:
             raise ProxyException(
@@ -617,8 +643,7 @@ async def update_project(
             )
 
         # Prepare update data
-        update_data = data.json(exclude_none=True, exclude={"project_id"})
-        update_data = prisma_client.jsonify_object(update_data)
+        update_data = _jsonified(prisma_client, data.model_dump(exclude_none=True, exclude={"project_id"}))
         update_data["updated_by"] = user_api_key_dict.user_id or litellm_proxy_admin_name
 
         # Handle budget updates
@@ -660,9 +685,10 @@ async def update_project(
         # Handle metadata fields
         for field in LiteLLM_ManagementEndpoint_MetadataFields:
             if field in update_data:
-                if update_data.get("metadata") is None:
-                    update_data["metadata"] = {}
-                update_data["metadata"][field] = update_data.pop(field)
+                existing_metadata = update_data.get("metadata")
+                metadata_dict: dict[str, object] = existing_metadata if isinstance(existing_metadata, dict) else {}
+                metadata_dict[field] = update_data.pop(field)
+                update_data["metadata"] = metadata_dict
 
         # Remove budget fields (following organization_endpoints.py pattern)
         update_data = _remove_budget_fields_from_project_data(update_data)
@@ -748,11 +774,11 @@ async def delete_project(
                 detail={"error": "Only admins can delete projects"},
             )
 
-        deleted_projects = []
+        deleted_projects: list[prisma_models.LiteLLM_ProjectTable | None] = []
 
         for project_id in data.project_ids:
             # Check if project exists
-            existing_project = await prisma_client.db.litellm_projecttable.find_unique(where={"project_id": project_id})
+            existing_project = await _project_table(prisma_client).find_unique(where={"project_id": project_id})
 
             if existing_project is None:
                 raise ProxyException(
@@ -765,7 +791,7 @@ async def delete_project(
             # Check if there are any keys associated with this project
             associated_keys: Sequence[
                 prisma_models.LiteLLM_VerificationToken
-            ] = await prisma_client.db.litellm_verificationtoken.find_many(where={"project_id": project_id})
+            ] = await _verification_token_table(prisma_client).find_many(where={"project_id": project_id})
 
             if len(associated_keys) > 0:
                 raise ProxyException(
@@ -778,7 +804,7 @@ async def delete_project(
             # Delete the project
             deleted_project: (
                 prisma_models.LiteLLM_ProjectTable | None
-            ) = await prisma_client.db.litellm_projecttable.delete(where={"project_id": project_id})
+            ) = await _project_table(prisma_client).delete(where={"project_id": project_id})
 
             await delete_cached_project_object(
                 project_id=project_id,
@@ -829,7 +855,7 @@ async def project_info(
             )
 
         # Fetch project
-        project: prisma_models.LiteLLM_ProjectTable | None = await prisma_client.db.litellm_projecttable.find_unique(
+        project: prisma_models.LiteLLM_ProjectTable | None = await _project_table(prisma_client).find_unique(
             where={"project_id": project_id},
             include={"litellm_budget_table": True, "object_permission": True},
         )
@@ -901,7 +927,7 @@ async def list_projects(
         if user_api_key_has_admin_view(user_api_key_dict):
             projects: Sequence[
                 prisma_models.LiteLLM_ProjectTable
-            ] = await prisma_client.db.litellm_projecttable.find_many(
+            ] = await _project_table(prisma_client).find_many(
                 include={"litellm_budget_table": True, "object_permission": True}
             )
         else:
@@ -911,9 +937,9 @@ async def list_projects(
             user_record: prisma_models.LiteLLM_UserTable | None = await prisma_client.db.litellm_usertable.find_unique(
                 where={"user_id": user_api_key_dict.user_id},
             )
-            user_team_ids: Sequence[str] = user_record.teams if user_record is not None and user_record.teams else []
+            user_team_ids: list[str] = user_record.teams if user_record is not None and user_record.teams else []
 
-            projects = await prisma_client.db.litellm_projecttable.find_many(
+            projects = await _project_table(prisma_client).find_many(
                 where={"team_id": {"in": user_team_ids}},
                 include={"litellm_budget_table": True, "object_permission": True},
             )

enterprise/litellm_enterprise/proxy/vector_stores/endpoints.py

@@ -10,7 +10,8 @@ All /vector_store management endpoints
 
 import copy
 import json
-from typing import List, Optional
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Final, List, Optional, Protocol
 
 from fastapi import APIRouter, Depends, HTTPException
 
@@ -32,9 +33,35 @@ from litellm.types.vector_stores import (
 )
 from litellm.vector_stores.vector_store_registry import VectorStoreRegistry
 
+if TYPE_CHECKING:
+    from litellm.proxy.utils import PrismaClient
+
 router = APIRouter()
 
 
+class ManagedVectorStoreRow(Protocol):
+    """A ``litellm_managedvectorstorestable`` row as returned by Prisma."""
+
+    def model_dump(self) -> LiteLLM_ManagedVectorStore: ...
+
+
+class ManagedVectorStoreTable(Protocol):
+    """The Prisma actions namespace for ``litellm_managedvectorstorestable``."""
+
+    async def find_unique(self, where: Mapping[str, str | None]) -> ManagedVectorStoreRow | None: ...
+
+    async def create(self, data: Mapping[str, object]) -> ManagedVectorStoreRow: ...
+
+    async def delete(self, where: Mapping[str, str | None]) -> ManagedVectorStoreRow | None: ...
+
+    async def update(self, where: Mapping[str, str | None], data: Mapping[str, object]) -> ManagedVectorStoreRow: ...
+
+
+def managed_vector_store_table(prisma_client: "PrismaClient") -> ManagedVectorStoreTable:
+    """The Prisma table actions for managed vector stores, behind a typed surface."""
+    return prisma_client.db.litellm_managedvectorstorestable
+
+
 ########################################################
 # Management Endpoints
 ########################################################
@@ -66,7 +93,7 @@ async def new_vector_store(
     try:
         # Check if vector store already exists
         existing_vector_store = (
-            await prisma_client.db.litellm_managedvectorstorestable.find_unique(
+            await managed_vector_store_table(prisma_client).find_unique(
                 where={"vector_store_id": vector_store.get("vector_store_id")}
             )
         )
@@ -92,7 +119,7 @@ async def new_vector_store(
             del vector_store["litellm_params"]
 
         _new_vector_store = (
-            await prisma_client.db.litellm_managedvectorstorestable.create(
+            await managed_vector_store_table(prisma_client).create(
                 data={
                     **vector_store,
                     "litellm_params": litellm_params_json,
@@ -213,7 +240,7 @@ async def delete_vector_store(
     try:
         # Check if vector store exists
         existing_vector_store = (
-            await prisma_client.db.litellm_managedvectorstorestable.find_unique(
+            await managed_vector_store_table(prisma_client).find_unique(
                 where={"vector_store_id": data.vector_store_id}
             )
         )
@@ -224,7 +251,7 @@ async def delete_vector_store(
             )
 
         # Delete vector store
-        await prisma_client.db.litellm_managedvectorstorestable.delete(
+        await managed_vector_store_table(prisma_client).delete(
             where={"vector_store_id": data.vector_store_id}
         )
 
@@ -288,7 +315,7 @@ async def get_vector_store_info(
                 return {"vector_store": vector_store_pydantic_obj}
 
         vector_store = (
-            await prisma_client.db.litellm_managedvectorstorestable.find_unique(
+            await managed_vector_store_table(prisma_client).find_unique(
                 where={"vector_store_id": data.vector_store_id}
             )
         )
@@ -298,7 +325,7 @@ async def get_vector_store_info(
                 detail=f"Vector store with ID {data.vector_store_id} not found",
             )
 
-        vector_store_dict = vector_store.model_dump()  # type: ignore[attr-defined]
+        vector_store_dict = vector_store.model_dump()
         return {"vector_store": vector_store_dict}
     except Exception as e:
         verbose_proxy_logger.exception(f"Error getting vector store info: {str(e)}")
@@ -322,13 +349,13 @@ async def update_vector_store(
 
     try:
         update_data = data.model_dump(exclude_unset=True)
-        vector_store_id = update_data.pop("vector_store_id")
+        vector_store_id: Final[str] = update_data.pop("vector_store_id")
         if update_data.get("vector_store_metadata") is not None:
             update_data["vector_store_metadata"] = safe_dumps(
                 update_data["vector_store_metadata"]
             )
 
-        updated = await prisma_client.db.litellm_managedvectorstorestable.update(
+        updated = await managed_vector_store_table(prisma_client).update(
             where={"vector_store_id": vector_store_id},
             data=update_data,
         )

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.54"
+version = "0.1.57"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.54"
+version = "0.1.57"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

gateway/routes/allowlist.py

@@ -83,6 +83,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
     "/azure_ai/",
     "/aws/",
     "/bedrock/",
+    "/comprehendmedical",
     "/cohere/",
     "/gemini/",
     "/google/",

helm/litellm-helm/Chart.yaml

@@ -18,7 +18,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 1.1.1
+version: 1.1.2
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm/litellm-helm/README.md

@@ -29,7 +29,7 @@ If `db.useStackgresOperator` is used (not yet implemented):
 | `masterkey`                 | The Master API Key for LiteLLM. If not specified, a random key in the `sk-...` format is generated.                                                                                                                                           | N/A                       |
 | `environmentSecrets`        | An optional array of Secret object names. The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object.                                                   | `[]`                      |
 | `environmentConfigMaps`     | An optional array of ConfigMap object names. The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object.                                             | `[]`                      |
-| `image.repository`          | LiteLLM Proxy image repository                                                                                                                                                                                                                | `docker.litellm.ai/berriai/litellm` |
+| `image.repository`          | LiteLLM Proxy image repository                                                                                                                                                                                                                | `ghcr.io/berriai/litellm` |
 | `image.pullPolicy`          | LiteLLM Proxy image pull policy                                                                                                                                                                                                               | `IfNotPresent`            |
 | `image.tag`                 | Overrides the image tag whose default the latest version of LiteLLM at the time this chart was published.                                                                                                                                     | `""`                      |
 | `imagePullSecrets`          | Registry credentials for the LiteLLM and initContainer images.                                                                                                                                                                                | `[]`                      |

helm/litellm-helm/templates/migrations-job.yaml

@@ -105,6 +105,10 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       restartPolicy: OnFailure
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       {{- with .Values.affinity }}
       affinity:
         {{- toYaml . | nindent 8 }}
@@ -115,4 +119,7 @@ spec:
       {{- end }}
   ttlSecondsAfterFinished: {{ .Values.migrationJob.ttlSecondsAfterFinished }}
   backoffLimit: {{ .Values.migrationJob.backoffLimit }}
+  {{- with .Values.migrationJob.activeDeadlineSeconds }}
+  activeDeadlineSeconds: {{ . }}
+  {{- end }}
 {{- end }}

helm/litellm-helm/tests/deployment_tests.yaml

@@ -15,7 +15,7 @@ tests:
           pattern: -litellm$
       - equal:
           path: spec.template.spec.containers[0].image
-          value: ghcr.io/berriai/litellm-database:test
+          value: ghcr.io/berriai/litellm:test
   - it: should work with tolerations
     template: deployment.yaml
     set:
@@ -337,7 +337,7 @@ tests:
     template: deployment.yaml
     set:
       image:
-        repository: ghcr.io/berriai/litellm-database
+        repository: ghcr.io/berriai/litellm
         tag: test
       extraInitContainers:
         - name: init-tpl
@@ -348,7 +348,7 @@ tests:
           path: spec.template.spec.initContainers
           content:
             name: init-tpl
-            image: "ghcr.io/berriai/litellm-database:test"
+            image: "ghcr.io/berriai/litellm:test"
             command: ["echo", "hello"]
   - it: should work with extraContainers
     template: deployment.yaml
@@ -366,7 +366,7 @@ tests:
     template: deployment.yaml
     set:
       image:
-        repository: ghcr.io/berriai/litellm-database
+        repository: ghcr.io/berriai/litellm
         tag: test
       extraContainers:
         - name: sidecar-tpl
@@ -376,12 +376,12 @@ tests:
           path: spec.template.spec.containers
           content:
             name: sidecar-tpl
-            image: "ghcr.io/berriai/litellm-database:test"
+            image: "ghcr.io/berriai/litellm:test"
   - it: should support tpl in podAnnotations
     template: deployment.yaml
     set:
       image:
-        repository: ghcr.io/berriai/litellm-database
+        repository: ghcr.io/berriai/litellm
         tag: test
       # Mirrors the real-world scenario this feature unblocks:
       # user disables the built-in ConfigMap (and its built-in checksum/config
@@ -398,7 +398,7 @@ tests:
           value: "test"
       - equal:
           path: spec.template.metadata.annotations["example.com/some-key"]
-          value: "ghcr.io/berriai/litellm-database"
+          value: "ghcr.io/berriai/litellm"
       - equal:
           path: spec.template.metadata.annotations["example.com/literal"]
           value: "plain-string-value"

helm/litellm-helm/tests/migrations-job_tests.yaml

@@ -208,7 +208,7 @@ tests:
     template: migrations-job.yaml
     set:
       image:
-        repository: ghcr.io/berriai/litellm-database
+        repository: ghcr.io/berriai/litellm
         tag: test
       migrationJob:
         enabled: true
@@ -221,7 +221,7 @@ tests:
           path: spec.template.spec.initContainers
           content:
             name: init-tpl
-            image: "ghcr.io/berriai/litellm-database:test"
+            image: "ghcr.io/berriai/litellm:test"
             command: ["echo", "hello"]
   - it: should work with extraContainers
     template: migrations-job.yaml
@@ -241,7 +241,7 @@ tests:
     template: migrations-job.yaml
     set:
       image:
-        repository: ghcr.io/berriai/litellm-database
+        repository: ghcr.io/berriai/litellm
         tag: test
       migrationJob:
         enabled: true
@@ -253,7 +253,7 @@ tests:
           path: spec.template.spec.containers
           content:
             name: sidecar-tpl
-            image: "ghcr.io/berriai/litellm-database:test"
+            image: "ghcr.io/berriai/litellm:test"
   - it: should render the pod-level securityContext from podSecurityContext
     template: migrations-job.yaml
     set:
@@ -290,3 +290,55 @@ tests:
           value:
             allowPrivilegeEscalation: false
             readOnlyRootFilesystem: true
+  - it: should schedule onto the same nodes as the gateway
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+      nodeSelector:
+        karpenter.sh/nodepool: litellm-e2e
+      tolerations:
+        - key: workload
+          operator: Equal
+          value: litellm-e2e
+          effect: NoSchedule
+    asserts:
+      - equal:
+          path: spec.template.spec.nodeSelector
+          value:
+            karpenter.sh/nodepool: litellm-e2e
+      - equal:
+          path: spec.template.spec.tolerations
+          value:
+            - key: workload
+              operator: Equal
+              value: litellm-e2e
+              effect: NoSchedule
+
+  - it: bounds the Job with a deadline by default, so a blocked migration cannot stall the release forever
+    set:
+      migrationJob:
+        enabled: true
+    asserts:
+      - equal:
+          path: spec.activeDeadlineSeconds
+          value: 1800
+
+  - it: honours an operator-supplied deadline
+    set:
+      migrationJob:
+        enabled: true
+        activeDeadlineSeconds: 600
+    asserts:
+      - equal:
+          path: spec.activeDeadlineSeconds
+          value: 600
+
+  - it: omits the deadline entirely when it is nulled out, restoring the unbounded behaviour
+    set:
+      migrationJob:
+        enabled: true
+        activeDeadlineSeconds: null
+    asserts:
+      - notExists:
+          path: spec.activeDeadlineSeconds

helm/litellm-helm/values.yaml

@@ -6,8 +6,9 @@ replicaCount: 1
 # numWorkers: 2
 
 image:
-  # Use "ghcr.io/berriai/litellm-database" for optimized image with database
-  repository: ghcr.io/berriai/litellm-database
+  # Bundles the prisma CLI and engines, which is what lets the migrations job
+  # and the proxy's own schema check run without network access.
+  repository: ghcr.io/berriai/litellm
   pullPolicy: Always
   # Overrides the image tag whose default is the chart appVersion.
   # tag: "latest"
@@ -427,6 +428,13 @@ migrationJob:
   enabled: true # Enable or disable the schema migration Job
   retries: 3 # Number of retries for the Job in case of failure
   backoffLimit: 4 # Backoff limit for Job restarts
+  # Wall-clock budget for the whole Job, shared across every `backoffLimit`
+  # retry rather than granted per attempt. Without it a migration that blocks
+  # on the database never fails, and when the Helm hook is enabled the release
+  # waits on it forever: `helm upgrade` and any GitOps controller driving it
+  # stop reconciling the whole chart until someone deletes the Job by hand.
+  # Set to null to opt out and restore the unbounded behaviour.
+  activeDeadlineSeconds: 1800
   disableSchemaUpdate: false # Skip schema migrations for specific environments. When True, the job will exit with code 0.
   # Optional service account for the migration job.
   # Only used when migrationJob.hooks.helm.enabled=true and serviceAccount.create=true.

helm/litellm/templates/backend/deployment.yaml

@@ -81,6 +81,10 @@ spec:
           readinessProbe:
             {{- toYaml . | nindent 12 }}
           {{- end }}
+          {{- with .Values.backend.startupProbe }}
+          startupProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
           {{- with .Values.backend.lifecycle }}
           lifecycle:
             {{- toYaml . | nindent 12 }}

helm/litellm/templates/backend/hpa.yaml

@@ -30,4 +30,8 @@ spec:
           type: Utilization
           averageUtilization: {{ .Values.backend.hpa.targetMemoryUtilizationPercentage }}
     {{- end }}
+  {{- with .Values.backend.hpa.behavior }}
+  behavior:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
 {{- end }}

helm/litellm/templates/gateway/deployment.yaml

@@ -83,6 +83,10 @@ spec:
           readinessProbe:
             {{- toYaml . | nindent 12 }}
           {{- end }}
+          {{- with .Values.gateway.startupProbe }}
+          startupProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
           {{- with .Values.gateway.lifecycle }}
           lifecycle:
             {{- toYaml . | nindent 12 }}

helm/litellm/templates/gateway/hpa.yaml

@@ -30,4 +30,8 @@ spec:
           type: Utilization
           averageUtilization: {{ .Values.gateway.hpa.targetMemoryUtilizationPercentage }}
     {{- end }}
+  {{- with .Values.gateway.hpa.behavior }}
+  behavior:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
 {{- end }}

helm/litellm/templates/ingress.yaml

@@ -24,7 +24,7 @@
     "/v1/video" "/v1/videos" "/video" "/videos" "/v1/search" "/search"
     "/v1/containers" "/containers" "/v1/evals" "/v1/memory" "/queue/chat"
     "/v1beta" "/interactions"
-    "/anthropic" "/azure" "/azure_ai" "/aws" "/bedrock" "/cohere" "/gemini" "/google"
+    "/anthropic" "/azure" "/azure_ai" "/aws" "/bedrock" "/comprehendmedical" "/cohere" "/gemini" "/google"
     "/vertex_ai" "/vertex-ai" "/assemblyai" "/eu.assemblyai" "/langfuse" "/vllm"
     "/mistral" "/groq" "/voyage" "/cursor" "/milvus" "/openai_passthrough"
     "/toolset"

helm/litellm/templates/migrations-job.yaml

@@ -21,6 +21,9 @@ metadata:
 spec:
   backoffLimit: {{ .Values.migrationJob.backoffLimit }}
   ttlSecondsAfterFinished: {{ .Values.migrationJob.ttlSecondsAfterFinished }}
+  {{- with .Values.migrationJob.activeDeadlineSeconds }}
+  activeDeadlineSeconds: {{ . }}
+  {{- end }}
   template:
     metadata:
       {{- /* The Job's selector is generated by the controller rather than

helm/litellm/templates/ui/deployment.yaml

@@ -69,6 +69,10 @@ spec:
           readinessProbe:
             {{- toYaml . | nindent 12 }}
           {{- end }}
+          {{- with .Values.ui.startupProbe }}
+          startupProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
           {{- with .Values.ui.lifecycle }}
           lifecycle:
             {{- toYaml . | nindent 12 }}

helm/litellm/templates/ui/hpa.yaml

@@ -30,4 +30,8 @@ spec:
           type: Utilization
           averageUtilization: {{ .Values.ui.hpa.targetMemoryUtilizationPercentage }}
     {{- end }}
+  {{- with .Values.ui.hpa.behavior }}
+  behavior:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
 {{- end }}

helm/litellm/tests/hpa_behavior_tests.yaml

@@ -0,0 +1,58 @@
+suite: test HPA scaling behavior passthrough
+templates:
+  - gateway/hpa.yaml
+  - backend/hpa.yaml
+  - ui/hpa.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: HPA omits spec.behavior by default, so Kubernetes' default scaling applies
+    templates:
+      - gateway/hpa.yaml
+      - backend/hpa.yaml
+    asserts:
+      - isKind:
+          of: HorizontalPodAutoscaler
+      - notExists:
+          path: spec.behavior
+
+  - it: gateway HPA renders spec.behavior verbatim when configured
+    template: gateway/hpa.yaml
+    set:
+      gateway.hpa.behavior:
+        scaleDown:
+          stabilizationWindowSeconds: 300
+          policies:
+            - { type: Percent, value: 50, periodSeconds: 60 }
+        scaleUp:
+          stabilizationWindowSeconds: 0
+          selectPolicy: Max
+          policies:
+            - { type: Percent, value: 100, periodSeconds: 30 }
+            - { type: Pods, value: 2, periodSeconds: 30 }
+    asserts:
+      - equal:
+          path: spec.behavior
+          value:
+            scaleDown:
+              stabilizationWindowSeconds: 300
+              policies:
+                - { type: Percent, value: 50, periodSeconds: 60 }
+            scaleUp:
+              stabilizationWindowSeconds: 0
+              selectPolicy: Max
+              policies:
+                - { type: Percent, value: 100, periodSeconds: 30 }
+                - { type: Pods, value: 2, periodSeconds: 30 }
+
+  - it: behavior passthrough works on every autoscaled component (ui parity)
+    template: ui/hpa.yaml
+    set:
+      ui.hpa.enabled: true
+      ui.hpa.behavior:
+        scaleUp:
+          stabilizationWindowSeconds: 0
+    asserts:
+      - equal:
+          path: spec.behavior.scaleUp.stabilizationWindowSeconds
+          value: 0

helm/litellm/tests/migration_job_tests.yaml

@@ -167,3 +167,24 @@ tests:
       - equal:
           path: spec.template.metadata.labels['app.kubernetes.io/component']
           value: batch-migrations
+
+  - it: bounds the Job with a deadline by default, so a blocked migration cannot stall the release forever
+    asserts:
+      - equal:
+          path: spec.activeDeadlineSeconds
+          value: 1800
+
+  - it: honours an operator-supplied deadline
+    set:
+      migrationJob.activeDeadlineSeconds: 600
+    asserts:
+      - equal:
+          path: spec.activeDeadlineSeconds
+          value: 600
+
+  - it: omits the deadline entirely when it is nulled out, restoring the unbounded behaviour
+    set:
+      migrationJob.activeDeadlineSeconds: null
+    asserts:
+      - notExists:
+          path: spec.activeDeadlineSeconds

helm/litellm/tests/probe_tests.yaml

@@ -104,3 +104,30 @@ tests:
             periodSeconds: 15
             timeoutSeconds: 4
             failureThreshold: 3
+
+  - it: no startupProbe by default, so existing installs are unchanged
+    templates:
+      - gateway/deployment.yaml
+      - backend/deployment.yaml
+    asserts:
+      - notExists:
+          path: spec.template.spec.containers[0].startupProbe
+
+  - it: startupProbe renders verbatim when configured, gating a slow cold start
+    template: gateway/deployment.yaml
+    set:
+      gateway.startupProbe:
+        httpGet: { path: /health/readiness, port: http }
+        failureThreshold: 30
+        periodSeconds: 10
+        timeoutSeconds: 5
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe
+          value:
+            httpGet:
+              path: /health/readiness
+              port: http
+            failureThreshold: 30
+            periodSeconds: 10
+            timeoutSeconds: 5

helm/litellm/values.yaml

@@ -56,6 +56,15 @@ migrationJob:
   enabled: true
   backoffLimit: 4
   ttlSecondsAfterFinished: 120
+  # Wall-clock budget for the whole Job, shared across every `backoffLimit`
+  # retry rather than granted per attempt. Without it a migration that blocks
+  # on the database never fails, and because this is a pre-upgrade hook the
+  # release waits on it forever: `helm upgrade` and any GitOps controller
+  # driving it stop reconciling the whole chart until someone deletes the Job
+  # by hand. A migration that has exhausted its retries is not going to
+  # succeed on the next one, so failing is strictly better than hanging.
+  # Set to null to opt out and restore the unbounded behaviour.
+  activeDeadlineSeconds: 1800
   resources: {}
   # ServiceAccount for the Job pod only.
   #
@@ -223,12 +232,28 @@ gateway:
     initialDelaySeconds: 5
     periodSeconds: 10
     timeoutSeconds: 10
+  # Optional startupProbe. Empty by default, so existing installs are unchanged
+  # and liveness/readiness apply from container start. Set it to gate
+  # liveness/readiness until a slow cold start finishes — a high failureThreshold
+  # tolerates long first-boot times without a liveness-kill loop, e.g.:
+  #   httpGet: { path: /health/readiness, port: http }
+  #   failureThreshold: 30
+  #   periodSeconds: 10
+  startupProbe: {}
   hpa:
     enabled: true
     minReplicas: 1
     maxReplicas: 10
     targetCPUUtilizationPercentage: 70
     targetMemoryUtilizationPercentage: 80
+    # Optional autoscaling/v2 scaling behavior (scaleUp / scaleDown policies and
+    # stabilization windows). Empty by default -> Kubernetes' default behavior.
+    # Rendered verbatim under spec.behavior, e.g.:
+    #   scaleUp:
+    #     stabilizationWindowSeconds: 0
+    #     policies:
+    #       - { type: Percent, value: 100, periodSeconds: 30 }
+    behavior: {}
   # PodDisruptionBudget for the gateway pods. Set exactly one of
   # `minAvailable` / `maxUnavailable` (minAvailable wins if both are set;
   # enabling without either falls back to `maxUnavailable: 1`). Disabled by
@@ -319,11 +344,15 @@ backend:
     initialDelaySeconds: 5
     periodSeconds: 10
     timeoutSeconds: 10
+  # Optional startupProbe; same shape as gateway.startupProbe. Empty by default.
+  startupProbe: {}
   hpa:
     enabled: true
     minReplicas: 1
     maxReplicas: 4
     targetCPUUtilizationPercentage: 70
+    # Optional autoscaling/v2 scaling behavior; same shape as gateway.hpa.behavior.
+    behavior: {}
   # Same shape as gateway.pdb.
   pdb:
     enabled: false
@@ -379,11 +408,15 @@ ui:
     httpGet: { path: /, port: http }
     initialDelaySeconds: 2
     periodSeconds: 10
+  # Optional startupProbe; same shape as gateway.startupProbe. Empty by default.
+  startupProbe: {}
   hpa:
     enabled: false
     minReplicas: 1
     maxReplicas: 3
     targetCPUUtilizationPercentage: 80
+    # Optional autoscaling/v2 scaling behavior; same shape as gateway.hpa.behavior.
+    behavior: {}
   # Same shape as gateway.pdb.
   pdb:
     enabled: false

litellm-proxy-extras/litellm_proxy_extras/migrations/20260810000000_add_verificationtoken_settings_updated_at/migration.sql

@@ -0,0 +1,5 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN IF NOT EXISTS "settings_updated_at" TIMESTAMP(3);
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DeletedVerificationToken" ADD COLUMN IF NOT EXISTS "settings_updated_at" TIMESTAMP(3);

litellm-proxy-extras/litellm_proxy_extras/migrations/20260811172448_add_shadow_eval/migration.sql

@@ -0,0 +1,49 @@
+-- CreateTable
+CREATE TABLE "LiteLLM_ShadowEvalJob" (
+    "id" TEXT NOT NULL,
+    "api_key_id" TEXT NOT NULL,
+    "router_name" TEXT NOT NULL,
+    "judge_model" TEXT NOT NULL,
+    "shadow_percentage" DOUBLE PRECISION NOT NULL,
+    "max_turns" INTEGER NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "created_by" TEXT,
+    "ends_at" TIMESTAMP(3) NOT NULL,
+    "stopped_at" TIMESTAMP(3),
+
+    CONSTRAINT "LiteLLM_ShadowEvalJob_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "LiteLLM_ShadowEvalAttempt" (
+    "id" TEXT NOT NULL,
+    "job_id" TEXT NOT NULL,
+    "request_id" TEXT NOT NULL,
+    "outcome" TEXT NOT NULL,
+    "tier" TEXT,
+    "real_model" TEXT,
+    "shadow_model" TEXT,
+    "confidence" DOUBLE PRECISION,
+    "judge_cost" DOUBLE PRECISION NOT NULL DEFAULT 0,
+    "error" TEXT,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "LiteLLM_ShadowEvalAttempt_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_ShadowEvalJob_api_key_id_idx" ON "LiteLLM_ShadowEvalJob"("api_key_id");
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_ShadowEvalJob_created_at_idx" ON "LiteLLM_ShadowEvalJob"("created_at");
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_ShadowEvalAttempt_job_id_idx" ON "LiteLLM_ShadowEvalAttempt"("job_id");
+
+
+-- One active job per key, enforced by the database rather than a read-then-create in the
+-- start endpoint, which races against a concurrent start on another pod. Partial indexes
+-- are not expressible in schema.prisma, so this lives here only. Active means not yet
+-- stopped; the start endpoint stamps stopped_at on expired jobs before creating.
+CREATE UNIQUE INDEX "LiteLLM_ShadowEvalJob_one_active_per_key"
+    ON "LiteLLM_ShadowEvalJob"("api_key_id") WHERE "stopped_at" IS NULL;

litellm-proxy-extras/litellm_proxy_extras/migrations/20260813180408_add_shadow_eval_direction/migration.sql

@@ -0,0 +1,8 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN     "baseline_model" TEXT,
+ADD COLUMN     "direction" TEXT NOT NULL DEFAULT 'forward';
+
+DROP INDEX IF EXISTS "LiteLLM_ShadowEvalJob_one_active_per_key";
+
+CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalJob_one_active_per_key_direction"
+    ON "LiteLLM_ShadowEvalJob"("api_key_id", "direction") WHERE "stopped_at" IS NULL;

litellm-proxy-extras/litellm_proxy_extras/migrations/20260814000000_add_proxy_worker_heartbeat/migration.sql

@@ -0,0 +1,9 @@
+-- CreateTable
+CREATE TABLE "LiteLLM_ProxyWorkerHeartbeat" (
+    "worker_id" TEXT NOT NULL,
+    "hostname" TEXT NOT NULL,
+    "started_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "last_heartbeat_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "LiteLLM_ProxyWorkerHeartbeat_pkey" PRIMARY KEY ("worker_id")
+);

litellm-proxy-extras/litellm_proxy_extras/migrations/20260817000000_shadow_eval_multi_key/migration.sql

@@ -0,0 +1,7 @@
+ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN IF NOT EXISTS "group_id" TEXT;
+
+UPDATE "LiteLLM_ShadowEvalJob" SET "group_id" = "id" WHERE "group_id" IS NULL;
+
+ALTER TABLE "LiteLLM_ShadowEvalJob" ALTER COLUMN "group_id" SET NOT NULL;
+
+CREATE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalJob_group_id_idx" ON "LiteLLM_ShadowEvalJob"("group_id");

litellm-proxy-extras/litellm_proxy_extras/migrations/20260817143646_add_daily_guardrail_usage_units/migration.sql

@@ -0,0 +1,16 @@
+-- CreateTable
+CREATE TABLE "LiteLLM_DailyGuardrailUsageUnits" (
+    "guardrail_id" TEXT NOT NULL,
+    "date" TEXT NOT NULL,
+    "team_id" TEXT NOT NULL,
+    "api_key" TEXT NOT NULL,
+    "usage_unit" TEXT NOT NULL,
+    "units" BIGINT NOT NULL DEFAULT 0,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "LiteLLM_DailyGuardrailUsageUnits_pkey" PRIMARY KEY ("guardrail_id","date","team_id","api_key","usage_unit")
+);
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_DailyGuardrailUsageUnits_date_idx" ON "LiteLLM_DailyGuardrailUsageUnits"("date");

litellm-proxy-extras/litellm_proxy_extras/migrations/20260818000000_add_spend_log_timestamps/migration.sql

@@ -0,0 +1,3 @@
+ALTER TABLE "LiteLLM_SpendLogs"
+ADD COLUMN IF NOT EXISTS "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ADD COLUMN IF NOT EXISTS "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP;

litellm-proxy-extras/litellm_proxy_extras/migrations/20260818224500_add_shadow_eval_stopped_by/migration.sql

@@ -0,0 +1,5 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN     "stopped_by" TEXT;
+
+UPDATE "LiteLLM_ShadowEvalJob" SET stopped_by = 'unknown'
+WHERE stopped_at IS NOT NULL AND ends_at > (NOW() AT TIME ZONE 'utc');

litellm-proxy-extras/litellm_proxy_extras/migrations/20260819000000_backfill_spend_log_timestamps/migration.sql

@@ -0,0 +1,4 @@
+UPDATE "LiteLLM_SpendLogs"
+SET "created_at" = "endTime",
+    "updated_at" = "endTime"
+WHERE "created_at" > "endTime" + interval '1 hour';

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -452,6 +452,7 @@ model LiteLLM_VerificationToken {
     created_by String?
     updated_at      DateTime?               @default(now()) @updatedAt @map("updated_at")
     updated_by String?
+    settings_updated_at DateTime?             @map("settings_updated_at")
     last_active     DateTime?  // When this key was last used
     rotation_count   Int?       @default(0) // Number of times key has been rotated
     auto_rotate      Boolean?   @default(false) // Whether this key should be auto-rotated
@@ -548,6 +549,7 @@ model LiteLLM_DeletedVerificationToken {
     created_by              String?   // Original creator
     updated_at              DateTime? // Last update timestamp before deletion
     updated_by              String?   // Last user who updated before deletion
+    settings_updated_at       DateTime? // Last configuration change before deletion
     last_active             DateTime? // When this key was last used before deletion
     rotation_count          Int?      @default(0)
     auto_rotate             Boolean?  @default(false)
@@ -639,6 +641,8 @@ model LiteLLM_SpendLogs {
   mcp_namespaced_tool_name String? 
   agent_id            String?
   proxy_server_request Json?     @default("{}")
+  created_at          DateTime   @default(now()) @map("created_at")
+  updated_at          DateTime   @default(now()) @updatedAt @map("updated_at")
   @@index([startTime])
   @@index([startTime, request_id])
   @@index([end_user])
@@ -943,6 +947,17 @@ model LiteLLM_DailyTagSpend {
 }
 
 
+// One row per live proxy worker process. Workers upsert their row on a fixed
+// heartbeat; counting rows with a recent heartbeat tells how many workers share
+// this database, which lets the Admin UI hide its "no Redis" warning for
+// deployments that are provably a single worker.
+model LiteLLM_ProxyWorkerHeartbeat {
+  worker_id         String   @id
+  hostname          String
+  started_at        DateTime @default(now())
+  last_heartbeat_at DateTime @default(now())
+}
+
 // Track the status of cron jobs running. Only allow one pod to run the job at a time
 model LiteLLM_CronJob {
   cronjob_id            String   @id @default(cuid()) // Unique ID for the record
@@ -1067,6 +1082,21 @@ model LiteLLM_DailyGuardrailMetrics {
   @@index([guardrail_id])
 }
 
+// Daily guardrail billable usage units (one row per guardrail/day/team/key/unit type)
+model LiteLLM_DailyGuardrailUsageUnits {
+  guardrail_id String
+  date         String   // YYYY-MM-DD
+  team_id      String   // empty string when the request had no team
+  api_key      String   // hashed virtual key; empty string when unknown
+  usage_unit   String   // provider counter name, e.g. Bedrock's contentPolicyUnits
+  units        BigInt   @default(0)
+  created_at   DateTime @default(now())
+  updated_at   DateTime @updatedAt
+
+  @@id([guardrail_id, date, team_id, api_key, usage_unit])
+  @@index([date])
+}
+
 // Daily policy metrics for usage dashboard (one row per policy per day)
 model LiteLLM_DailyPolicyMetrics {
   policy_id           String
@@ -1448,6 +1478,59 @@ model LiteLLM_AutoRouterSession {
   @@index([last_turn_at], map: "idx_autorouter_session_last_turn")
 }
 
+// Shadow eval: evaluation of an auto-router against one or more keys' live traffic, in
+// either direction. forward duplicates the requests the keys did not route through the
+// router through it, answering whether they should adopt it; reverse duplicates the
+// requests the router did serve against a fixed baseline model, answering whether a key
+// already on it still benefits. Either way a sampled slice runs in a detached task and an
+// LLM judge compares real vs shadow responses blind. Each row is ONE key's leg of a job:
+// immutable config plus that key's own turn budget and stop state, so one key exhausting
+// its budget never ends a sibling's sampling. A job is the set of legs sharing group_id
+// (the id the API reports), written together by one atomic create_many with identical
+// config; single-key jobs predating group_id were backfilled group_id = id. "One active
+// job per (key, direction)" is a partial unique index on (api_key_id, direction) WHERE
+// stopped_at IS NULL, expressed only in the migration because schema.prisma cannot state
+// partial indexes; it is what makes a concurrent start on another pod race-safe rather
+// than read-then-create. Every count, status, and spend figure is derived from the
+// append-only attempt rows, so nothing can disagree across pods or stop races.
+model LiteLLM_ShadowEvalJob {
+  id                String   @id @default(cuid())
+  group_id          String   // legs of one job share this; the API's job id
+  api_key_id        String   // hashed virtual key whose traffic this leg shadows
+  router_name       String   // the auto-router under evaluation, in either direction
+  direction         String   @default("forward") // forward | reverse
+  baseline_model    String?  // reverse only: the fixed model the router is judged against
+  judge_model       String
+  shadow_percentage Float
+  max_turns         Int      // this key's sample budget: judge at most this many turns
+  created_at        DateTime @default(now())
+  created_by        String?
+  ends_at           DateTime
+  stopped_at        DateTime?
+  stopped_by        String?  // operator who stopped it early; null when it ended on its own
+
+  @@index([group_id])
+  @@index([api_key_id])
+  @@index([created_at])
+}
+
+// One row per sampled pipeline: a blind verdict (real | shadow | tie) or an error.
+model LiteLLM_ShadowEvalAttempt {
+  id           String   @id @default(cuid())
+  job_id       String
+  request_id   String   // the judged real request
+  outcome      String   // real | shadow | tie | error
+  tier         String?  // router's tier for the prompt, when classified
+  real_model   String?
+  shadow_model String?
+  confidence   Float?
+  judge_cost   Float    @default(0)
+  error        String?
+  created_at   DateTime @default(now())
+
+  @@index([job_id])
+}
+
 // ---------------------------------------------------------------------------
 // Workflow Run Tracking
 //

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.84"
+version = "0.4.87"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.84"
+version = "0.4.87"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm/__init__.py

@@ -26,6 +26,7 @@ def _dev_env_hot_reload_enabled() -> bool:
 if os.getenv("LITELLM_MODE", "DEV") == "DEV":
     _dotenv.load_dotenv(override=_dev_env_hot_reload_enabled())
 
+from collections.abc import Sequence
 from typing import (
     Any,
     Callable,
@@ -172,6 +173,7 @@ callbacks: List[
 callback_settings: Dict[str, Dict[str, Any]] = {}
 initialized_langfuse_clients: int = 0
 langfuse_default_tags: Optional[List[str]] = None
+langfuse_enable_update_trace_keys: bool = False
 langsmith_batch_size: Optional[int] = None
 prometheus_initialize_budget_metrics: Optional[bool] = False
 prometheus_latency_buckets: Optional[List[float]] = None
@@ -216,7 +218,10 @@ add_user_information_to_llm_headers: Optional[bool] = (
 overwrite_user_with_key_hash: bool = (
     False  # force the outgoing `user` param to the hashed api key, so providers see a stable, tamper-proof id
 )
-store_audit_logs = False  # Enterprise feature, allow users to see audit logs
+bedrock_request_metadata_fields: Optional[Sequence[str]] = (
+    None  # allow-list of `user_api_key_*` fields (+ `spend_logs_metadata`) sent as Bedrock `requestMetadata`
+)
+store_audit_logs: bool | None = None
 skip_system_message_in_guardrail: bool = False
 skip_tool_message_in_guardrail: bool = False
 ### end of callbacks #############
@@ -246,6 +251,7 @@ use_chat_completions_url_for_anthropic_messages: bool = bool(
 # config.yaml.
 strip_anthropic_total_tokens: bool = False
 anthropic_sse_ping_interval_seconds: float = 15.0
+sse_keepalive_ping_interval_seconds: float | None = None
 route_all_chat_openai_to_responses: bool = (
     os.getenv("LITELLM_ROUTE_ALL_CHAT_OPENAI_TO_RESPONSES", "false").lower() == "true"
 )  # When True, routes all OpenAI /chat/completions requests through the Responses API bridge
@@ -786,6 +792,8 @@ def _populate_provider_model_sets(model_cost_map: Dict) -> None:
             nlp_cloud_models.add(key)
         elif value.get("litellm_provider") == "aleph_alpha":
             aleph_alpha_models.add(key)
+        elif value.get("litellm_provider") == "bedrock" and value.get("mode") == "guardrail":
+            pass
         elif value.get("litellm_provider") == "bedrock" and not is_bedrock_pricing_only_model(key):
             bedrock_models.add(key)
         elif value.get("litellm_provider") == "bedrock_converse":

litellm/_logging.py

@@ -10,7 +10,7 @@ from typing import Any, Final
 import litellm
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
-from litellm.litellm_core_utils.secret_redaction import redact_string
+from litellm.litellm_core_utils.secret_redaction import redact_string, redact_structured_value
 
 set_verbose = False
 
@@ -59,6 +59,12 @@ def _redact_string(value: str) -> str:
     return redact_string(value)
 
 
+def _redact_structured_value(key: str | None, value: str) -> str:
+    if not _ENABLE_SECRET_REDACTION:
+        return value
+    return redact_structured_value(key, value)
+
+
 def redact_secrets(value: str) -> str:
     """Public API: redact known secret/credential patterns from an arbitrary string.
 
@@ -265,7 +271,7 @@ class JsonFormatter(Formatter):
         if record.exc_info:
             json_record["stacktrace"] = record.exc_text or self.formatException(record.exc_info)
 
-        return safe_dumps(json_record)
+        return safe_dumps(json_record, value_transform=_redact_structured_value)
 
 
 class CorrelationPlainFormatter(logging.Formatter):
@@ -276,7 +282,7 @@ class CorrelationPlainFormatter(logging.Formatter):
     """
 
     def format(self, record: logging.LogRecord) -> str:
-        formatted: Final = super().format(record)
+        formatted: Final = _redact_string(super().format(record))
         trace_id: Final = getattr(record, "trace_id", None)
         session_id: Final = getattr(record, "session_id", None)
         if not trace_id and not session_id:

litellm/_redis.py

@@ -67,12 +67,20 @@ def _init_arg_names(cls: type) -> frozenset[str]:
 
     Keyword-only parameters are included, and the MRO is walked because redis-py splits a
     connection's parameters between ``AbstractConnection`` and its concrete subclasses.
+
+    Each ``__init__`` is unwrapped before introspection: redis-py >= 7.4 decorates
+    ``AbstractConnection.__init__`` with ``@deprecated_args``, whose wrapper is declared
+    ``(self, *args, **kwargs)`` — introspecting the wrapper directly loses every real
+    parameter (``socket_timeout`` included), which silently emptied this allowlist and
+    dropped the socket timeouts from url-configured connections. ``inspect.unwrap``
+    follows the ``__wrapped__`` chain to the true signature and is a no-op on
+    undecorated ``__init__``s.
     """
     return frozenset(
         name
         for klass in inspect.getmro(cls)
         if klass is not object
-        for spec in (inspect.getfullargspec(klass.__init__),)
+        for spec in (inspect.getfullargspec(inspect.unwrap(klass.__init__)),)
         for name in spec.args + spec.kwonlyargs
     )
 

litellm/a2a_protocol/litellm_completion_bridge/handler.py

@@ -10,7 +10,7 @@ A2A Streaming Events (in order):
 4. Status update (kind: "status-update") - Final status "completed" with final=true
 """
 
-from collections.abc import AsyncIterator, Mapping
+from collections.abc import AsyncIterator, Callable, Coroutine, Mapping
 from typing import Any, Final
 
 import litellm
@@ -54,7 +54,7 @@ class A2ACompletionBridgeHandler:
         agent_extra_headers: Mapping[str, str] | None,
         *,
         stream: bool,
-    ) -> Mapping[str, Any]:
+    ) -> Mapping[str, object]:
         # Extract message from params
         message: Final = params.get("message", {})
 
@@ -63,7 +63,7 @@ class A2ACompletionBridgeHandler:
 
         # Get completion params
         custom_llm_provider: Final = litellm_params.get("custom_llm_provider")
-        model: Final = litellm_params.get("model", "agent")
+        model: Final[str] = litellm_params.get("model", "agent")
 
         # Build full model string if provider specified
         # Skip prepending if model already starts with the provider prefix
@@ -109,13 +109,16 @@ class A2ACompletionBridgeHandler:
         return completion_params
 
     @staticmethod
-    async def _acompletion(completion_params: Mapping[str, Any]) -> ModelResponse | CustomStreamWrapper:
-        return await litellm.acompletion(**completion_params)
+    async def _acompletion(completion_params: Mapping[str, object]) -> ModelResponse | CustomStreamWrapper:
+        acompletion_fn: Final[Callable[..., Coroutine[object, object, ModelResponse | CustomStreamWrapper]]] = vars(
+            litellm
+        )["acompletion"]
+        return await acompletion_fn(**completion_params)
 
     @staticmethod
     async def handle_non_streaming(
         request_id: str,
-        params: dict[str, Any],
+        params: dict[str, object],
         litellm_params: dict[str, Any],
         api_base: str | None = None,
         agent_extra_headers: dict[str, str] | None = None,
@@ -296,8 +299,8 @@ class A2ACompletionBridgeHandler:
 # Convenience functions that delegate to the class methods
 async def handle_a2a_completion(
     request_id: str,
-    params: dict[str, Any],
-    litellm_params: dict[str, Any],
+    params: dict[str, object],
+    litellm_params: dict[str, object],
     api_base: str | None = None,
     agent_extra_headers: dict[str, str] | None = None,
 ) -> dict[str, object]:
@@ -313,8 +316,8 @@ async def handle_a2a_completion(
 
 async def handle_a2a_completion_streaming(
     request_id: str,
-    params: dict[str, Any],
-    litellm_params: dict[str, Any],
+    params: dict[str, object],
+    litellm_params: dict[str, object],
     api_base: str | None = None,
     agent_extra_headers: dict[str, str] | None = None,
 ) -> AsyncIterator[dict[str, object]]:

litellm/a2a_protocol/main.py

@@ -12,7 +12,8 @@ Provides standalone functions with @client decorator for LiteLLM logging integra
 import asyncio
 import datetime
 import uuid
-from collections.abc import AsyncIterator, Coroutine
+from collections.abc import AsyncIterator, Coroutine, Mapping
+from types import ModuleType
 from typing import TYPE_CHECKING, Any, Final, Optional, cast
 
 import litellm
@@ -38,12 +39,15 @@ if TYPE_CHECKING:
         SendMessageResponse,
         SendStreamingMessageRequest,
         SendStreamingMessageResponse,
+        SendStreamingMessageSuccessResponse,
         Task,
     )
+    from a2a.types.a2a_pb2 import SendMessageRequest as CoreSendMessageRequest
+    from a2a.types.a2a_pb2 import StreamResponse as CoreStreamResponse
 
 # Runtime imports — requires a2a-sdk>=1.1.0
 A2A_SDK_AVAILABLE = False
-_a2a_conversions: Any = None
+_a2a_conversions: ModuleType | None = None
 
 try:
     from a2a.client import Client, ClientCallContext, ClientConfig, create_client
@@ -128,7 +132,7 @@ _A2A_COST_PARAM_KEYS: Final = ("cost_per_query", "input_cost_per_token", "output
 
 def _set_litellm_params_on_logging_obj(
     kwargs: dict[str, Any],
-    litellm_params: dict[str, Any],
+    litellm_params: Mapping[str, object],
 ) -> None:
     """
     Merge the agent's pricing params into model_call_details["litellm_params"]
@@ -150,7 +154,7 @@ def _set_litellm_params_on_logging_obj(
     logging_obj.model_call_details["litellm_params"] = {**existing, **cost_params}
 
 
-def _get_a2a_model_info(a2a_client: Any, kwargs: dict[str, Any]) -> str:
+def _get_a2a_model_info(a2a_client: "A2AClientType", kwargs: dict[str, Any]) -> str:
     """
     Extract agent info and set model/custom_llm_provider for cost tracking.
 
@@ -179,7 +183,7 @@ def _get_a2a_model_info(a2a_client: Any, kwargs: dict[str, Any]) -> str:
     return agent_name
 
 
-def _get_a2a_client_agent_card(a2a_client: Any) -> Optional["AgentCard"]:
+def _get_a2a_client_agent_card(a2a_client: "A2AClientType") -> Optional["AgentCard"]:
     agent_card = cast(Optional["AgentCard"], getattr(a2a_client, "_litellm_agent_card", None))
     if agent_card is not None:
         return agent_card
@@ -191,9 +195,9 @@ def _get_a2a_client_agent_card(a2a_client: Any) -> Optional["AgentCard"]:
 
 async def _send_message_via_completion_bridge(
     request: "SendMessageRequest",
-    custom_llm_provider: str,
+    custom_llm_provider: object,
     api_base: str | None,
-    litellm_params: dict[str, Any],
+    litellm_params: dict[str, object],
     agent_extra_headers: dict[str, str] | None = None,
 ) -> LiteLLMSendMessageResponse:
     """
@@ -224,6 +228,20 @@ def _get_a2a_call_context(a2a_client: "A2AClientType") -> Optional["A2ACallConte
     return getattr(a2a_client, "_litellm_call_context", None)
 
 
+def _to_core_send_message_request(request: "SendMessageRequest") -> "CoreSendMessageRequest":
+    from a2a.compat.v0_3 import conversions
+
+    return conversions.to_core_send_message_request(request)
+
+
+def _to_compat_stream_response(
+    event: "CoreStreamResponse", request_id: str | int
+) -> "SendStreamingMessageSuccessResponse":
+    from a2a.compat.v0_3 import conversions
+
+    return conversions.to_compat_stream_response(event, request_id=request_id)
+
+
 async def _send_message(a2a_client: "A2AClientType", request: "SendMessageRequest") -> "SendMessageResponse":
     """Send a non-streaming message via a2a-sdk 1.x and return JSON-RPC response."""
     if _a2a_conversions is None:
@@ -231,17 +249,14 @@ async def _send_message(a2a_client: "A2AClientType", request: "SendMessageReques
             "The 'a2a' package is required for A2A agent invocation. Install it with: pip install a2a-sdk"
         )
 
-    pb_request: Final = _a2a_conversions.to_core_send_message_request(request)
+    pb_request: Final = _to_core_send_message_request(request)
     last_event = None
     async for event in a2a_client.send_message(pb_request, context=_get_a2a_call_context(a2a_client)):
         last_event = event
     if last_event is None:
         raise RuntimeError("A2A send_message failed: no response received from agent.")
 
-    stream_compat: Final = _a2a_conversions.to_compat_stream_response(
-        last_event,
-        request_id=request.id,
-    )
+    stream_compat: Final = _to_compat_stream_response(last_event, request_id=request.id)
     result: Final = stream_compat.result
     if not isinstance(result, (Message, Task)):
         raise RuntimeError(
@@ -306,12 +321,9 @@ async def _stream_messages(
             "The 'a2a' package is required for A2A agent invocation. Install it with: pip install a2a-sdk"
         )
 
-    pb_request: Final = _a2a_conversions.to_core_send_message_request(request)
+    pb_request: Final[CoreSendMessageRequest] = _a2a_conversions.to_core_send_message_request(request)
     async for event in a2a_client.send_message(pb_request, context=_get_a2a_call_context(a2a_client)):
-        compat_chunk = _a2a_conversions.to_compat_stream_response(
-            event,
-            request_id=request.id,
-        )
+        compat_chunk = _to_compat_stream_response(event, request_id=request.id)
         yield SendStreamingMessageResponse(root=compat_chunk)
 
 
@@ -368,10 +380,10 @@ async def asend_message(
     a2a_client: Optional["A2AClientType"] = None,
     request: Optional["SendMessageRequest"] = None,
     api_base: str | None = None,
-    litellm_params: dict[str, Any] | None = None,
+    litellm_params: dict[str, object] | None = None,
     agent_id: str | None = None,
     agent_extra_headers: dict[str, str] | None = None,
-    **kwargs: Any,
+    **kwargs: object,
 ) -> LiteLLMSendMessageResponse:
     """
     Async: Send a message to an A2A agent.
@@ -485,7 +497,7 @@ async def asend_message(
     response: Final = LiteLLMSendMessageResponse.from_a2a_response(a2a_response, request_id=str(request.id))
 
     # Calculate token usage from request and response
-    response_dict: Final = a2a_response.model_dump(mode="json", exclude_none=True)
+    response_dict: Final[dict[str, object]] = a2a_response.model_dump(mode="json", exclude_none=True)
     (
         prompt_tokens,
         completion_tokens,
@@ -516,7 +528,7 @@ def send_message(
     a2a_client: "A2AClientType",
     request: "SendMessageRequest",
     **kwargs: Any,
-) -> LiteLLMSendMessageResponse | Coroutine[Any, Any, LiteLLMSendMessageResponse]:
+) -> LiteLLMSendMessageResponse | Coroutine[object, object, LiteLLMSendMessageResponse]:
     """
     Sync: Send a message to an A2A agent.
 
@@ -545,9 +557,9 @@ def _build_streaming_logging_obj(
     request: "SendStreamingMessageRequest",
     agent_name: str,
     agent_id: str | None,
-    litellm_params: dict[str, Any] | None,
-    metadata: dict[str, Any] | None,
-    proxy_server_request: dict[str, Any] | None,
+    litellm_params: dict[str, object] | None,
+    metadata: dict[str, object] | None,
+    proxy_server_request: dict[str, object] | None,
 ) -> Logging:
     """Build logging object for streaming A2A requests."""
     start_time: Final = datetime.datetime.now()
@@ -588,10 +600,10 @@ async def asend_message_streaming(
     a2a_client: Optional["A2AClientType"] = None,
     request: Optional["SendStreamingMessageRequest"] = None,
     api_base: str | None = None,
-    litellm_params: dict[str, Any] | None = None,
+    litellm_params: dict[str, object] | None = None,
     agent_id: str | None = None,
-    metadata: dict[str, Any] | None = None,
-    proxy_server_request: dict[str, Any] | None = None,
+    metadata: dict[str, object] | None = None,
+    proxy_server_request: dict[str, object] | None = None,
     agent_extra_headers: dict[str, str] | None = None,
     **kwargs: object,
 ) -> AsyncIterator[Any]:

litellm/a2a_protocol/providers/watsonx_orchestrate/handler.py

@@ -7,9 +7,10 @@ import hashlib
 import json
 import time
 from collections.abc import AsyncIterator
-from typing import Any, Final, NamedTuple, cast
+from typing import Any, Final, NamedTuple, Protocol
 
 import httpx
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 from litellm._logging import verbose_logger
 from litellm.a2a_protocol.providers.watsonx_orchestrate.transformation import (
@@ -38,11 +39,59 @@ class WXORequestParams(NamedTuple):
     thread_id: str | None
 
 
+class WXOLitellmParams(TypedDict, total=False):
+    """litellm_params keys read when routing an A2A request to watsonx Orchestrate."""
+
+    cp4d_host: ReadOnly[str]
+    instance_id: ReadOnly[str]
+    wxo_agent_id: ReadOnly[str]
+    api_key: ReadOnly[str]
+    username: ReadOnly[str | None]
+    auth_mode: ReadOnly[str]
+    thread_id: ReadOnly[str | None]
+
+
+class _IBMCloudTokenBody(TypedDict):
+    """Fields read from the IBM Cloud IAM token response."""
+
+    access_token: ReadOnly[str]
+    expires_in: ReadOnly[NotRequired[int]]
+
+
+class _CP4DTokenBody(TypedDict):
+    """Fields read from the CP4D authorize response."""
+
+    token: ReadOnly[str]
+    expiration: ReadOnly[NotRequired[float]]
+
+
+class _WXORun(TypedDict, total=False):
+    """Fields the handler reads from a WXO run object or run event."""
+
+    status: ReadOnly[str]
+    run_id: ReadOnly[str]
+    id: ReadOnly[str]
+
+
+class _SSELineSource(Protocol):
+    def aiter_lines(self) -> AsyncIterator[str]: ...
+
+
+class _WXOView(TypedDict, total=False):
+    """Typed reads of otherwise untyped watsonx Orchestrate and httpx values."""
+
+    ibm_cloud_token: ReadOnly[_IBMCloudTokenBody]
+    cp4d_token: ReadOnly[_CP4DTokenBody]
+    run: ReadOnly[_WXORun]
+    content_type: ReadOnly[str]
+    sse_source: ReadOnly[_SSELineSource]
+
+
 class WatsonxOrchestrateHandler:
     @staticmethod
     def _http_client(timeout: float = 90.0) -> AsyncHTTPHandler:
         return get_async_httpx_client(
-            llm_provider=cast(Any, httpxSpecialProvider.A2AProvider),
+            llm_provider=httpxSpecialProvider.A2AProvider,
             params={"timeout": timeout},
         )
 
@@ -57,7 +106,7 @@ class WatsonxOrchestrateHandler:
         return hashlib.sha256(material.encode()).hexdigest()
 
     @staticmethod
-    def _cp4d_token_ttl_seconds(expiration: Any, now_wall: float | None = None) -> int:
+    def _cp4d_token_ttl_seconds(expiration: float, now_wall: float | None = None) -> int:
         # CP4D returns expiration as absolute Unix epoch seconds, not a duration.
         expires_at: Final = int(expiration)
         wall: Final = now_wall if now_wall is not None else time.time()
@@ -90,9 +139,9 @@ class WatsonxOrchestrateHandler:
                 headers={"Content-Type": "application/x-www-form-urlencoded"},
             )
             response.raise_for_status()
-            payload = response.json()
-            token = str(payload["access_token"])
-            ttl_s = int(payload.get("expires_in", 3600))
+            iam_payload: Final[_WXOView] = {"ibm_cloud_token": response.json()}
+            token = str(iam_payload["ibm_cloud_token"]["access_token"])
+            ttl_s = int(iam_payload["ibm_cloud_token"].get("expires_in", 3600))
         else:
             if not username:
                 raise ValueError("'username' is required in litellm_params when auth_mode='cp4d'")
@@ -103,9 +152,9 @@ class WatsonxOrchestrateHandler:
                 headers={"Content-Type": "application/json"},
             )
             response.raise_for_status()
-            payload = response.json()
-            token = str(payload["token"])
-            expiration: Final = payload.get("expiration")
+            cp4d_payload: Final[_WXOView] = {"cp4d_token": response.json()}
+            token = str(cp4d_payload["cp4d_token"]["token"])
+            expiration: Final = cp4d_payload["cp4d_token"].get("expiration")
             if expiration is None:
                 ttl_s = 3600
             else:
@@ -118,6 +167,16 @@ class WatsonxOrchestrateHandler:
                 del _token_cache[stale_key]
         return token
 
+    @staticmethod
+    def _run_body(response: httpx.Response) -> _WXORun:
+        view: Final[_WXOView] = {"run": response.json()}
+        return view["run"]
+
+    @staticmethod
+    def _decode_run_event(payload: str | bytes) -> _WXORun:
+        view: Final[_WXOView] = {"run": json.loads(payload)}
+        return view["run"]
+
     @staticmethod
     async def _poll_run(
         base_url: str,
@@ -126,14 +185,14 @@ class WatsonxOrchestrateHandler:
         client: AsyncHTTPHandler,
         max_attempts: int = _MAX_POLL_ATTEMPTS,
         interval_s: float = _POLL_INTERVAL_S,
-    ) -> dict[str, Any]:
+    ) -> _WXORun:
         url: Final = f"{base_url}/v1/orchestrate/runs/{run_id}"
 
         for attempt in range(max_attempts):
             await asyncio.sleep(interval_s)
             response = await client.get(url, headers=auth_headers)
             response.raise_for_status()
-            result: dict[str, Any] = response.json()
+            result = WatsonxOrchestrateHandler._run_body(response)
             status = result.get("status", "")
             verbose_logger.debug("WXO: Poll %s/%s run='%s' status='%s'", attempt + 1, max_attempts, run_id, status)
             if status in WatsonxOrchestrateTransformation.TERMINAL_STATES:
@@ -145,11 +204,11 @@ class WatsonxOrchestrateHandler:
 
     @staticmethod
     async def _get_successful_run_data(
-        run_data: dict[str, Any],
+        run_data: _WXORun,
         base_url: str,
         auth_headers: dict[str, str],
         client: AsyncHTTPHandler,
-    ) -> dict[str, Any]:
+    ) -> _WXORun:
         status = run_data.get("status", "")
         if status not in WatsonxOrchestrateTransformation.TERMINAL_STATES:
             run_id: Final = run_data.get("run_id") or run_data.get("id") or ""
@@ -170,15 +229,16 @@ class WatsonxOrchestrateHandler:
 
     @staticmethod
     async def _accumulate_wxo_sse_text(response: Any) -> str:
+        source: Final[_WXOView] = {"sse_source": response}
         accumulated_text = ""
-        async for line in response.aiter_lines():
+        async for line in source["sse_source"].aiter_lines():
             if not line.startswith("data:"):
                 continue
             data_str = line[5:].strip()
             if not data_str or data_str == "[DONE]":
                 continue
             try:
-                event = json.loads(data_str)
+                event = WatsonxOrchestrateHandler._decode_run_event(data_str)
             except json.JSONDecodeError:
                 continue
             chunk_text = WatsonxOrchestrateTransformation.extract_text_from_wxo_result(event)
@@ -187,7 +247,7 @@ class WatsonxOrchestrateHandler:
         return accumulated_text
 
     @staticmethod
-    def _extract_litellm_params(litellm_params: dict[str, Any]) -> WXORequestParams:
+    def _extract_litellm_params(litellm_params: WXOLitellmParams) -> WXORequestParams:
         cp4d_host: Final = litellm_params.get("cp4d_host") or ""
         instance_id: Final = litellm_params.get("instance_id") or ""
         wxo_agent_id: Final = litellm_params.get("wxo_agent_id") or ""
@@ -215,9 +275,9 @@ class WatsonxOrchestrateHandler:
     @staticmethod
     async def handle_non_streaming(
         request_id: str,
-        params: dict[str, Any],
-        litellm_params: dict[str, Any],
-    ) -> dict[str, Any]:
+        params: dict[str, object],
+        litellm_params: WXOLitellmParams,
+    ) -> dict[str, object]:
         wxo: Final = WatsonxOrchestrateHandler._extract_litellm_params(litellm_params)
 
         client: Final = WatsonxOrchestrateHandler._http_client(timeout=90.0)
@@ -246,7 +306,8 @@ class WatsonxOrchestrateHandler:
             headers=auth_headers,
         )
         run_response.raise_for_status()
-        run_data: dict[str, Any] = run_response.json()
+        started: Final[_WXOView] = {"run": run_response.json()}
+        run_data: _WXORun = started["run"]
 
         run_data = await WatsonxOrchestrateHandler._get_successful_run_data(
             run_data=run_data,
@@ -261,11 +322,11 @@ class WatsonxOrchestrateHandler:
     @staticmethod
     async def handle_streaming(
         request_id: str,
-        params: dict[str, Any],
-        litellm_params: dict[str, Any],
+        params: dict[str, object],
+        litellm_params: WXOLitellmParams,
         chunk_size: int = 50,
         delay_ms: int = 10,
-    ) -> AsyncIterator[dict[str, Any]]:
+    ) -> AsyncIterator[dict[str, object]]:
         wxo: Final = WatsonxOrchestrateHandler._extract_litellm_params(litellm_params)
 
         client: Final = WatsonxOrchestrateHandler._http_client(timeout=120.0)
@@ -316,10 +377,11 @@ class WatsonxOrchestrateHandler:
                 yield chunk
             return
 
-        content_type: Final = response.headers.get("content-type", "").lower()
+        header_view: Final[_WXOView] = {"content_type": response.headers.get("content-type", "")}
+        content_type: Final = header_view["content_type"].lower()
         if "text/event-stream" not in content_type:
             response_body: Final = await response.aread()
-            result = json.loads(response_body)
+            result = WatsonxOrchestrateHandler._decode_run_event(response_body)
             result = await WatsonxOrchestrateHandler._get_successful_run_data(
                 run_data=result,
                 base_url=base_url,

litellm/batches/batch_utils.py

@@ -1,11 +1,12 @@
 import json
-from collections.abc import Iterable, Iterator
+from collections.abc import Iterable, Iterator, Mapping
 from dataclasses import dataclass
 from typing import Any, Final, Literal
 
 import litellm
 from litellm._logging import verbose_logger
-from litellm.litellm_core_utils.llm_cost_calc.utils import _parse_prompt_tokens_details
+from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS
+from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details
 from litellm.types.llms.openai import Batch
 from litellm.types.utils import CallTypes, ModelInfo, Usage
 from litellm.utils import token_counter
@@ -47,6 +48,7 @@ async def _handle_completed_batch(
     custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"],
     model_name: str | None = None,
     litellm_params: dict | None = None,
+    model_info: ModelInfo | None = None,
 ) -> tuple[float, Usage, list[str]]:
     """Fetch a completed batch's output file and aggregate its cost, usage, and
     models in a single pass over the JSONL lines, so the parsed file content is
@@ -57,7 +59,21 @@ async def _handle_completed_batch(
         custom_llm_provider: The LLM provider
         model_name: Optional model name
         litellm_params: Optional litellm parameters containing credentials (api_key, api_base, etc.)
+        model_info: Optional deployment-level model info with custom pricing,
+            threaded through so a deployment's configured rates win over the
+            global cost map.
     """
+    # A completed batch whose request lines all failed has no output file - the
+    # results are written to a separate error_file_id and output_file_id is None.
+    # There is nothing to price or measure, so report an empty result set instead
+    # of calling _fetch_batch_output_file_content, which raises on a missing
+    # output file. Without this guard the logging worker crashes on every
+    # aretrieve_batch poll and the completed batch's zero-cost accounting is lost.
+    # The generic retrieval helper keeps raising for callers that explicitly ask
+    # for a missing output file.
+    if batch.output_file_id is None:
+        return 0.0, Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0), []
+
     file_content = await _fetch_batch_output_file_content(batch, custom_llm_provider, litellm_params=litellm_params)
 
     if (
@@ -71,9 +87,10 @@ async def _handle_completed_batch(
         return batch_cost, batch_usage, [model_name]
 
     return _aggregate_batch_cost_usage_models(
-        entries=_iter_batch_input_entries(file_content),
+        entries=_iter_batch_output_entries(file_content),
         custom_llm_provider=custom_llm_provider,
         model_name=model_name,
+        model_info=model_info,
     )
 
 
@@ -94,43 +111,91 @@ def _iter_successful_output_line_stats(
     model_name: str | None,
     model_info: ModelInfo | None,
 ) -> Iterator[_BatchOutputLineStats]:
+    for entry in entries:
+        stats = _safe_output_line_stats(entry, custom_llm_provider, model_name, model_info)
+        if stats is not None:
+            yield stats
+
+
+def _safe_output_line_stats(
+    entry: Mapping[str, Any],
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"],
+    model_name: str | None,
+    model_info: ModelInfo | None,
+) -> _BatchOutputLineStats | None:
+    """Return the stats for one batch output line, or None for a line that is
+    unsuccessful or cannot be costed, so a single bad line never aborts the
+    whole batch's cost accounting."""
+    custom_id: Final = entry.get("custom_id") if isinstance(entry, dict) else None
+    try:
+        if not _batch_response_was_successful(entry, custom_llm_provider):
+            return None
+        return _compute_output_line_stats(entry, custom_llm_provider, model_name, model_info)
+    except Exception as e:  # noqa: BLE001  # any single line's costing failure must not abort the whole batch
+        verbose_logger.warning(
+            "batch output line could not be costed, so it is billed at $0 and the rest of the batch "
+            "is still billed. custom_id=%s error=%s",
+            custom_id,
+            str(e),
+        )
+        return None
+
+
+def _compute_output_line_stats(
+    entry: Mapping[str, Any],
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"],
+    model_name: str | None,
+    model_info: ModelInfo | None,
+) -> _BatchOutputLineStats:
+    response_body: Final = _get_response_from_batch_job_output_file(entry, custom_llm_provider)
+    usage: Final = _get_batch_job_usage_from_response_body(response_body, custom_llm_provider)
+    prompt_details: Final = parse_prompt_tokens_details(usage)
+    raw_model: Final = response_body.get("model")
+    response_model: Final = raw_model if isinstance(raw_model, str) and raw_model else None
+    return _BatchOutputLineStats(
+        cost=_output_line_cost(
+            response_body=response_body,
+            usage=usage,
+            custom_llm_provider=custom_llm_provider,
+            model_name=model_name,
+            response_model=response_model,
+            model_info=model_info,
+        ),
+        prompt_tokens=usage.prompt_tokens,
+        completion_tokens=usage.completion_tokens,
+        total_tokens=usage.total_tokens,
+        cache_read_tokens=prompt_details["cache_hit_tokens"],
+        cache_creation_tokens=prompt_details["cache_creation_tokens"],
+        model=response_model,
+    )
+
+
+def _output_line_cost(
+    response_body: Mapping[str, Any],
+    usage: Usage,
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"],
+    model_name: str | None,
+    response_model: str | None,
+    model_info: ModelInfo | None,
+) -> float:
     from litellm.cost_calculator import batch_cost_calculator
 
-    for entry in entries:
-        if not _batch_response_was_successful(entry, custom_llm_provider):
-            continue
-        response_body = _get_response_from_batch_job_output_file(entry, custom_llm_provider)
-        usage = _get_batch_job_usage_from_response_body(response_body, custom_llm_provider)
-        prompt_details = _parse_prompt_tokens_details(usage)
-        raw_model = response_body.get("model")
-        response_model = raw_model if isinstance(raw_model, str) and raw_model else None
-        if model_info is not None or custom_llm_provider in ("anthropic", "bedrock"):
-            if custom_llm_provider == "bedrock" and model_name:
-                cost_model = model_name
-            else:
-                cost_model = response_model or model_name or ""
-            prompt_cost, completion_cost = batch_cost_calculator(
-                usage=usage,
-                model=cost_model,
-                custom_llm_provider=custom_llm_provider,
-                model_info=model_info,
-            )
-            line_cost = prompt_cost + completion_cost
-        else:
-            line_cost = litellm.completion_cost(
-                completion_response=response_body,
-                custom_llm_provider=custom_llm_provider,
-                call_type=CallTypes.aretrieve_batch.value,
-            )
-        yield _BatchOutputLineStats(
-            cost=line_cost,
-            prompt_tokens=usage.prompt_tokens,
-            completion_tokens=usage.completion_tokens,
-            total_tokens=usage.total_tokens,
-            cache_read_tokens=prompt_details["cache_hit_tokens"],
-            cache_creation_tokens=prompt_details["cache_creation_tokens"],
-            model=response_model,
+    if model_info is None and custom_llm_provider not in ("anthropic", "bedrock"):
+        return litellm.completion_cost(
+            completion_response=response_body,
+            custom_llm_provider=custom_llm_provider,
+            call_type=CallTypes.aretrieve_batch.value,
         )
+    cost_model: Final = (
+        model_name if custom_llm_provider == "bedrock" and model_name else response_model or model_name or ""
+    )
+    prompt_cost, completion_cost = batch_cost_calculator(
+        usage=usage,
+        model=cost_model,
+        custom_llm_provider=custom_llm_provider,
+        model_info=model_info,
+    )
+    return prompt_cost + completion_cost
 
 
 def _aggregate_batch_cost_usage_models(
@@ -295,7 +360,7 @@ def _extract_file_access_credentials(litellm_params: dict | None) -> dict:
 
     if litellm_params:
         # List of credential keys that should be passed to file operations
-        credential_keys: Final = [
+        credential_keys: Final = (
             "api_key",
             "api_base",
             "api_version",
@@ -309,7 +374,9 @@ def _extract_file_access_credentials(litellm_params: dict | None) -> dict:
             "bucket_name",
             "timeout",
             "max_retries",
-        ]
+            "_litellm_internal_model_credentials",
+            *AWS_CREDENTIAL_KWARGS_KEYS,
+        )
         for key in credential_keys:
             if key in litellm_params:
                 credentials[key] = litellm_params[key]
@@ -319,9 +386,10 @@ def _extract_file_access_credentials(litellm_params: dict | None) -> dict:
 
 def _get_file_content_as_dictionary(file_content: bytes) -> list[dict]:
     """
-    Get the file content as a list of dictionaries from JSON Lines format
+    Get the file content as a list of dictionaries from JSON Lines format,
+    skipping malformed lines
     """
-    return list(_iter_batch_input_entries(file_content))
+    return list(_iter_batch_output_entries(file_content))
 
 
 def _iter_batch_input_lines(file_content: bytes) -> Iterator[bytes]:
@@ -342,15 +410,29 @@ def _iter_batch_input_lines(file_content: bytes) -> Iterator[bytes]:
             yield line
 
 
-def _iter_batch_input_entries(file_content: bytes) -> Iterator[dict]:
+def _iter_batch_output_entries(file_content: bytes) -> Iterator[dict]:
     """
-    Yield parsed batch input JSONL entries one at a time without materializing the
-    whole file as a list, so peak memory stays bounded. Raises on a malformed line;
-    callers that must survive bad rows should iterate ``_iter_batch_input_lines``
-    and parse per-row instead.
+    Yield parsed batch output JSONL entries one at a time without materializing
+    the whole file as a list, so peak memory stays bounded. A malformed or
+    non-object line is skipped with a warning so one bad line never aborts the
+    whole batch's cost accounting.
     """
     for line in _iter_batch_input_lines(file_content):
-        yield json.loads(line)
+        entry = _parse_batch_output_line(line)
+        if entry is not None:
+            yield entry
+
+
+def _parse_batch_output_line(line: bytes) -> dict | None:
+    try:
+        parsed: Final = json.loads(line)
+    except ValueError as e:
+        verbose_logger.warning("skipping malformed batch output line: %s", str(e))
+        return None
+    if isinstance(parsed, dict):
+        return parsed
+    verbose_logger.warning("skipping non-object batch output line of type %s", type(parsed).__name__)
+    return None
 
 
 # A batch request's input tokens scale roughly with its serialized size, so this
@@ -421,17 +503,31 @@ def _count_prompt_or_input_tokens(model: str, value: Any) -> int:
     return 0
 
 
-def _get_batch_job_usage_from_response_body(response_body: dict, custom_llm_provider: str = "openai") -> Usage:
+def _get_batch_job_usage_from_response_body(
+    response_body: Mapping[str, Any], custom_llm_provider: str = "openai"
+) -> Usage:
     """
     Get the tokens of a batch job from the response body
     """
     if custom_llm_provider in ("anthropic", "bedrock"):
         from litellm.llms.anthropic.chat.transformation import AnthropicConfig
+        from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
 
-        return AnthropicConfig().calculate_usage(
-            usage_object=response_body.get("usage", None) or {},
+        usage_object: Final = response_body.get("usage", None) or {}
+        if custom_llm_provider == "bedrock" and AmazonConverseConfig.is_converse_usage_shape(usage_object):
+            return AmazonConverseConfig().usage_from_batch_output(usage_object)
+        anthropic_usage: Final = AnthropicConfig().calculate_usage(
+            usage_object=usage_object,
             reasoning_content=None,
         )
+        if usage_object and anthropic_usage.total_tokens == 0:
+            verbose_logger.warning(
+                "batch output line reported usage this parser does not understand, so it will be billed at $0. "
+                "provider=%s usage_keys=%s",
+                custom_llm_provider,
+                sorted(usage_object.keys()),
+            )
+        return anthropic_usage
     from litellm.responses.utils import ResponseAPILoggingUtils
 
     _usage_dict: Final = response_body.get("usage", None) or {}
@@ -441,7 +537,7 @@ def _get_batch_job_usage_from_response_body(response_body: dict, custom_llm_prov
     return usage
 
 
-def _get_anthropic_result_from_batch_results_line(batch_results_line: dict) -> dict:
+def _get_anthropic_result_from_batch_results_line(batch_results_line: Mapping[str, Any]) -> dict:
     """
     Get the ``result`` object from a line of an Anthropic message batch results JSONL file.
 
@@ -451,7 +547,9 @@ def _get_anthropic_result_from_batch_results_line(batch_results_line: dict) -> d
     return batch_results_line.get("result", None) or {}
 
 
-def _get_response_from_batch_job_output_file(batch_job_output_file: dict, custom_llm_provider: str = "openai") -> Any:
+def _get_response_from_batch_job_output_file(
+    batch_job_output_file: Mapping[str, Any], custom_llm_provider: str = "openai"
+) -> Any:
     """
     Get the response from the batch job output file
     """
@@ -464,7 +562,9 @@ def _get_response_from_batch_job_output_file(batch_job_output_file: dict, custom
     return _response_body
 
 
-def _batch_response_was_successful(batch_job_output_file: dict, custom_llm_provider: str = "openai") -> bool:
+def _batch_response_was_successful(
+    batch_job_output_file: Mapping[str, Any], custom_llm_provider: str = "openai"
+) -> bool:
     """
     Check if the batch job response was successful
 

litellm/batches/main.py

@@ -22,6 +22,7 @@ from openai.types.batch import BatchRequestCounts
 
 import litellm
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.get_litellm_params import add_trusted_model_credentials_to_litellm_params
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.anthropic.batches.handler import AnthropicBatchesHandler
 from litellm.llms.azure.batches.handler import AzureBatchesAPI
@@ -106,7 +107,7 @@ async def acreate_batch(
     completion_window: Literal["24h"],
     endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions", "/v1/responses"],
     input_file_id: str,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy"] = "openai",
     metadata: dict[str, str] | None = None,
     extra_headers: dict[str, str] | None = None,
     extra_body: dict[str, str] | None = None,
@@ -156,7 +157,7 @@ def create_batch(
     completion_window: Literal["24h"],
     endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions", "/v1/responses"],
     input_file_id: str,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy"] = "openai",
     metadata: dict[str, str] | None = None,
     extra_headers: dict[str, str] | None = None,
     extra_body: dict[str, str] | None = None,
@@ -338,7 +339,9 @@ def create_batch(
 @client
 async def aretrieve_batch(
     batch_id: str,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "anthropic"] = "openai",
+    custom_llm_provider: Literal[
+        "openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic"
+    ] = "openai",
     metadata: dict[str, str] | None = None,
     extra_headers: dict[str, str] | None = None,
     extra_body: dict[str, str] | None = None,
@@ -384,7 +387,9 @@ def _handle_retrieve_batch_providers_without_provider_config(
     litellm_params: dict,
     _retrieve_batch_request: RetrieveBatchRequest,
     _is_async: bool,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "anthropic"] = "openai",
+    custom_llm_provider: Literal[
+        "openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic"
+    ] = "openai",
     logging_obj: Any | None = None,
 ):
     api_base: str | None = None
@@ -507,7 +512,9 @@ def _handle_retrieve_batch_providers_without_provider_config(
 @client
 def retrieve_batch(
     batch_id: str,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "anthropic"] = "openai",
+    custom_llm_provider: Literal[
+        "openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic"
+    ] = "openai",
     metadata: dict[str, str] | None = None,
     extra_headers: dict[str, str] | None = None,
     extra_body: dict[str, str] | None = None,
@@ -527,6 +534,7 @@ def retrieve_batch(
             custom_llm_provider=custom_llm_provider,
             **kwargs,
         )
+        add_trusted_model_credentials_to_litellm_params(litellm_params, kwargs)
         if litellm_logging_obj is not None:
             litellm_logging_obj.update_from_kwargs(
                 kwargs=kwargs,
@@ -824,7 +832,7 @@ def list_batches(
 async def acancel_batch(
     batch_id: str,
     model: str | None = None,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "litellm_proxy"] = "openai",
     metadata: dict[str, str] | None = None,
     extra_headers: dict[str, str] | None = None,
     extra_body: dict[str, str] | None = None,
@@ -870,7 +878,7 @@ async def acancel_batch(
 def cancel_batch(
     batch_id: str,
     model: str | None = None,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai"] | str = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "litellm_proxy"] | str = "openai",
     metadata: dict[str, str] | None = None,
     extra_headers: dict[str, str] | None = None,
     extra_body: dict[str, str] | None = None,
@@ -991,9 +999,14 @@ def cancel_batch(
                 timeout=timeout,
                 max_retries=optional_params.max_retries,
             )
+        elif custom_llm_provider == "bedrock":
+            response = BedrockBatchesHandler.cancel_batch(
+                batch_id=batch_id,
+                **kwargs,
+            )
         else:
             raise litellm.exceptions.BadRequestError(
-                message=f"LiteLLM doesn't support {custom_llm_provider} for 'cancel_batch'. Only 'openai', 'azure', and 'vertex_ai' are supported.",
+                message=f"LiteLLM doesn't support {custom_llm_provider} for 'cancel_batch'. Only 'openai', 'azure', 'vertex_ai', and 'bedrock' are supported.",
                 model="n/a",
                 llm_provider=custom_llm_provider,
                 response=httpx.Response(

litellm/caching/_embedding_router.py

@@ -12,8 +12,11 @@ This module is dependency-injected: callers pass the proxy ``llm_router`` and
 
 from __future__ import annotations
 
+from collections.abc import Sequence
 from typing import TYPE_CHECKING, Any, Final
 
+import litellm
+
 if TYPE_CHECKING:
     from litellm.router import Router
 
@@ -41,3 +44,28 @@ def build_router_embedding_metadata(
     metadata: Final[dict[str, Any]] = dict(request_metadata or {})
     metadata["semantic-cache-embedding"] = True
     return metadata
+
+
+def resolve_embedding_max_input_tokens(
+    configured_max_input_tokens: int | None,
+    embedding_model: str,
+    router: Router | None,
+) -> int | None:
+    """Explicit cache setting first, else the Router deployment's configured ``max_input_tokens``."""
+    if configured_max_input_tokens is not None:
+        return configured_max_input_tokens
+    if router is None:
+        return None
+    deployment_max_input_tokens, _ = router.get_configured_token_limits(embedding_model)
+    return deployment_max_input_tokens
+
+
+def truncate_embedding_input(prompt: str, embedding_model: str, max_input_tokens: int | None) -> str:
+    """Keep only the first ``max_input_tokens`` tokens of ``prompt`` for the embedding call."""
+    if max_input_tokens is None:
+        return prompt
+    tokens: Final[Sequence[int]] = litellm.encode(model=embedding_model, text=prompt)
+    if len(tokens) <= max_input_tokens:
+        return prompt
+    truncated: Final[str] = litellm.decode(model=embedding_model, tokens=tokens[:max_input_tokens])
+    return truncated

litellm/caching/caching.py

@@ -66,20 +66,7 @@ class Cache:
         default_in_memory_ttl: float | None = None,
         default_in_redis_ttl: float | None = None,
         similarity_threshold: float | None = None,
-        supported_call_types: list[CachingSupportedCallTypes] | None = [
-            "completion",
-            "acompletion",
-            "embedding",
-            "aembedding",
-            "atranscription",
-            "transcription",
-            "atext_completion",
-            "text_completion",
-            "arerank",
-            "rerank",
-            "responses",
-            "aresponses",
-        ],
+        supported_call_types: list[CachingSupportedCallTypes] | None = list(DEFAULT_CACHING_SUPPORTED_CALL_TYPES),
         # s3 Bucket, boto3 configuration
         azure_account_url: str | None = None,
         azure_blob_container: str | None = None,
@@ -110,6 +97,7 @@ class Cache:
         qdrant_quantization_config: str | None = None,
         qdrant_semantic_cache_embedding_model: str = "text-embedding-ada-002",
         qdrant_semantic_cache_vector_size: int | None = None,
+        semantic_cache_embedding_max_input_tokens: int | None = None,
         # GCP IAM authentication parameters
         gcp_service_account: str | None = None,
         gcp_ssl_ca_certs: str | None = None,
@@ -135,6 +123,7 @@ class Cache:
             qdrant_api_key (str, optional): The api_key for the local or cloud qdrant cluster.
             qdrant_collection_name (str, optional): The name for your qdrant collection. Required if type is "qdrant-semantic".
             similarity_threshold (float, optional): The similarity threshold for semantic-caching, Required if type is "redis-semantic" or "qdrant-semantic".
+            semantic_cache_embedding_max_input_tokens (int, optional): Truncate prompts to this many tokens before embedding them for semantic caching. Defaults to the embedding deployment's configured max_input_tokens.
 
             # Disk Cache Args
             disk_cache_dir (str, optional): The directory for the disk cache. Defaults to None.
@@ -205,6 +194,7 @@ class Cache:
                 similarity_threshold=similarity_threshold,
                 embedding_model=redis_semantic_cache_embedding_model,
                 index_name=redis_semantic_cache_index_name,
+                embedding_max_input_tokens=semantic_cache_embedding_max_input_tokens,
                 **kwargs,
             )
         elif type == LiteLLMCacheType.VALKEY_SEMANTIC:
@@ -220,6 +210,7 @@ class Cache:
                 embedding_model=valkey_semantic_cache_embedding_model,
                 index_name=valkey_semantic_cache_index_name,
                 startup_nodes=redis_startup_nodes,
+                embedding_max_input_tokens=semantic_cache_embedding_max_input_tokens,
                 **kwargs,
             )
         elif type == LiteLLMCacheType.QDRANT_SEMANTIC:
@@ -231,6 +222,7 @@ class Cache:
                 quantization_config=qdrant_quantization_config,
                 embedding_model=qdrant_semantic_cache_embedding_model,
                 vector_size=qdrant_semantic_cache_vector_size,
+                embedding_max_input_tokens=semantic_cache_embedding_max_input_tokens,
             )
         elif type == LiteLLMCacheType.LOCAL:
             self.cache = InMemoryCache()
@@ -927,20 +919,7 @@ def enable_cache(
     host: str | None = None,
     port: str | None = None,
     password: str | None = None,
-    supported_call_types: list[CachingSupportedCallTypes] | None = [
-        "completion",
-        "acompletion",
-        "embedding",
-        "aembedding",
-        "atranscription",
-        "transcription",
-        "atext_completion",
-        "text_completion",
-        "arerank",
-        "rerank",
-        "responses",
-        "aresponses",
-    ],
+    supported_call_types: list[CachingSupportedCallTypes] | None = list(DEFAULT_CACHING_SUPPORTED_CALL_TYPES),
     **kwargs,
 ):
     """
@@ -987,20 +966,7 @@ def update_cache(
     host: str | None = None,
     port: str | None = None,
     password: str | None = None,
-    supported_call_types: list[CachingSupportedCallTypes] | None = [
-        "completion",
-        "acompletion",
-        "embedding",
-        "aembedding",
-        "atranscription",
-        "transcription",
-        "atext_completion",
-        "text_completion",
-        "arerank",
-        "rerank",
-        "responses",
-        "aresponses",
-    ],
+    supported_call_types: list[CachingSupportedCallTypes] | None = list(DEFAULT_CACHING_SUPPORTED_CALL_TYPES),
     **kwargs,
 ):
     """

litellm/caching/caching_handler.py

@@ -18,8 +18,8 @@ import asyncio
 import datetime
 import inspect
 import time
-from collections.abc import AsyncGenerator, Callable, Generator
-from typing import TYPE_CHECKING, Any, Final, Optional
+from collections.abc import AsyncGenerator, AsyncIterator, Callable, Generator, Mapping
+from typing import TYPE_CHECKING, Any, Final, Optional, TypeVar
 
 from pydantic import BaseModel
 
@@ -49,10 +49,15 @@ from litellm.types.utils import (
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+    from litellm.llms.anthropic.experimental_pass_through.messages.response_cache import (
+        AnthropicMessagesStreamCacheWriter,
+    )
     from litellm.types.utils import PromptTokensDetailsWrapper
 else:
     LiteLLMLoggingObj = Any
 
+_StreamResultT = TypeVar("_StreamResultT")
+
 
 from litellm.litellm_core_utils.core_helpers import (
     _get_parent_otel_span_from_kwargs,
@@ -101,23 +106,34 @@ def _is_chat_completion_cached_dict(cached_result: dict) -> bool:
     return "choices" in cached_result
 
 
-def _should_defer_streaming_cache_hit_callbacks(*, kwargs: dict[str, Any]) -> bool:
+def _should_defer_streaming_cache_hit_callbacks(*, kwargs: dict[str, object]) -> bool:
     """
     When stream=True, do not run success callbacks at cache-hit time.
 
     Cached chat/text completion replay uses CustomStreamWrapper; cached Responses
-    replay uses CachedResponsesAPIStreamingIterator. Both invoke logging success
+    replay uses CachedResponsesAPIStreamingIterator; cached Anthropic Messages
+    replay uses CachedAnthropicMessagesStreamIterator. All invoke logging success
     handlers when the stream finishes; firing them here too would double-count
     spend and callback records.
     """
     return kwargs.get("stream", False) is True
 
 
+def _prompt_tokens_details_as_mapping(details: "PromptTokensDetailsWrapper") -> Mapping[str, object]:
+    """Dump prompt token details to an opaque field mapping, tolerating non-pydantic stand-ins."""
+    return details.model_dump(exclude_none=True) if hasattr(details, "model_dump") else {}
+
+
+def _request_cache_key(request_kwargs: Mapping[str, Any]) -> str | None:
+    """Read the caller-supplied ``cache_key`` off the request kwargs."""
+    return request_kwargs.get("cache_key", None)
+
+
 class LLMCachingHandler:
     def __init__(
         self,
         original_function: Callable,
-        request_kwargs: dict[str, Any],
+        request_kwargs: dict[str, object],
         start_time: datetime.datetime,
     ):
         from litellm.caching import DualCache, RedisCache
@@ -144,7 +160,7 @@ class LLMCachingHandler:
         start_time: datetime.datetime,
         call_type: str,
         kwargs: dict[str, Any],
-        args: tuple[Any, ...] | None = None,
+        args: tuple[object, ...] | None = None,
     ) -> CachingHandlerResponse | None:
         """
         Internal method to get from the cache.
@@ -283,7 +299,7 @@ class LLMCachingHandler:
         start_time: datetime.datetime,
         call_type: str,
         kwargs: dict[str, Any],
-        args: tuple[Any, ...] | None = None,
+        args: tuple[object, ...] | None = None,
     ) -> CachingHandlerResponse:
         cached_result: Any | None = None
 
@@ -360,7 +376,7 @@ class LLMCachingHandler:
                     return CachingHandlerResponse(cached_result=cached_result)
         return CachingHandlerResponse(cached_result=cached_result)
 
-    def handle_kwargs_input_list_or_str(self, kwargs: dict[str, Any]) -> list[str]:
+    def handle_kwargs_input_list_or_str(self, kwargs: dict[str, object]) -> list[str]:
         """
         Handles the input of kwargs['input'] being a list or a string
         """
@@ -542,8 +558,8 @@ class LLMCachingHandler:
         if details2 is None:
             return details1
 
-        dict1: Final = details1.model_dump(exclude_none=True) if hasattr(details1, "model_dump") else {}
-        dict2: Final = details2.model_dump(exclude_none=True) if hasattr(details2, "model_dump") else {}
+        dict1: Final = _prompt_tokens_details_as_mapping(details1)
+        dict2: Final = _prompt_tokens_details_as_mapping(details2)
 
         merged: Final[dict] = {}
         for key in set(dict1.keys()) | set(dict2.keys()):
@@ -665,7 +681,9 @@ class LLMCachingHandler:
             cache_hit=cache_hit,
         )
 
-    async def _retrieve_from_cache(self, call_type: str, kwargs: dict[str, Any], args: tuple[Any, ...]) -> Any | None:
+    async def _retrieve_from_cache(
+        self, call_type: str, kwargs: dict[str, object], args: tuple[object, ...]
+    ) -> Any | None:
         """
         Internal method to
         - get cache key
@@ -721,7 +739,8 @@ class LLMCachingHandler:
                     cached_result = None
         else:
             request_kwargs: Final = new_kwargs.copy()
-            request_cache_key: Final = request_kwargs.pop("cache_key", None)
+            request_cache_key: Final = _request_cache_key(request_kwargs)
+            request_kwargs.pop("cache_key", None)
             if litellm.cache._supports_async() is True:
                 ## check if dual cache is supported ##
                 self.preset_cache_key = request_cache_key or litellm.cache.get_cache_key(**request_kwargs)
@@ -743,10 +762,10 @@ class LLMCachingHandler:
         self,
         cached_result: Any,
         call_type: str,
-        kwargs: dict[str, Any],
+        kwargs: dict[str, object],
         logging_obj: LiteLLMLoggingObj,
         model: str,
-        args: tuple[Any, ...],
+        args: tuple[object, ...],
         custom_llm_provider: str | None = None,
     ) -> (
         ModelResponse
@@ -835,6 +854,18 @@ class LLMCachingHandler:
                 response_type="audio_transcription",
                 hidden_params=hidden_params,
             )
+        elif (
+            call_type == CallTypes.anthropic_messages.value or call_type == CallTypes.aanthropic_messages.value
+        ) and isinstance(cached_result, dict):
+            from litellm.llms.anthropic.experimental_pass_through.messages.response_cache import (
+                convert_cached_anthropic_messages_result,
+            )
+
+            cached_result = convert_cached_anthropic_messages_result(
+                cached_result=cached_result,
+                logging_obj=logging_obj,
+                kwargs=kwargs,
+            )
         elif (call_type == "aresponses" or call_type == "responses") and isinstance(cached_result, dict):
             use_chat_completion_cache: Final = _is_chat_completion_cached_dict(cached_result)
             if use_chat_completion_cache:
@@ -930,7 +961,7 @@ class LLMCachingHandler:
         result: Any,
         original_function: Callable,
         kwargs: dict[str, Any],
-        args: tuple[Any, ...] | None = None,
+        args: tuple[object, ...] | None = None,
     ):
         """
         Internal method to check the type of the result & cache used and adds the result to the cache accordingly
@@ -995,8 +1026,8 @@ class LLMCachingHandler:
     def sync_set_cache(
         self,
         result: Any,
-        kwargs: dict[str, Any],
-        args: tuple[Any, ...] | None = None,
+        kwargs: dict[str, object],
+        args: tuple[object, ...] | None = None,
     ):
         """
         Sync internal method to add the result to the cache
@@ -1031,6 +1062,26 @@ class LLMCachingHandler:
             and (kwargs.get("cache", {}).get("no-store", False) is not True)
         )
 
+    def wrap_streaming_result_for_cache(
+        self, result: _StreamResultT, call_type: str
+    ) -> "_StreamResultT | AnthropicMessagesStreamCacheWriter":
+        if call_type not in (
+            CallTypes.anthropic_messages.value,
+            CallTypes.aanthropic_messages.value,
+        ):
+            return result
+        if litellm.cache is None or not self._should_store_result_in_cache(
+            original_function=self.original_function, kwargs=self.request_kwargs
+        ):
+            return result
+        if not isinstance(result, AsyncIterator):
+            return result
+        from litellm.llms.anthropic.experimental_pass_through.messages.response_cache import (
+            AnthropicMessagesStreamCacheWriter,
+        )
+
+        return AnthropicMessagesStreamCacheWriter(stream=result, caching_handler=self)
+
     def _is_call_type_supported_by_cache(
         self,
         original_function: Callable,
@@ -1166,8 +1217,8 @@ class LLMCachingHandler:
 
 def convert_args_to_kwargs(
     original_function: Callable,
-    args: tuple[Any, ...] | None = None,
-) -> dict[str, Any]:
+    args: tuple[object, ...] | None = None,
+) -> dict[str, object]:
     # Get the signature of the original function
     signature: Final = inspect.signature(original_function)
 

litellm/caching/qdrant_semantic_cache.py

@@ -12,7 +12,7 @@ import ast
 import asyncio
 import json
 import os
-from typing import Any, Final, cast
+from typing import TYPE_CHECKING, Any, Final, cast
 
 import litellm
 from litellm._logging import print_verbose
@@ -22,12 +22,21 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
 )
 from litellm.types.utils import EmbeddingResponse
 
-from ._embedding_router import build_router_embedding_metadata, resolve_embedding_router
+from ._embedding_router import (
+    build_router_embedding_metadata,
+    resolve_embedding_max_input_tokens,
+    resolve_embedding_router,
+    truncate_embedding_input,
+)
 from .base_cache import BaseCache
 
+if TYPE_CHECKING:
+    from litellm.router import Router
+
 
 class QdrantSemanticCache(BaseCache):
     CACHE_KEY_FIELD_NAME = "litellm_cache_key"
+    embedding_max_input_tokens: int | None = None
 
     def __init__(
         self,
@@ -39,6 +48,7 @@ class QdrantSemanticCache(BaseCache):
         embedding_model="text-embedding-ada-002",
         host_type=None,
         vector_size=None,
+        embedding_max_input_tokens: int | None = None,
     ):
         from litellm.llms.custom_httpx.http_handler import (
             _get_httpx_client,
@@ -57,6 +67,7 @@ class QdrantSemanticCache(BaseCache):
             raise Exception("similarity_threshold must be provided, passed None")
         self.similarity_threshold = similarity_threshold
         self.embedding_model = embedding_model
+        self.embedding_max_input_tokens = embedding_max_input_tokens
         self.vector_size = vector_size if vector_size is not None else QDRANT_VECTOR_SIZE
         headers = {}
 
@@ -188,6 +199,13 @@ class QdrantSemanticCache(BaseCache):
         cached_key: Final = payload.get(self.CACHE_KEY_FIELD_NAME)
         return cached_key is not None and str(cached_key) == str(key)
 
+    def _embedding_input(self, prompt: str, router: "Router | None") -> str:
+        return truncate_embedding_input(
+            prompt,
+            self.embedding_model,
+            resolve_embedding_max_input_tokens(self.embedding_max_input_tokens, self.embedding_model, router),
+        )
+
     def _get_embedding(self, prompt: str, metadata: dict[str, Any] | None = None) -> EmbeddingResponse:
         """Embed via the proxy Router when it serves the model, else direct."""
         try:
@@ -197,16 +215,17 @@ class QdrantSemanticCache(BaseCache):
             llm_router = None
 
         router: Final = resolve_embedding_router(self.embedding_model, llm_router, llm_model_list)
+        embedding_input: Final = self._embedding_input(prompt, router)
         if router is not None:
             return router.embedding(
                 model=self.embedding_model,
-                input=prompt,
+                input=embedding_input,
                 cache={"no-store": True, "no-cache": True},
                 metadata=build_router_embedding_metadata(metadata),
             )
         return litellm.embedding(
             model=self.embedding_model,
-            input=prompt,
+            input=embedding_input,
             cache={"no-store": True, "no-cache": True},
         )
 
@@ -218,17 +237,18 @@ class QdrantSemanticCache(BaseCache):
             llm_router = None
 
         router: Final = resolve_embedding_router(self.embedding_model, llm_router, llm_model_list)
+        embedding_input: Final = self._embedding_input(prompt, router)
         if router is not None:
             return await router.aembedding(
                 model=self.embedding_model,
-                input=prompt,
+                input=embedding_input,
                 cache={"no-store": True, "no-cache": True},
                 metadata=build_router_embedding_metadata(metadata),
             )
 
         return await litellm.aembedding(
             model=self.embedding_model,
-            input=prompt,
+            input=embedding_input,
             cache={"no-store": True, "no-cache": True},
         )
 

litellm/caching/redis_cache.py

@@ -49,7 +49,7 @@ if TYPE_CHECKING:
     cluster_pipeline = ClusterPipeline
     async_redis_client = Redis
     async_redis_cluster_client = RedisCluster
-    Span = _Span | Any
+    Span = _Span
 else:
     pipeline = Any
     cluster_pipeline = Any
@@ -625,7 +625,11 @@ class RedisCache(BaseCache):
             f"{self.namespace}-{hashlib.sha256(script.encode()).hexdigest()[:16]}"
         )
 
-        async def run_script(keys: Sequence[str], args: Sequence[Any], client: Any = None) -> Any:
+        async def run_script(
+            keys: Sequence[str],
+            args: Sequence[str | bytes | int | float],
+            client: object = None,
+        ) -> object:
             async def execute() -> object:
                 executor: Callable[..., Awaitable[Any]] | None = litellm.in_memory_llm_clients_cache.get_cache(
                     key=script_cache_key
@@ -650,7 +654,11 @@ class RedisCache(BaseCache):
         if hasattr(_redis_client, "register_script"):
             registered_script: Final = _redis_client.register_script(script)
 
-            async def standalone_executor(keys: Sequence[str], args: Sequence[Any], client: Any = None) -> Any:
+            async def standalone_executor(
+                keys: Sequence[str],
+                args: Sequence[str | bytes | int | float],
+                client: object = None,
+            ) -> object:
                 namespaced_keys: Final = tuple(self.check_and_fix_namespace(key=key) for key in keys)
                 return await registered_script(keys=namespaced_keys, args=args, client=client)
 
@@ -659,7 +667,11 @@ class RedisCache(BaseCache):
         if hasattr(_redis_client, "script_load"):
             script_sha: Final = _redis_client.script_load(script)
 
-            async def cluster_executor(keys: Sequence[str], args: Sequence[Any], client: Any = None) -> Any:
+            async def cluster_executor(
+                keys: Sequence[str],
+                args: Sequence[str | bytes | int | float],
+                client: object = None,
+            ) -> object:
                 namespaced_keys: Final = tuple(self.check_and_fix_namespace(key=key) for key in keys)
                 return await _redis_client.evalsha(script_sha, len(namespaced_keys), *namespaced_keys, *args)
 
@@ -757,7 +769,7 @@ class RedisCache(BaseCache):
     async def _pipeline_helper(
         self,
         pipe: pipeline | cluster_pipeline,
-        cache_list: list[tuple[Any, Any]],
+        cache_list: Sequence[tuple[str, object]],
         ttl: float | None,
     ) -> list:
         """
@@ -783,7 +795,9 @@ class RedisCache(BaseCache):
         return results
 
     @_redis_circuit_breaker_guard
-    async def async_set_cache_pipeline(self, cache_list: list[tuple[Any, Any]], ttl: float | None = None, **kwargs):
+    async def async_set_cache_pipeline(
+        self, cache_list: Sequence[tuple[str, object]], ttl: float | None = None, **kwargs
+    ):
         """
         Use Redis Pipelines for bulk write operations
         """
@@ -795,7 +809,7 @@ class RedisCache(BaseCache):
         start_time: Final = time.time()
 
         print_verbose(f"Set Async Redis Cache: key list: {cache_list}\nttl={ttl}, redis_version={self.redis_version}")
-        cache_value: Final[Any] = None
+        cache_value: Final = None
         try:
             async with _redis_client.pipeline(transaction=False) as pipe:
                 results: Final = await self._pipeline_helper(pipe, cache_list, ttl)
@@ -1074,7 +1088,7 @@ class RedisCache(BaseCache):
             # NON blocking - notify users Redis is throwing an exception
             verbose_logger.error("litellm.caching.caching: get() - Got exception from REDIS: ", e)
 
-    def _run_redis_mget_operation(self, keys: list[str]) -> list[Any]:
+    def _run_redis_mget_operation(self, keys: list[str]) -> Sequence[bytes | str | None]:
         """
         Wrapper to call `mget` on the redis client
 
@@ -1082,7 +1096,7 @@ class RedisCache(BaseCache):
         """
         return self.redis_client.mget(keys=keys)
 
-    async def _async_run_redis_mget_operation(self, keys: list[str]) -> list[Any]:
+    async def _async_run_redis_mget_operation(self, keys: list[str]) -> Sequence[bytes | str | None]:
         """
         Wrapper to call `mget` on the redis client
 
@@ -1115,7 +1129,7 @@ class RedisCache(BaseCache):
                 cache_key = self.check_and_fix_namespace(key=cache_key or "")
                 _keys.append(cache_key)
             start_time: Final = time.time()
-            results: Final[list] = self._run_redis_mget_operation(keys=_keys)
+            results: Final = self._run_redis_mget_operation(keys=_keys)
             end_time: Final = time.time()
             _duration: Final = end_time - start_time
             self.service_logger_obj.service_success_hook(
@@ -1522,7 +1536,7 @@ class RedisCache(BaseCache):
     async def async_rpush(
         self,
         key: str,
-        values: list[Any],
+        values: Sequence[str | bytes | int | float],
         parent_otel_span: Span | None = None,
         **kwargs,
     ) -> int:
@@ -1572,7 +1586,7 @@ class RedisCache(BaseCache):
     async def _pipeline_rpush_helper(
         self,
         pipe: pipeline,
-        rpush_list: list[RedisPipelineRpushOperation],
+        rpush_list: Sequence[RedisPipelineRpushOperation],
     ) -> list[int]:
         """Helper function for pipeline rpush operations"""
         for rpush_op in rpush_list:
@@ -1588,7 +1602,7 @@ class RedisCache(BaseCache):
     @_redis_circuit_breaker_guard
     async def async_rpush_pipeline(
         self,
-        rpush_list: list[RedisPipelineRpushOperation],
+        rpush_list: Sequence[RedisPipelineRpushOperation],
     ) -> list[int]:
         """
         Use Redis Pipelines for bulk RPUSH operations

litellm/caching/redis_semantic_cache.py

@@ -14,7 +14,7 @@ import asyncio
 import json
 import os
 from collections.abc import Callable, Mapping
-from typing import Any, Final, cast
+from typing import TYPE_CHECKING, Any, Final, cast
 
 import litellm
 from litellm._logging import print_verbose, verbose_logger
@@ -23,9 +23,17 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
 )
 from litellm.types.utils import EmbeddingResponse
 
-from ._embedding_router import build_router_embedding_metadata, resolve_embedding_router
+from ._embedding_router import (
+    build_router_embedding_metadata,
+    resolve_embedding_max_input_tokens,
+    resolve_embedding_router,
+    truncate_embedding_input,
+)
 from .base_cache import BaseCache
 
+if TYPE_CHECKING:
+    from litellm.router import Router
+
 
 class RedisSemanticCache(BaseCache):
     """
@@ -38,6 +46,7 @@ class RedisSemanticCache(BaseCache):
 
     DEFAULT_REDIS_INDEX_NAME: str = "litellm_semantic_cache_index"
     CACHE_KEY_FIELD_NAME: str = "litellm_cache_key"
+    embedding_max_input_tokens: int | None = None
 
     def __init__(
         self,
@@ -48,6 +57,7 @@ class RedisSemanticCache(BaseCache):
         similarity_threshold: float | None = None,
         embedding_model: str = "text-embedding-ada-002",
         index_name: str | None = None,
+        embedding_max_input_tokens: int | None = None,
         **kwargs: object,
     ):
         """
@@ -62,6 +72,8 @@ class RedisSemanticCache(BaseCache):
                 where 1.0 requires exact matches and 0.0 accepts any match
             embedding_model: Model to use for generating embeddings
             index_name: Name for the Redis index
+            embedding_max_input_tokens: Truncate prompts to this many tokens before
+                embedding; defaults to the Router deployment's configured max_input_tokens
             ttl: Default time-to-live for cache entries in seconds
             **kwargs: Additional arguments passed to the Redis client
 
@@ -86,6 +98,7 @@ class RedisSemanticCache(BaseCache):
         # While similarity: 1 = most similar, 0 = least similar
         self.distance_threshold = 1 - similarity_threshold
         self.embedding_model = embedding_model
+        self.embedding_max_input_tokens = embedding_max_input_tokens
 
         # Set up Redis connection
         if redis_url is None:
@@ -307,6 +320,13 @@ class RedisSemanticCache(BaseCache):
             return dict_method()
         return value
 
+    def _embedding_input(self, prompt: str, router: "Router | None") -> str:
+        return truncate_embedding_input(
+            prompt,
+            self.embedding_model,
+            resolve_embedding_max_input_tokens(self.embedding_max_input_tokens, self.embedding_model, router),
+        )
+
     def _get_embedding(self, prompt: str, metadata: dict[str, Any] | None = None) -> list[float]:
         """
         Routes through the proxy Router when the embedding model is a Router
@@ -320,12 +340,13 @@ class RedisSemanticCache(BaseCache):
             llm_router = None
 
         router: Final = resolve_embedding_router(self.embedding_model, llm_router, llm_model_list)
+        embedding_input: Final = self._embedding_input(prompt, router)
         if router is not None:
             embedding_response = cast(
                 EmbeddingResponse,
                 router.embedding(
                     model=self.embedding_model,
-                    input=prompt,
+                    input=embedding_input,
                     cache={"no-store": True, "no-cache": True},
                     metadata=build_router_embedding_metadata(metadata),
                 ),
@@ -335,7 +356,7 @@ class RedisSemanticCache(BaseCache):
                 EmbeddingResponse,
                 litellm.embedding(
                     model=self.embedding_model,
-                    input=prompt,
+                    input=embedding_input,
                     cache={"no-store": True, "no-cache": True},
                 ),
             )
@@ -490,18 +511,19 @@ class RedisSemanticCache(BaseCache):
             llm_router = None
 
         router: Final = resolve_embedding_router(self.embedding_model, llm_router, llm_model_list)
+        embedding_input: Final = self._embedding_input(prompt, router)
         try:
             if router is not None:
                 embedding_response = await router.aembedding(
                     model=self.embedding_model,
-                    input=prompt,
+                    input=embedding_input,
                     cache={"no-store": True, "no-cache": True},
                     metadata=build_router_embedding_metadata(metadata),
                 )
             else:
                 embedding_response = await litellm.aembedding(
                     model=self.embedding_model,
-                    input=prompt,
+                    input=embedding_input,
                     cache={"no-store": True, "no-cache": True},
                 )
             return embedding_response["data"][0]["embedding"]

litellm/caching/valkey_semantic_cache.py

@@ -17,7 +17,6 @@ RedisSemanticCache since those are backend agnostic.
 import asyncio
 import hashlib
 import os
-import struct
 from dataclasses import dataclass
 from typing import Any, Final
 
@@ -29,6 +28,7 @@ from redis.commands.search.query import Query
 
 from litellm._logging import print_verbose
 from litellm._uuid import uuid
+from litellm.llms.valkey.common_utils import build_valkey_url, pack_vector
 
 from .redis_semantic_cache import RedisSemanticCache
 
@@ -61,6 +61,7 @@ class ValkeySemanticCache(RedisSemanticCache):
         startup_nodes: list | None = None,
         sync_client: Redis | None = None,
         async_client: AsyncRedis | None = None,
+        embedding_max_input_tokens: int | None = None,
         **kwargs: Any,
     ):
         if similarity_threshold is None:
@@ -78,6 +79,7 @@ class ValkeySemanticCache(RedisSemanticCache):
 
         self.similarity_threshold = similarity_threshold
         self.embedding_model = embedding_model
+        self.embedding_max_input_tokens = embedding_max_input_tokens
         self.index_name = index_name or self.DEFAULT_VALKEY_INDEX_NAME
         self.key_prefix = f"{self.index_name}:"
         self._index_dim: int | None = None
@@ -92,19 +94,17 @@ class ValkeySemanticCache(RedisSemanticCache):
 
     @staticmethod
     def _build_valkey_url(host: str | None, port: str | None, password: str | None, ssl: bool = False) -> str:
-        host = host or os.environ.get("VALKEY_HOST") or os.environ.get("REDIS_HOST")
-        port = port or os.environ.get("VALKEY_PORT") or os.environ.get("REDIS_PORT")
-        password = password or os.environ.get("VALKEY_PASSWORD") or os.environ.get("REDIS_PASSWORD")
+        resolved_host: Final = host or os.environ.get("VALKEY_HOST") or os.environ.get("REDIS_HOST")
+        resolved_port: Final = port or os.environ.get("VALKEY_PORT") or os.environ.get("REDIS_PORT")
+        resolved_password: Final = password or os.environ.get("VALKEY_PASSWORD") or os.environ.get("REDIS_PASSWORD")
 
-        if not host or not port:
+        if not resolved_host or not resolved_port:
             raise ValueError(
                 "Missing required Valkey configuration. Provide host and port "
                 "(or VALKEY_HOST/VALKEY_PORT), or pass redis_url."
             )
 
-        credentials: Final = f":{password}@" if password else ""
-        scheme: Final = "rediss" if ssl else "redis"
-        return f"{scheme}://{credentials}{host}:{port}"
+        return build_valkey_url(host=resolved_host, port=resolved_port, password=resolved_password, ssl=ssl)
 
     @classmethod
     def _scope_tag(cls, key: str) -> str:
@@ -116,7 +116,7 @@ class ValkeySemanticCache(RedisSemanticCache):
 
     @staticmethod
     def _embedding_to_bytes(embedding: list[float]) -> bytes:
-        return struct.pack(f"<{len(embedding)}f", *embedding)
+        return pack_vector(embedding)
 
     def _index_schema(self, dim: int) -> tuple[TagField, VectorField]:
         return (

litellm/completion_extras/litellm_responses_transformation/handler.py

@@ -22,6 +22,7 @@ class ResponsesToCompletionBridgeHandlerInputKwargs(TypedDict):
     model_response: "ModelResponse"
     logging_obj: "LiteLLMLoggingObj"
     custom_llm_provider: str
+    encoding: object
 
 
 class ResponsesToCompletionBridgeHandler:
@@ -102,35 +103,37 @@ class ResponsesToCompletionBridgeHandler:
         from litellm import LiteLLMLoggingObj
         from litellm.types.utils import ModelResponse
 
-        model: Final = kwargs.get("model")
+        typed_kwargs: Final[dict[str, object]] = kwargs
+
+        model: Final = typed_kwargs.get("model")
         if model is None or not isinstance(model, str):
             raise ValueError("model is required")
 
-        custom_llm_provider: Final = kwargs.get("custom_llm_provider")
+        custom_llm_provider: Final = typed_kwargs.get("custom_llm_provider")
         if custom_llm_provider is None or not isinstance(custom_llm_provider, str):
             raise ValueError("custom_llm_provider is required")
 
-        messages: Final = kwargs.get("messages")
+        messages: Final = typed_kwargs.get("messages")
         if messages is None or not isinstance(messages, list):
             raise ValueError("messages is required")
 
-        optional_params: Final = kwargs.get("optional_params")
+        optional_params: Final = typed_kwargs.get("optional_params")
         if optional_params is None or not isinstance(optional_params, dict):
             raise ValueError("optional_params is required")
 
-        litellm_params: Final = kwargs.get("litellm_params")
+        litellm_params: Final = typed_kwargs.get("litellm_params")
         if litellm_params is None or not isinstance(litellm_params, dict):
             raise ValueError("litellm_params is required")
 
-        headers: Final = kwargs.get("headers")
+        headers: Final = typed_kwargs.get("headers")
         if headers is None or not isinstance(headers, dict):
             raise ValueError("headers is required")
 
-        model_response: Final = kwargs.get("model_response")
+        model_response: Final = typed_kwargs.get("model_response")
         if model_response is None or not isinstance(model_response, ModelResponse):
             raise ValueError("model_response is required")
 
-        logging_obj: Final = kwargs.get("logging_obj")
+        logging_obj: Final = typed_kwargs.get("logging_obj")
         if logging_obj is None or not isinstance(logging_obj, LiteLLMLoggingObj):
             raise ValueError("logging_obj is required")
 
@@ -143,6 +146,7 @@ class ResponsesToCompletionBridgeHandler:
             model_response=model_response,
             logging_obj=logging_obj,
             custom_llm_provider=custom_llm_provider,
+            encoding=typed_kwargs.get("encoding"),
         )
 
     def completion(
@@ -205,7 +209,7 @@ class ResponsesToCompletionBridgeHandler:
                 messages=messages,
                 optional_params=optional_params,
                 litellm_params=litellm_params,
-                encoding=kwargs.get("encoding"),
+                encoding=validated_kwargs["encoding"],
                 api_key=kwargs.get("api_key"),
                 json_mode=kwargs.get("json_mode"),
             )
@@ -230,7 +234,7 @@ class ResponsesToCompletionBridgeHandler:
                 messages=messages,
                 optional_params=optional_params,
                 litellm_params=litellm_params,
-                encoding=kwargs.get("encoding"),
+                encoding=validated_kwargs["encoding"],
                 api_key=kwargs.get("api_key"),
                 json_mode=kwargs.get("json_mode"),
             )
@@ -303,7 +307,7 @@ class ResponsesToCompletionBridgeHandler:
                 messages=messages,
                 optional_params=optional_params,
                 litellm_params=litellm_params,
-                encoding=kwargs.get("encoding"),
+                encoding=validated_kwargs["encoding"],
                 api_key=kwargs.get("api_key"),
                 json_mode=kwargs.get("json_mode"),
             )
@@ -328,7 +332,7 @@ class ResponsesToCompletionBridgeHandler:
                 messages=messages,
                 optional_params=optional_params,
                 litellm_params=litellm_params,
-                encoding=kwargs.get("encoding"),
+                encoding=validated_kwargs["encoding"],
                 api_key=kwargs.get("api_key"),
                 json_mode=kwargs.get("json_mode"),
             )

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -4,8 +4,8 @@ Handler for transforming /chat/completions api requests to litellm.responses req
 
 import json
 import os
-from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping
-from typing import TYPE_CHECKING, Any, Final, Literal, Union, cast
+from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
+from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, Union, cast
 
 from openai.types.responses.custom_tool_param import CustomToolParam
 from openai.types.responses.response_input_param import (
@@ -45,6 +45,9 @@ from litellm.types.utils import GenericStreamingChunk, ModelResponseStream
 
 if TYPE_CHECKING:
     from openai.types.responses import ResponseInputImageParam
+    from openai.types.responses.response_text_config_param import (
+        ResponseTextConfigParam as ResponseText,
+    )
     from pydantic import BaseModel
 
     from litellm import LiteLLMLoggingObj, ModelResponse
@@ -57,6 +60,19 @@ if TYPE_CHECKING:
         ChatCompletionThinkingBlock,
         OpenAIMessageContentListBlock,
     )
+    from litellm.types.utils import Choices
+
+
+class _ReasoningSummaryText(TypedDict):
+    type: str
+    text: str
+
+
+class _BuiltReasoningItem(TypedDict):
+    type: Literal["reasoning"]
+    id: str
+    encrypted_content: str | None
+    summary: Sequence[_ReasoningSummaryText]
 
 
 def _get_reasoning_items(
@@ -72,13 +88,13 @@ def _get_reasoning_items(
 def _build_reasoning_item(
     item_id: str,
     encrypted_content: str | None,
-    summary_raw: Any,
-) -> dict[str, Any]:
+    summary_raw: Iterable[object] | None,
+) -> _BuiltReasoningItem:
     """Build a ChatCompletionReasoningItem-shaped dict from raw response data.
 
     Handles both pydantic objects (attribute access) and plain dicts.
     """
-    summary: Final[list[dict[str, Any]]] = []
+    summary: Final[list[_ReasoningSummaryText]] = []
     for s in summary_raw or []:
         if isinstance(s, dict):
             summary.append({"type": s.get("type", "summary_text"), "text": s.get("text", "")})
@@ -98,7 +114,7 @@ def _build_reasoning_item(
 
 
 class _ChatToolCallDict(ChatCompletionToolCallChunk, total=False):
-    provider_specific_fields: Mapping[str, Any]
+    provider_specific_fields: Mapping[str, object]
 
 
 def _tool_call_dict_from_output_item(item: Mapping[str, Any], index: int) -> _ChatToolCallDict:
@@ -142,10 +158,10 @@ def _flat_responses_tool_choice(choice_type: str, name: str) -> ToolChoiceFuncti
 
 
 def _reasoning_item_to_response_input(
-    r_item: ChatCompletionReasoningItem | dict[str, Any],
-) -> dict[str, Any]:
+    r_item: ChatCompletionReasoningItem,
+) -> dict[str, object]:
     """Convert a stored ChatCompletionReasoningItem back to a Responses API input item."""
-    r_input: Final[dict[str, Any]] = {
+    r_input: Final[dict[str, object]] = {
         "type": "reasoning",
         "id": r_item.get("id") or f"rs_{id(r_item)}",
         # summary is always required by the Responses API, even when empty
@@ -169,6 +185,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         if not isinstance(tool_choice, dict):
             return tool_choice
         choice_type: Final = tool_choice.get("type")
+        if isinstance(choice_type, str) and choice_type in ("auto", "none", "required"):
+            return choice_type
         if choice_type not in ("function", "custom"):
             return tool_choice
         if isinstance(tool_choice.get("name"), str) and tool_choice.get("name"):
@@ -181,7 +199,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                 return _flat_responses_tool_choice(choice_type, nested_name)
         return tool_choice
 
-    def _handle_raw_dict_response_item(self, item: dict[str, Any], index: int) -> tuple[Any | None, int]:
+    def _handle_raw_dict_response_item(self, item: dict[str, Any], index: int) -> tuple["Choices | None", int]:
         """
         Handle raw dict response items from Responses API (e.g., GPT-5 Codex format).
 
@@ -228,8 +246,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
 
     def convert_chat_completion_messages_to_responses_api(
         self, messages: list["AllMessageValues"]
-    ) -> tuple[list[Any], str | None]:
-        input_items: Final[list[Any]] = []
+    ) -> tuple[list[object], str | None]:
+        input_items: Final[list[object]] = []
         instructions: str | None = None
         custom_tool_call_ids: Final = frozenset(
             tool_call["id"]
@@ -270,7 +288,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                 # Convert tool message to function call output format
                 # The Responses API expects 'output' to be a list with input_text/input_image types
                 # Using list format for consistency across text and multimodal content
-                tool_output: list[dict[str, Any]]
+                tool_output: list[dict[str, object]]
                 if content is None:
                     tool_output = []
                 elif isinstance(content, str):
@@ -308,7 +326,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                     function = tool_call.get("function")
                     custom = tool_call.get("custom")
                     if function:
-                        input_tool_call: dict[str, Any] = {
+                        input_tool_call: dict[str, object] = {
                             "type": "function_call",
                             "call_id": tool_call["id"],
                         }
@@ -376,15 +394,15 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             elif key == "web_search_options":
                 self._add_web_search_tool(responses_api_request, value)
 
-    def _build_sanitized_litellm_params(self, litellm_params: dict) -> dict[str, Any]:
+    def _build_sanitized_litellm_params(self, litellm_params: dict) -> dict[str, object]:
         """Build sanitized litellm_params with merged metadata."""
         responses_optional_param_keys: Final = set(ResponsesAPIOptionalRequestParams.__annotations__.keys())
-        sanitized: Final[dict[str, Any]] = {
+        sanitized: Final[dict[str, object]] = {
             key: value for key, value in litellm_params.items() if key not in responses_optional_param_keys
         }
         legacy_metadata: Final = litellm_params.get("metadata")
         existing_litellm_metadata: Final = litellm_params.get("litellm_metadata")
-        merged_litellm_metadata: Final[dict[str, Any]] = {}
+        merged_litellm_metadata: Final[dict[str, object]] = {}
         if isinstance(legacy_metadata, dict):
             merged_litellm_metadata.update(legacy_metadata)
         if isinstance(existing_litellm_metadata, dict):
@@ -424,7 +442,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         litellm_params: dict,
         headers: dict,
         litellm_logging_obj: "LiteLLMLoggingObj",
-        client: Any | None = None,
+        client: object | None = None,
     ) -> dict:
         (
             input_items,
@@ -498,9 +516,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
 
     @staticmethod
     def _convert_response_output_to_choices(
-        output_items: list[Any],
-        handle_raw_dict_callback: Callable | None = None,
-    ) -> list[Any]:
+        output_items: Sequence[object],
+        handle_raw_dict_callback: Callable[..., tuple["Choices | None", int]] | None = None,
+    ) -> list["Choices"]:
         """
         Convert Responses API output items to chat completion choices.
 
@@ -529,11 +547,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         choices: Final[list[Choices]] = []
         index = 0
         reasoning_content: str | None = None
-        pending_reasoning_item: dict[str, Any] | None = None
+        pending_reasoning_item: _BuiltReasoningItem | None = None
 
         # Collect all tool calls to put them in a single choice
         # (Chat Completions API expects all tool calls in one message)
-        accumulated_tool_calls: Final[list[dict[str, Any]]] = []
+        accumulated_tool_calls: Final[list[Mapping[str, object]]] = []
         tool_call_index = 0
 
         for item in output_items:
@@ -640,7 +658,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         return choices
 
     @classmethod
-    def _extract_output_from_completed_event(cls, parsed_chunk: dict[str, Any]) -> list[dict[str, Any]] | None:
+    def _extract_output_from_completed_event(cls, parsed_chunk: Mapping[str, object]) -> list[dict[str, object]] | None:
         response_payload: Final = parsed_chunk.get("response")
         if not isinstance(response_payload, dict):
             return None
@@ -650,12 +668,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         return cast(list[dict[str, Any]], response_output)
 
     @classmethod
-    def _recover_output_items_from_raw_sse(cls, raw_sse: str | None) -> list[dict[str, Any]]:
+    def _recover_output_items_from_raw_sse(cls, raw_sse: str | None) -> list[dict[str, object]]:
         if not raw_sse or not isinstance(raw_sse, str):
             return []
 
-        recovered_output_items: Final[dict[int, dict[str, Any]]] = {}
-        recovered_text_only_items: Final[dict[int, dict[str, Any]]] = {}
+        recovered_output_items: Final[dict[int, dict[str, object]]] = {}
+        recovered_text_only_items: Final[dict[int, dict[str, object]]] = {}
 
         for chunk in raw_sse.splitlines():
             parsed_chunk = parse_sse_json_chunk(chunk)
@@ -690,7 +708,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         # but text-only items at indices without a matching OUTPUT_ITEM_DONE
         # must still be preserved (e.g. multi-output responses where some
         # indices only emitted OUTPUT_TEXT_DONE).
-        merged_items: Final[dict[int, dict[str, Any]]] = {**recovered_text_only_items}
+        merged_items: Final[dict[int, dict[str, object]]] = {**recovered_text_only_items}
         merged_items.update(recovered_output_items)
 
         if merged_items:
@@ -699,7 +717,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         return []
 
     @classmethod
-    def _recover_output_items_from_logging(cls, logging_obj: "LiteLLMLoggingObj") -> list[dict[str, Any]]:
+    def _recover_output_items_from_logging(cls, logging_obj: "LiteLLMLoggingObj") -> list[dict[str, object]]:
         model_call_details: Final = getattr(logging_obj, "model_call_details", {}) or {}
         original_response: Final = model_call_details.get("original_response")
         return cls._recover_output_items_from_raw_sse(original_response)
@@ -714,7 +732,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         messages: list["AllMessageValues"],
         optional_params: dict,
         litellm_params: dict,
-        encoding: Any,
+        encoding: object,
         api_key: str | None = None,
         json_mode: bool | None = None,
     ) -> "ModelResponse":
@@ -788,7 +806,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
     ) -> BaseModelResponseIterator:
         return OpenAiResponsesToChatCompletionStreamIterator(streaming_response, sync_stream, json_mode)
 
-    def _convert_content_str_to_input_text(self, content: str, role: str) -> dict[str, Any]:
+    def _convert_content_str_to_input_text(self, content: str, role: str) -> dict[str, object]:
         if role == "user" or role == "system" or role == "tool":
             return {"type": "input_text", "text": content}
         else:
@@ -825,13 +843,13 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
     def _convert_content_to_responses_format(
         self,
         content: str
-        | list[Any]
+        | list[object]
         | Iterable[
             Union["OpenAIMessageContentListBlock", "ChatCompletionThinkingBlock", "ChatCompletionRedactedThinkingBlock"]
         ]
         | None,
         role: str,
-    ) -> list[dict[str, Any]]:
+    ) -> list[dict[str, object]]:
         """Convert chat completion content to responses API format"""
         from litellm.types.llms.openai import ChatCompletionImageObject
 
@@ -973,7 +991,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
 
         return optional_params
 
-    def _map_reasoning_effort(self, reasoning_effort: str | dict[str, Any]) -> Reasoning | None:
+    def _map_reasoning_effort(self, reasoning_effort: str | Reasoning) -> Reasoning | None:
         # If dict is passed, convert it directly to Reasoning object
         if isinstance(reasoning_effort, dict):
             return Reasoning(**reasoning_effort)
@@ -1006,7 +1024,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
     def _add_web_search_tool(
         self,
         responses_api_request: ResponsesAPIOptionalRequestParams,
-        web_search_options: Any,
+        web_search_options: object,
     ) -> None:
         """
         Add web search tool to responses API request.
@@ -1024,14 +1042,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             tools = []
             responses_api_request["tools"] = tools
 
-        web_search_tool: Final[dict[str, Any]] = {"type": "web_search"}
+        web_search_tool: Final[dict[str, object]] = {"type": "web_search"}
         if isinstance(web_search_options, dict):
             web_search_tool.update(web_search_options)
 
         # Cast to Any to match the expected union type for tools list items
         tools.append(cast(Any, web_search_tool))
 
-    def _transform_response_format_to_text_format(self, response_format: dict[str, Any] | Any) -> dict[str, Any] | None:
+    def _transform_response_format_to_text_format(self, response_format: object) -> "ResponseText | None":
         """
         Transform Chat Completion response_format parameter to Responses API text.format parameter.
 
@@ -1130,7 +1148,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
 
 
 class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
-    def __init__(self, streaming_response, sync_stream: bool, json_mode: bool | None = False):
+    def __init__(
+        self,
+        streaming_response: Union[Iterator[str], AsyncIterator[str], "ModelResponse", "BaseModel"],
+        sync_stream: bool,
+        json_mode: bool | None = False,
+    ):
         super().__init__(streaming_response, sync_stream, json_mode)
         self._chat_completion_id: str | None = None
         self._tool_call_index_map: dict[int, int] = {}  # mutable-ok: per-stream accumulator state
@@ -1387,7 +1410,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
             finish_reason: Final = "tool_calls" if has_function_calls else "stop"
 
             # Extract reasoning items with encrypted_content for round-tripping
-            completed_reasoning_items: list[dict[str, Any]] | None = None
+            completed_reasoning_items: list[_BuiltReasoningItem] | None = None
             for item in output_items:
                 if not isinstance(item, dict) or item.get("type") != "reasoning":
                     continue
@@ -1439,7 +1462,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
             ]
         )
 
-    def chunk_parser(self, chunk: dict) -> "ModelResponseStream":
+    def chunk_parser(self, chunk: dict[str, object]) -> "ModelResponseStream":
         """
         Parse a Responses API streaming chunk and convert to OpenAI format.
 

litellm/constants.py

@@ -1,5 +1,6 @@
 import os
 import sys
+from types import MappingProxyType
 from typing import Final, Literal
 
 from litellm.litellm_core_utils.env_utils import get_env_int, get_env_int_or_none
@@ -141,6 +142,8 @@ LITELLM_UI_ALLOW_HEADERS: Final = [
     "x-litellm-semantic-filter",
     "x-litellm-semantic-filter-tools",
     "x-litellm-adaptive-router-model",
+    "x-litellm-applied-guardrails",
+    "x-litellm-guardrail-scan-id",
 ]
 
 # Gemini model-specific minimal thinking budget constants
@@ -472,6 +475,8 @@ EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE: Final = float(
 ### ANTHROPIC CONSTANTS ###
 ANTHROPIC_TOKEN_COUNTING_BETA_VERSION = os.getenv("ANTHROPIC_TOKEN_COUNTING_BETA_VERSION", "token-counting-2024-11-01")
 ANTHROPIC_SKILLS_API_BETA_VERSION: Final = "skills-2025-10-02"
+ANTHROPIC_BATCHES_ROUTE: Final = "/v1/messages/batches"
+VERTEX_BATCH_PREDICTION_JOBS_ROUTE: Final = "batchPredictionJobs"
 ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES: Final = {
     "low": 1,
     "medium": 5,
@@ -1323,6 +1328,7 @@ LITELLM_METADATA_FIELD: Final = "litellm_metadata"
 OLD_LITELLM_METADATA_FIELD: Final = "metadata"
 RETURN_RAW_MODEL_NAME_METADATA_KEY: Final = "_complexity_router_return_raw_model_name"
 SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: Final = "_session_deployment_affinity_ttl"
+CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags"
 INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
 LITELLM_TRUNCATED_PAYLOAD_FIELD: Final = "litellm_truncated"
 LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE: Final = (
@@ -1478,17 +1484,26 @@ CLOUDZERO_MAX_FETCHED_DATA_RECORDS: Final = int(os.getenv("CLOUDZERO_MAX_FETCHED
 SPEND_LOG_CLEANUP_JOB_NAME: Final = "spend_log_cleanup"
 KEY_ROTATION_JOB_NAME: Final = "litellm_key_rotation_job"
 EXPIRED_UI_SESSION_KEY_CLEANUP_JOB_NAME: Final = "litellm_expired_ui_session_key_cleanup_job"
+WEEKLY_SPEND_REPORT_JOB_ID: Final = "weekly_spend_report_job"
+MONTHLY_SPEND_REPORT_JOB_ID: Final = "monthly_spend_report_job"
+PROMETHEUS_FALLBACK_STATS_JOB_ID: Final = "prometheus_fallback_stats_job"
+SLACK_DAILY_REPORT_LOCK_ID: Final = "slack_daily_report"
+SLACK_MODEL_DEPRECATION_LOCK_ID: Final = "slack_model_deprecation_warning"
 SPEND_LOG_RUN_LOOPS: Final = int(os.getenv("SPEND_LOG_RUN_LOOPS", 500))
 SPEND_LOG_CLEANUP_BATCH_SIZE: Final = int(os.getenv("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000))
 SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES = int(os.getenv("SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3))
 SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS: Final = float(
     os.getenv("SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.5)
 )
+SPEND_LOG_CLEANUP_RUN_BUDGET_SECONDS: Final = float(os.getenv("SPEND_LOG_CLEANUP_RUN_BUDGET_SECONDS", "300"))
+SPEND_LOG_CLEANUP_BATCH_TIMEOUT_SECONDS: Final = float(os.getenv("SPEND_LOG_CLEANUP_BATCH_TIMEOUT_SECONDS", "30"))
+SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP: Final = int(os.getenv("SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP", "100000"))
 TOOL_SPEND_TOP_TOOLS: Final = 100
 SPEND_LOG_PARTITION_INTERVAL: Final = os.getenv("SPEND_LOG_PARTITION_INTERVAL", "day")
 SPEND_LOG_PARTITION_PRECREATE_AHEAD: Final = int(os.getenv("SPEND_LOG_PARTITION_PRECREATE_AHEAD", 7))
 SPEND_LOG_WRITE_BATCH_MAX_BYTES: Final = max(1, int(os.getenv("SPEND_LOG_WRITE_BATCH_MAX_BYTES", 2_000_000)))
 SPEND_LOG_QUEUE_SIZE_THRESHOLD: Final = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100))
+SPEND_LOG_QUEUE_MAX_BYTES: Final = max(1, int(os.getenv("SPEND_LOG_QUEUE_MAX_BYTES", "64000000")))
 SPEND_LOG_QUEUE_POLL_INTERVAL: Final = float(os.getenv("SPEND_LOG_QUEUE_POLL_INTERVAL", 2.0))
 SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE: Final = int(os.getenv("SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE", 10000))
 DEFAULT_CRON_JOB_LOCK_TTL_SECONDS: Final = int(os.getenv("DEFAULT_CRON_JOB_LOCK_TTL_SECONDS", 60))  # 1 minute
@@ -1523,6 +1538,10 @@ APSCHEDULER_REPLACE_EXISTING: Final = os.getenv("APSCHEDULER_REPLACE_EXISTING",
     "1",
 ]  # always replace existing jobs
 
+# Width of the window scheduled background jobs are spread across, so they do not all fire
+# on one instant on every replica. Tunable per deployment via general_settings.
+DEFAULT_STAGGER_WINDOW_SECONDS: Final = 300
+
 # The number of tag entries are higher than number of user, team entries. This leads to a higher QPS.
 # This will run tag spcific tasks at a later time to smooth QPS
 DAILY_TAG_SPEND_BATCH_MULTIPLIER: Final = 2.3
@@ -1576,6 +1595,13 @@ DEFAULT_MCP_ACCESS_GROUP_NEGATIVE_CACHE_TTL: Final = 10
 # in a single ``/{name1,name2,...}/mcp`` URL. Bounds the per-request DB / cache
 # fan-out an authenticated caller can trigger by stuffing the path with tokens.
 DEFAULT_MCP_NAMESPACE_CSV_MAX_TOKENS: Final = 16
+# Ceilings on the cached auth registries; larger tables fall back to per-row lookups
+# instead of holding an unbounded id set in every worker.
+TAG_REGISTRY_MAX_SIZE: Final = 5000
+END_USER_RESTRICTED_REGISTRY_MAX_SIZE: Final = 5000
+# How long a failed registry load is remembered as "unusable", so a degraded Postgres
+# is not re-scanned on every request on top of the per-id lookups it falls back to.
+REGISTRY_ERROR_NEGATIVE_CACHE_TTL: Final = 30
 
 # Sentry Scrubbing Configuration
 SENTRY_DENYLIST: Final = [
@@ -1731,8 +1757,26 @@ PTU_ROLLUP_LOCK_TTL_SECONDS: Final[int] = 900
 # Furthest back the catch-up pass looks for unpriced PTU days when a deployment
 # declares no ptu_effective_from, bounding the scan for an open-ended window.
 PTU_ROLLUP_MAX_BACKFILL_DAYS: Final[int] = 90
+# Deployments named in the lapsed-window alert before it is truncated, so a fleet-wide
+# expiry cannot produce an alert too large for the channel delivering it.
+PTU_LAPSED_ALERT_LIMIT: Final[int] = 10
 # Slack allowed when deciding a sentinel row is stale. The row's updated_at and the
 # run's cutoff are stamped by different hosts, so clock skew between them must not let
 # one run delete a charge another just wrote. A stale row is hours old and a concurrent
 # one is seconds old, so a few minutes separates them.
 PTU_PRUNE_SKEW_GRACE_SECONDS: Final[int] = 300
+
+# How long enqueued-token reservations for batches live without a refund. Providers
+# complete or expire batches within their completion window (24h for OpenAI), so a
+# reservation still unrefunded after 8 days belongs to a batch whose terminal state
+# was never observed (e.g. proxy restart); expiry returns the tokens to the caller.
+BATCH_ENQUEUED_TOKEN_TTL_SECONDS: Final[int] = 8 * 24 * 60 * 60
+
+# Key/team metadata field that opts batches into enqueued-token limiting. Only proxy
+# admins may write it: when present it replaces the standard RPM/TPM checks for
+# batch submissions.
+BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY: Final = "batch_enqueued_token_limit"
+
+# Shared read-only empty mapping, for defaulting optional Mapping parameters without
+# constructing a fresh mutable dict at each call site.
+EMPTY_MAPPING: Final = MappingProxyType({})

litellm/containers/main.py

@@ -1,9 +1,11 @@
 import asyncio
 import contextvars
 import json
-from collections.abc import Coroutine
+from collections.abc import Coroutine, Mapping
 from functools import partial
-from typing import Any, Final, Literal, overload
+from typing import Final, Literal, overload
+
+import httpx
 
 import litellm
 from litellm.constants import request_timeout as DEFAULT_REQUEST_TIMEOUT
@@ -48,16 +50,16 @@ __all__ = [
 @client
 async def acreate_container(
     name: str,
-    expires_after: dict[str, Any] | None = None,
+    expires_after: Mapping[str, object] | None = None,
     file_ids: list[str] | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     # LiteLLM specific params,
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
     # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
     # The extra values given here take precedence over values defined on the client or passed to this method.
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
 ) -> ContainerObject:
     """Asynchronously calls the `create_container` function with the given arguments and keyword arguments.
@@ -120,9 +122,9 @@ async def acreate_container(
 @overload
 def create_container(
     name: str,
-    expires_after: dict[str, Any] | None = None,
+    expires_after: Mapping[str, object] | None = None,
     file_ids: list[str] | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -130,16 +132,16 @@ def create_container(
     *,
     acreate_container: Literal[True],
     **kwargs,
-) -> Coroutine[Any, Any, ContainerObject]:
+) -> Coroutine[object, object, ContainerObject]:
     ...
 
 
 @overload
 def create_container(
     name: str,
-    expires_after: dict[str, Any] | None = None,
+    expires_after: Mapping[str, object] | None = None,
     file_ids: list[str] | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -156,20 +158,20 @@ def create_container(
 @client
 def create_container(
     name: str,
-    expires_after: dict[str, Any] | None = None,
+    expires_after: Mapping[str, object] | None = None,
     file_ids: list[str] | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
     # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
     # The extra values given here take precedence over values defined on the client or passed to this method.
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
-) -> ContainerObject | Coroutine[Any, Any, ContainerObject]:
+) -> ContainerObject | Coroutine[object, object, ContainerObject]:
     """Create a container using the OpenAI Container API.
 
     Currently supports OpenAI
@@ -281,13 +283,13 @@ async def alist_containers(
     after: str | None = None,
     limit: int | None = None,
     order: str | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
     # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
     # The extra values given here take precedence over values defined on the client or passed to this method.
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
 ) -> ContainerListResponse:
     """Asynchronously list containers.
@@ -351,7 +353,7 @@ def list_containers(
     after: str | None = None,
     limit: int | None = None,
     order: str | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -359,7 +361,7 @@ def list_containers(
     *,
     alist_containers: Literal[True],
     **kwargs,
-) -> Coroutine[Any, Any, ContainerListResponse]:
+) -> Coroutine[object, object, ContainerListResponse]:
     ...
 
 
@@ -368,7 +370,7 @@ def list_containers(
     after: str | None = None,
     limit: int | None = None,
     order: str | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -387,18 +389,18 @@ def list_containers(
     after: str | None = None,
     limit: int | None = None,
     order: str | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
     # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
     # The extra values given here take precedence over values defined on the client or passed to this method.
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
-) -> ContainerListResponse | Coroutine[Any, Any, ContainerListResponse]:
+) -> ContainerListResponse | Coroutine[object, object, ContainerListResponse]:
     """List containers using the OpenAI Container API.
 
     Currently supports OpenAI
@@ -481,13 +483,13 @@ def list_containers(
 @client
 async def aretrieve_container(
     container_id: str,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
     # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
     # The extra values given here take precedence over values defined on the client or passed to this method.
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
 ) -> ContainerObject:
     """Asynchronously retrieve a container.
@@ -545,7 +547,7 @@ async def aretrieve_container(
 @overload
 def retrieve_container(
     container_id: str,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -553,14 +555,14 @@ def retrieve_container(
     *,
     aretrieve_container: Literal[True],
     **kwargs,
-) -> Coroutine[Any, Any, ContainerObject]:
+) -> Coroutine[object, object, ContainerObject]:
     ...
 
 
 @overload
 def retrieve_container(
     container_id: str,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -577,18 +579,18 @@ def retrieve_container(
 @client
 def retrieve_container(
     container_id: str,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
     # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
     # The extra values given here take precedence over values defined on the client or passed to this method.
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
-) -> ContainerObject | Coroutine[Any, Any, ContainerObject]:
+) -> ContainerObject | Coroutine[object, object, ContainerObject]:
     """Retrieve a container using the OpenAI Container API.
 
     Currently supports OpenAI
@@ -696,13 +698,13 @@ def retrieve_container(
 @client
 async def adelete_container(
     container_id: str,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
     # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
     # The extra values given here take precedence over values defined on the client or passed to this method.
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
 ) -> DeleteContainerResult:
     """Asynchronously delete a container.
@@ -760,7 +762,7 @@ async def adelete_container(
 @overload
 def delete_container(
     container_id: str,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -768,14 +770,14 @@ def delete_container(
     *,
     adelete_container: Literal[True],
     **kwargs,
-) -> Coroutine[Any, Any, DeleteContainerResult]:
+) -> Coroutine[object, object, DeleteContainerResult]:
     ...
 
 
 @overload
 def delete_container(
     container_id: str,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -792,18 +794,18 @@ def delete_container(
 @client
 def delete_container(
     container_id: str,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
     # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
     # The extra values given here take precedence over values defined on the client or passed to this method.
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
-) -> DeleteContainerResult | Coroutine[Any, Any, DeleteContainerResult]:
+) -> DeleteContainerResult | Coroutine[object, object, DeleteContainerResult]:
     """Delete a container using the OpenAI Container API.
 
     Currently supports OpenAI
@@ -914,11 +916,11 @@ async def alist_container_files(
     after: str | None = None,
     limit: int | None = None,
     order: str | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
 ) -> ContainerFileListResponse:
     """Asynchronously list files in a container.
@@ -985,7 +987,7 @@ def list_container_files(
     after: str | None = None,
     limit: int | None = None,
     order: str | None = None,
-    timeout=600,
+    timeout: float | httpx.Timeout = 600,
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -993,7 +995,7 @@ def list_container_files(
     *,
     alist_container_files: Literal[True],
     **kwargs,
-) -> Coroutine[Any, Any, ContainerFileListResponse]:
+) -> Coroutine[object, object, ContainerFileListResponse]:
     ...
 
 
@@ -1003,7 +1005,7 @@ def list_container_files(
     after: str | None = None,
     limit: int | None = None,
     order: str | None = None,
-    timeout=600,
+    timeout: float | httpx.Timeout = 600,
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -1023,16 +1025,16 @@ def list_container_files(
     after: str | None = None,
     limit: int | None = None,
     order: str | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
-) -> ContainerFileListResponse | Coroutine[Any, Any, ContainerFileListResponse]:
+) -> ContainerFileListResponse | Coroutine[object, object, ContainerFileListResponse]:
     """List files in a container using the OpenAI Container API.
 
     Currently supports OpenAI
@@ -1125,11 +1127,11 @@ def list_container_files(
 async def aupload_container_file(
     container_id: str,
     file: FileTypes,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
 ) -> ContainerFileObject:
     """Asynchronously upload a file to a container.
@@ -1211,7 +1213,7 @@ async def aupload_container_file(
 def upload_container_file(
     container_id: str,
     file: FileTypes,
-    timeout=600,
+    timeout: float | httpx.Timeout = 600,
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -1219,7 +1221,7 @@ def upload_container_file(
     *,
     aupload_container_file: Literal[True],
     **kwargs,
-) -> Coroutine[Any, Any, ContainerFileObject]:
+) -> Coroutine[object, object, ContainerFileObject]:
     ...
 
 
@@ -1227,7 +1229,7 @@ def upload_container_file(
 def upload_container_file(
     container_id: str,
     file: FileTypes,
-    timeout=600,
+    timeout: float | httpx.Timeout = 600,
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -1245,16 +1247,16 @@ def upload_container_file(
 def upload_container_file(
     container_id: str,
     file: FileTypes,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
-) -> ContainerFileObject | Coroutine[Any, Any, ContainerFileObject]:
+) -> ContainerFileObject | Coroutine[object, object, ContainerFileObject]:
     """Upload a file to a container using the OpenAI Container API.
 
     This endpoint allows uploading files directly to a container session,

litellm/cost_calculator.py

@@ -26,11 +26,11 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
     _generic_cost_per_character,
     _get_regional_uplift_multiplier,
     _get_service_tier_cost_key,
-    _parse_prompt_tokens_details,
     calculate_cost_component,
     generic_cost_per_token,
     get_billable_input_tokens,
     get_token_type_cost_breakdown,
+    parse_prompt_tokens_details,
     select_cost_metric_for_model,
 )
 from litellm.llms.anthropic.cost_calculation import (
@@ -102,6 +102,7 @@ from litellm.types.utils import (
     LlmProviders,
     LlmProvidersSet,
     ModelInfo,
+    PromptTokensDetailsWrapper,
     ServiceTier,
     StandardBuiltInToolsParams,
     TranscriptionUsageDurationObject,
@@ -286,7 +287,7 @@ def _transcription_usage_has_token_details(
 
     prompt_tokens_val: Final = getattr(usage_block, "prompt_tokens", 0) or 0
     completion_tokens_val: Final = getattr(usage_block, "completion_tokens", 0) or 0
-    prompt_details: Final = getattr(usage_block, "prompt_tokens_details", None)
+    prompt_details: Final[PromptTokensDetailsWrapper | None] = getattr(usage_block, "prompt_tokens_details", None)
 
     if prompt_details is not None:
         audio_token_count: Final = getattr(prompt_details, "audio_tokens", 0) or 0
@@ -326,6 +327,8 @@ def cost_per_token(
     service_tier: str | None = None,  # for OpenAI service tier pricing
     ### DATA RESIDENCY ###
     data_residency: str | None = None,  # for OpenAI regional-processing uplift (e.g. "eu", "us")
+    ### VERTEX LOCATION ###
+    vertex_location: str | None = None,  # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global")
     response: Any | None = None,
     ### REQUEST MODEL ###
     request_model: str | None = None,  # original request model for router detection
@@ -375,7 +378,7 @@ def cost_per_token(
     _is_anthropic_style = False
 
     if usage_object is not None:
-        _pt_details: Final = getattr(usage_object, "prompt_tokens_details", None)
+        _pt_details: Final[PromptTokensDetailsWrapper | None] = getattr(usage_object, "prompt_tokens_details", None)
         if _pt_details is not None:
             _cache_read_tokens = float(getattr(_pt_details, "cached_tokens", 0) or 0)
             # OpenAI-compatible providers report cache-write tokens under
@@ -385,8 +388,8 @@ def cost_per_token(
                 getattr(_pt_details, "cache_write_tokens", 0) or getattr(_pt_details, "cache_creation_tokens", 0) or 0
             )
 
-        _anthropic_read: Final = getattr(usage_object, "cache_read_input_tokens", None)
-        _anthropic_create: Final = getattr(usage_object, "cache_creation_input_tokens", None)
+        _anthropic_read: Final[int | None] = getattr(usage_object, "cache_read_input_tokens", None)
+        _anthropic_create: Final[int | None] = getattr(usage_object, "cache_creation_input_tokens", None)
         if _anthropic_read is not None or _anthropic_create is not None:
             _is_anthropic_style = True
             if _anthropic_read is not None:
@@ -586,6 +589,7 @@ def cost_per_token(
                 prompt_characters=prompt_characters,
                 completion_characters=completion_characters,
                 usage=usage_block,
+                vertex_location=vertex_location,
             )
         elif cost_router == "cost_per_token":
             return google_cost_per_token(
@@ -593,6 +597,7 @@ def cost_per_token(
                 custom_llm_provider=custom_llm_provider,
                 usage=usage_block,
                 service_tier=service_tier,
+                vertex_location=vertex_location,
             )
     elif custom_llm_provider == "anthropic":
         return anthropic_cost_per_token(model=model, usage=usage_block, service_tier=service_tier)
@@ -645,7 +650,11 @@ def cost_per_token(
     else:
         model_info: Final = _cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
 
-        if (model_info.get("input_cost_per_token") or 0.0) > 0 or (model_info.get("output_cost_per_token") or 0.0) > 0:
+        if (
+            (model_info.get("input_cost_per_token") or 0.0) > 0
+            or (model_info.get("output_cost_per_token") or 0.0) > 0
+            or model_info.get("tiered_pricing") is not None
+        ):
             return generic_cost_per_token(
                 model=model,
                 usage=usage_block,
@@ -699,7 +708,7 @@ def get_replicate_completion_pricing(completion_response: dict, total_time=0.0):
     return a100_80gb_price_per_second_public * total_time / 1000
 
 
-def has_hidden_params(obj: Any) -> bool:
+def has_hidden_params(obj: object) -> bool:
     return hasattr(obj, "_hidden_params")
 
 
@@ -724,7 +733,7 @@ def _get_provider_for_cost_calc(
 
 def _select_model_name_for_cost_calc(
     model: str | None,
-    completion_response: Any | None,
+    completion_response: object | None,
     base_model: str | None = None,
     custom_pricing: bool | None = None,
     custom_llm_provider: str | None = None,
@@ -800,7 +809,7 @@ def _model_contains_known_llm_provider(model: str) -> bool:
     return _provider_prefix in LlmProvidersSet
 
 
-def _get_response_model(completion_response: Any) -> str | None:
+def _get_response_model(completion_response: object) -> str | None:
     """
     Extract the model name from a completion response object.
 
@@ -862,8 +871,18 @@ def _normalize_service_tier(service_tier: object) -> str | None:
     return service_tier
 
 
+def _extract_service_tier(source: object) -> str | None:
+    """Read a raw ``service_tier`` off a response body or usage object, dict or pydantic model alike."""
+    if isinstance(source, BaseModel):
+        return getattr(source, "service_tier", None)
+    elif isinstance(source, dict):
+        return source.get("service_tier")
+
+    return None
+
+
 def _get_usage_object(
-    completion_response: Any,
+    completion_response: object,
 ) -> Usage | None:
     usage_obj: Final = cast(
         Usage | ResponseAPIUsage | dict | BaseModel,
@@ -1056,6 +1075,7 @@ def _store_cost_breakdown_in_logging_obj(
     reasoning_cost: float | None = None,
     service_tier: str | None = None,
     data_residency: str | None = None,
+    vertex_location: str | None = None,
 ) -> None:
     """
     Helper function to store cost breakdown in the logging object.
@@ -1075,6 +1095,7 @@ def _store_cost_breakdown_in_logging_obj(
         margin_total_amount: Total margin added in USD
         service_tier: Tier the costs above were priced on, already resolved
         data_residency: Region uplift the costs above were priced on, already resolved
+        vertex_location: Vertex AI location the costs above were priced on, already resolved
     """
     if litellm_logging_obj is None:
         return
@@ -1098,6 +1119,7 @@ def _store_cost_breakdown_in_logging_obj(
             reasoning_cost=reasoning_cost,
             service_tier=service_tier,
             data_residency=data_residency,
+            vertex_location=vertex_location,
         )
 
     except Exception as breakdown_error:
@@ -1106,7 +1128,7 @@ def _store_cost_breakdown_in_logging_obj(
 
 
 def completion_cost(
-    completion_response=None,
+    completion_response: object | None = None,
     model: str | None = None,
     prompt="",
     messages: list = [],
@@ -1134,6 +1156,8 @@ def completion_cost(
     service_tier: str | None = None,  # for OpenAI service tier pricing
     ### DATA RESIDENCY ###
     data_residency: str | None = None,  # for OpenAI regional-processing uplift (e.g. "eu", "us")
+    ### VERTEX LOCATION ###
+    vertex_location: str | None = None,  # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global")
 ) -> float:
     """
     Calculate the cost of a given completion call fot GPT-3.5-turbo, llama2, any litellm supported llm.
@@ -1193,19 +1217,13 @@ def completion_cost(
 
         # Extract service_tier from completion_response if not provided
         if service_tier is None and completion_response is not None:
-            if isinstance(completion_response, BaseModel):
-                service_tier = getattr(completion_response, "service_tier", None)
-            elif isinstance(completion_response, dict):
-                service_tier = completion_response.get("service_tier")
+            service_tier = _extract_service_tier(completion_response)
 
         service_tier = _normalize_service_tier(service_tier)
 
         # Extract service_tier from usage object if not provided
         if service_tier is None and cost_per_token_usage_object is not None:
-            if isinstance(cost_per_token_usage_object, BaseModel):
-                service_tier = getattr(cost_per_token_usage_object, "service_tier", None)
-            elif isinstance(cost_per_token_usage_object, dict):
-                service_tier = cost_per_token_usage_object.get("service_tier")
+            service_tier = _extract_service_tier(cost_per_token_usage_object)
 
         service_tier = _normalize_service_tier(service_tier)
 
@@ -1408,7 +1426,7 @@ def completion_cost(
                     if completion_response is not None and isinstance(completion_response, RerankResponse):
                         meta_obj = completion_response.meta
                         if meta_obj is not None:
-                            billed_units = meta_obj.get("billed_units", {}) or {}
+                            billed_units: RerankBilledUnits = meta_obj.get("billed_units") or {}
                         else:
                             billed_units = {}
 
@@ -1568,6 +1586,7 @@ def completion_cost(
                     rerank_billed_units=rerank_billed_units,
                     service_tier=service_tier,
                     data_residency=data_residency,
+                    vertex_location=vertex_location,
                     response=completion_response,
                     request_model=request_model_for_cost,
                 )
@@ -1655,6 +1674,7 @@ def completion_cost(
                             usage=cost_per_token_usage_object,
                             service_tier=service_tier,
                             data_residency=data_residency,
+                            vertex_location=vertex_location,
                         )
                         _reasoning_cost = _token_type_breakdown.reasoning_cost
                         _cache_read_cost = _token_type_breakdown.cache_read_cost
@@ -1677,6 +1697,7 @@ def completion_cost(
                         reasoning_cost=_reasoning_cost,
                         service_tier=service_tier,
                         data_residency=data_residency,
+                        vertex_location=vertex_location,
                     )
 
                 return _final_cost
@@ -1756,6 +1777,8 @@ def response_cost_calculator(
     service_tier: str | None = None,  # for OpenAI service tier pricing
     ### DATA RESIDENCY ###
     data_residency: str | None = None,  # for OpenAI regional-processing uplift (e.g. "eu", "us")
+    ### VERTEX LOCATION ###
+    vertex_location: str | None = None,  # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global")
 ) -> float:
     """
     Returns
@@ -1788,6 +1811,7 @@ def response_cost_calculator(
                 litellm_logging_obj=litellm_logging_obj,
                 service_tier=service_tier,
                 data_residency=data_residency,
+                vertex_location=vertex_location,
             )
         return response_cost
     except Exception as e:
@@ -1797,7 +1821,7 @@ def response_cost_calculator(
 def ocr_cost(
     model: str,
     custom_llm_provider: str | None,
-    response: Any | None = None,
+    response: object | None = None,
 ) -> tuple[float, float]:
     """
     Args:
@@ -2156,10 +2180,10 @@ def batch_cost_calculator(
     output_cost_per_token: Final = model_info.get("output_cost_per_token")
     total_prompt_cost = 0.0
     total_completion_cost = 0.0
-    if input_cost_per_token_batches:
+    if input_cost_per_token_batches is not None:
         total_prompt_cost = usage.prompt_tokens * input_cost_per_token_batches
     elif input_cost_per_token:
-        details: Final = _parse_prompt_tokens_details(usage)
+        details: Final = parse_prompt_tokens_details(usage)
         cache_read_tokens: Final = details["cache_hit_tokens"]
         cache_creation_tokens: Final = details["cache_creation_tokens"]
 
@@ -2176,7 +2200,7 @@ def batch_cost_calculator(
 
         cache_creation_cost: Final = model_info.get("cache_creation_input_token_cost") or input_cost_per_token
         total_prompt_cost += cache_creation_tokens * cache_creation_cost / 2
-    if output_cost_per_token_batches:
+    if output_cost_per_token_batches is not None:
         total_completion_cost = usage.completion_tokens * output_cost_per_token_batches
     elif output_cost_per_token:
         total_completion_cost = (

litellm/experimental_mcp_client/client.py

@@ -6,10 +6,11 @@ import asyncio
 import base64
 import os
 from collections.abc import Awaitable, Callable, Generator
+from datetime import timedelta
 from typing import Any, Final, TypeVar
 
 import httpx
-from mcp import ClientSession, ReadResourceResult, Resource, StdioServerParameters
+from mcp import ClientSession, McpError, ReadResourceResult, Resource, StdioServerParameters
 from mcp.client.sse import sse_client
 from mcp.client.stdio import stdio_client
 
@@ -69,6 +70,29 @@ def _first_non_cancelled_cause(exc: BaseException) -> BaseException | None:
     return None
 
 
+_SDK_READ_TIMEOUT_CODE: Final = int(httpx.codes.REQUEST_TIMEOUT)
+"""The code the MCP SDK puts on its own elapsed read timeout, an HTTP status in a field that
+otherwise carries JSON-RPC error codes."""
+
+
+def _as_read_timeout(exc: BaseException) -> TimeoutError | None:
+    """The session read timeout elapsing, re-expressed as a ``TimeoutError``, or ``None``.
+
+    The SDK reports its own elapsed read timeout as ``McpError`` carrying an HTTP status code in a
+    field that otherwise holds JSON-RPC error codes, and it relays an upstream's JSON-RPC error
+    through that same class and field. The numeric code alone therefore cannot separate the two, and
+    an upstream answering with application code 408 would be reported as a gateway timeout it never
+    caused. The SDK raises its own from inside an ``except TimeoutError``, so the elapsed timeout is
+    on the context chain, while a relayed error is built from a received message and has no such
+    chain; that is the discriminator.
+    """
+    if not isinstance(exc, McpError) or exc.error.code != _SDK_READ_TIMEOUT_CODE:
+        return None
+    if not isinstance(exc.__context__, TimeoutError):
+        return None
+    return TimeoutError(exc.error.message)
+
+
 TSessionResult = TypeVar("TSessionResult")
 
 
@@ -347,7 +371,14 @@ class MCPClient:
                 session_kwargs["elicitation_callback"] = self._elicitation_callback
             if self._logging_callback is not None:
                 session_kwargs["logging_callback"] = self._logging_callback
-            session_ctx: Final = ClientSession(read_stream, write_stream, **session_kwargs)
+            # The SDK drops a response stream that ends without a JSON-RPC reply, so nothing else
+            # ever fails the request.
+            session_ctx: Final = ClientSession(
+                read_stream,
+                write_stream,
+                read_timeout_seconds=timedelta(seconds=self.timeout),
+                **session_kwargs,
+            )
             session: Final = await session_ctx.__aenter__()
             try:
                 init_result: Final = await session.initialize()
@@ -390,7 +421,16 @@ class MCPClient:
             self._last_initialize_instructions = None
             transport_ctx, http_client = self._create_transport_context()
             return await self._execute_session_operation(transport_ctx, operation)
-        except Exception:
+        except Exception as e:
+            read_timeout: Final = _as_read_timeout(e)
+            if read_timeout is not None:
+                verbose_logger.warning(
+                    "MCP client timed out after %ss waiting for %s to answer; the server accepted the "
+                    "request and ended its response stream without a JSON-RPC reply",
+                    self.timeout,
+                    self.server_url or "stdio",
+                )
+                raise read_timeout from e
             _log: Final = verbose_logger.debug if quiet_on_error else verbose_logger.warning
             _log("MCP client run_with_session failed for %s", self.server_url or "stdio")
             raise