Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_pr36033_drive

.circleci/config.yml

@@ -2744,84 +2744,6 @@ jobs:
           file: ./coverage.xml
           flags: circleci
 
-  ui_build:
-    docker:
-      - image: cimg/node:24.19@sha256:8966565f07189a67d64d6808a2b127f31dafae566508e3547f55640e1070bfad
-        auth:
-          username: ${DOCKERHUB_USERNAME}
-          password: ${DOCKERHUB_PASSWORD}
-    resource_class: medium+
-    working_directory: ~/project
-    steps:
-      - checkout
-      - skip_if_unrelated_changes:
-          category: client
-      - setup_google_dns
-      - restore_cache:
-          keys:
-            - ui-build-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
-            - ui-build-deps-v1-
-      - restore_cache:
-          keys:
-            - ui-nextjs-cache-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
-            - ui-nextjs-cache-v1-
-      - run:
-          name: Install dependencies
-          command: |
-            cd ui/litellm-dashboard
-            npm ci
-      - save_cache:
-          key: ui-build-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
-          paths:
-            - ui/litellm-dashboard/node_modules
-      - run:
-          name: Build UI
-          command: |
-            cd ui/litellm-dashboard
-            source ./build_ui.sh
-      - save_cache:
-          key: ui-nextjs-cache-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
-          paths:
-            - ui/litellm-dashboard/.next/cache
-      - persist_to_workspace:
-          root: .
-          paths:
-            - litellm/proxy/_experimental/out
-
-  ui_unit_tests:
-    docker:
-      - image: cimg/node:24.19@sha256:8966565f07189a67d64d6808a2b127f31dafae566508e3547f55640e1070bfad
-        auth:
-          username: ${DOCKERHUB_USERNAME}
-          password: ${DOCKERHUB_PASSWORD}
-    resource_class: xlarge
-    working_directory: ~/project
-    steps:
-      - checkout
-      - skip_if_unrelated_changes:
-          category: client
-      - setup_google_dns
-      - restore_cache:
-          keys:
-            - ui-unit-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
-            - ui-unit-deps-v1-
-      - run:
-          name: Install dependencies
-          command: |
-            cd ui/litellm-dashboard
-            npm ci
-      - save_cache:
-          key: ui-unit-deps-v1-{{ checksum "ui/litellm-dashboard/package-lock.json" }}
-          paths:
-            - ui/litellm-dashboard/node_modules
-      - run:
-          name: Run UI unit tests (Vitest)
-          command: |
-            cd ui/litellm-dashboard
-
-            CI=true npm run test -- --run \
-              --pool forks --poolOptions.forks.maxForks=6
-
   e2e_ui_testing:
     docker:
       - image: cimg/python:3.12-browsers@sha256:b432899af01c9a311bf74f4f22e9ada2e5306d4b1b4383f8d29e1228a5844ef2
@@ -3181,12 +3103,6 @@ workflows:
           filters: *main_branches
       - litellm_router_unit_testing:
           filters: *main_branches
-      - ui_build:
-          filters: *main_branches
-      - ui_unit_tests:
-          requires:
-            - ui_build
-          filters: *main_branches
       - auth_ui_unit_tests:
           filters: *main_branches
       - proxy_behavior_tests:

.git-blame-ignore-revs

@@ -17,3 +17,24 @@
 
 # style: unify ruff format width on 120 (#31518)
 48b5a5a0cc5a694a11219416ee0b6eb6e620e74e
+
+# refactor(imports): move collections.abc names out of typing (#35495)
+397e8e4918777e4e60a7f5e88699e0a9a7dabb3d
+
+# refactor(lint): apply every safe ruff autofix and zero 28 strict-rule budgets (#35495)
+b604e2b20c6db2099085a2f0e59b7e99e87eed6f
+
+# refactor(logging): drop redundant !s conversion flags from f-strings (#35546)
+7b2d3440cba3160277470f7a0180098ae9b87864
+
+# perf: build log messages lazily so filtered-out log records cost nothing (#35703)
+c9887a1f94bc1e7e4bdfe64d640f0509a0bc19dd
+
+# feat(lint): enforce Final on locals and freeze function parameters (#35807)
+2708620d6a599cc73c1950a942d26ac26a7ed3d4
+
+# chore(lint): remove litellm/types from the ruff lint exclusion (#35926)
+4e32a8bf6a1e1af1e04b67c759841ccef44b2235
+
+# chore(lint): strip inert type: ignore comments and zero LIT009/LIT010/LIT011 headroom (#35928)
+338e411103ad5d7003e97f34f04fa36bca542dbe

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -23,30 +23,56 @@ body:
       label: What happened?
       description: Also tell us, what did you expect to happen?
       placeholder: Tell us what you see!
-      value: "A bug happened!"
     validations:
       required: true
   - type: textarea
-    id: steps-to-reproduce
+    id: user-flow
     attributes:
-      label: Steps to Reproduce
-      description: Please provide a numbered list of the exact steps to reproduce this bug (include a curl/python snippet to reproduce it). Number each step (1., 2., 3., ...) in the order you performed them.
+      label: User Flow
+      description: |
+        Two ordered lists, "Before a (hypothetical) fix" and "After a (hypothetical) fix", walking the same end user through the same task, written strictly from that user's seat. Every rule below applies.
+
+        - Describe the real application and the routes its users actually hit, not a generic scenario
+        - Lead each list with one plain sentence saying where the flow fails (before) or would succeed (after), then number the steps
+        - Every step is something the user does or observes: the HTTP method and full URL they hit, what they sent, and what visibly came back (status code, error text, the shape of an ID). UI steps name the page URL and what is on screen
+        - No LiteLLM internals: never name functions, files, DB tables, config classes, hooks, callbacks, or code paths. "The upload hands back an ID that looks like OpenAI's own `file-abc123` instead of the scrambled one the gateway returned" is right, "no managed-file row was registered" is wrong
+        - Keep the two lists step-for-step identical until they diverge, so the broken step is obvious
+        - If the bug has a security or authorization consequence, end each list with what another user can do that they shouldn't be able to, and what they could no longer do after a fix
       placeholder: |
-        1. config.yaml file/ .env file/ etc.
-        2. Run the following code...
-        3. Observe the error...
-      value: |
-        1. 
-        2. 
-        3. 
+        Before a (hypothetical) fix: a developer whose app streams chat completions gets no token counts back, so their cost dashboard reads zero
+
+        1. They send POST https://litellm-domain/v1/chat/completions with "stream": true and no stream_options
+        2. The last SSE chunk arrives with "usage": null, so their app records 0 prompt and 0 completion tokens
+        3. They open https://litellm-domain/ui/?page=logs and see the request logged at $0 spend
+
+        After a (hypothetical) fix: the same request comes back with real token counts, so the dashboard shows real spend
+
+        1. The proxy admin sets always_include_stream_usage: true and restarts the proxy
+        2. The developer sends the same POST https://litellm-domain/v1/chat/completions with "stream": true and no stream_options
+        3. The last SSE chunk now carries a usage object with real prompt and completion token counts
+        4. https://litellm-domain/ui/?page=logs shows that request at non-zero spend
     validations:
       required: true
   - type: textarea
-    id: logs
+    id: proof-of-bug
     attributes:
-      label: Relevant log output
-      description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks.
-      render: shell
+      label: Proof the bug occurs
+      description: |
+        The commands (e.g., curl) and their full output, screenshots, or a screen recording demonstrating that the bug happens. Every rule below applies.
+
+        - The proof must be completely e2e with no mocks, against a live proxy you ran yourself (e.g., `litellm --config config.yaml --detailed_debug` on localhost:4000), hitting real LLM provider APIs, costing real $ if needed, where the bug involves a provider call. `pytest` commands are not enough
+        - Show exactly what the end user sees or does, matching the User Flow above step for step
+        - Start with the config.yaml (or SDK setup) and any env vars the proxy ran with, then the exact version or commit hash the proof was captured at, so a maintainer can stand up the same proxy before running your commands. Keep the real values for env vars that aren't sensitive, they are often the reason the bug happens, and redact only the secrets: never paste a real API key, virtual key, database URL, or other credential, here or anywhere else in the issue
+        - If the bug applies to more than one of the LLM endpoints (/v1/responses, /v1/chat/completions, /v1/messages), include proof for every one of them, not just one
+        - For UI bugs: include screenshots and the page URLs you were on. Scrub keys and tokens out of screenshots too (for example, the virtual key is briefly shown in the panel right after you create a virtual key)
+      placeholder: |
+        Config / setup the proxy ran with:
+
+        Version or commit:
+
+        Commands and their full output:
+    validations:
+      required: true
   - type: dropdown
     id: component
     attributes:

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -24,10 +24,53 @@ body:
     validations:
       required: true
   - type: textarea
-    id: motivation
+    id: user-flow
     attributes:
-      label: Motivation, pitch
-      description: Please outline the motivation for the proposal. Is your feature request related to a specific problem? e.g., "I'm working on X and would like Y to be possible". If this is related to another GitHub issue, please link here too.
+      label: User Flow
+      description: |
+        Two ordered lists, "Before this feature (today)" and "After this feature (ideal user flow)", walking the same end user through the same task, written strictly from that user's seat. Every rule below applies.
+
+        - Describe the real application and the routes its users actually hit, not a generic scenario. Link any related GitHub issue or provider API docs
+        - Lead each list with one plain sentence saying where the flow dead-ends today and what it would let them do instead, then number the steps
+        - Every step is something the user does or observes: the HTTP method and full URL they hit, what they sent, and what visibly came back (status code, error text, the shape of an ID). UI steps name the page URL and what is on screen
+        - No LiteLLM internals: never name functions, files, DB tables, config classes, hooks, callbacks, or code paths. Ask for the behavior you need, not the implementation you imagine
+        - Keep the two lists step-for-step identical until they diverge, so the missing capability is obvious
+        - "Before this feature" is also where you show the workaround you're living with, which is what tells us how badly this is needed
+      placeholder: |
+        Before this feature (today): a developer batching nightly summaries has no way to mark those calls as low priority, so they compete with live traffic for the same rate limit
+
+        1. They send POST https://litellm-domain/v1/chat/completions for 500 documents in a loop
+        2. Around document 120 they start getting 429s naming the rpm limit, and their user-facing chat app starts getting them too
+        3. Their workaround is a hand-rolled sleep between calls, which stretches the batch to 3 hours and still collides at peak
+
+        After this feature (ideal user flow): the same batch runs as background work that yields to live traffic
+
+        1. The developer sends the same POST with "service_tier": "flex"
+        2. Batch calls queue behind interactive ones instead of 429ing, and the response comes back with the tier it was served at
+        3. The live chat app keeps returning 200s throughout the batch
+        4. https://litellm-domain/ui/?page=logs shows the batch requests tagged with that tier
+    validations:
+      required: true
+  - type: textarea
+    id: how-far-you-got
+    attributes:
+      label: How far you got
+      description: |
+        Run as many steps of the "After this feature (ideal user flow)" list as you can against a live proxy you ran yourself (e.g., `litellm --config config.yaml --detailed_debug` on localhost:4000), then paste the commands (e.g., curl) and their full output, ending at the step that dead-ends. Every rule below applies.
+
+        - Say plainly what stopped you there, in user terms: the option you passed came back ignored, the response 400'd naming an unsupported field, there is no button on the page for it. This is what proves the feature is genuinely missing rather than undocumented
+        - No mocks. Where the flow involves a provider call, hit the real provider API, even if it costs real $. `pytest` commands are not enough
+        - Include the config.yaml (or SDK setup) and env vars the proxy ran with, plus the version or commit you were on. Keep the real values for env vars that aren't sensitive, and redact only the secrets: never paste a real API key, virtual key, database URL, or other credential, here or anywhere else in the issue
+        - If the provider already supports this, link their API docs and paste a direct call to them succeeding, so we can see the shape LiteLLM should be sending
+        - For UI asks: include screenshots of the page you got stuck on and its URL. Scrub keys and tokens out of screenshots too (for example, the virtual key is briefly shown in the panel right after you create a virtual key)
+      placeholder: |
+        Config / setup the proxy ran with:
+
+        Version or commit:
+
+        Commands and their full output, up to the step that dead-ends:
+
+        What stopped me there:
     validations:
       required: true
   - type: dropdown

.github/actions/cache-prisma-binaries/action.yml

@@ -0,0 +1,40 @@
+name: "Cache Prisma binaries"
+description: >-
+  Cache the Prisma CLI and engine binaries that `prisma generate` downloads, so
+  only the first job on a given prisma-client-py version pays for the download.
+
+  prisma-client-py shells out to `npm install prisma@<version>` whenever its
+  binary cache directory has no CLI entrypoint, which pulls ~85 MB of query and
+  schema engines over the network. That normally takes a few seconds, but it is
+  unbounded: one shard of a proxy-db run took 5m18s on that single step versus
+  3.8s on its eleven siblings, which pushed the job past its timeout and got a
+  fully passing test run cancelled.
+
+  Callers must not set PRISMA_BINARY_CACHE_DIR. The prisma-client-py default
+  (~/.cache/prisma-python/binaries/<prisma-version>/<engine-version>) is already
+  keyed by both versions, so a cache entry can never be served to a run that
+  expects different binaries.
+
+runs:
+  using: composite
+  steps:
+    - name: Resolve prisma-client-py version
+      id: version
+      shell: bash
+      run: |
+        version="$(grep -A1 '^name = "prisma"$' uv.lock | sed -n 's/^version = "\(.*\)"$/\1/p' | head -1)"
+        if [ -z "${version}" ]; then
+          echo "could not resolve the prisma package version from uv.lock" >&2
+          exit 1
+        fi
+        echo "version=${version}" >> "$GITHUB_OUTPUT"
+
+    - name: Restore Prisma binaries
+      uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+      with:
+        # ~/.cache/prisma-python holds the npm install tree prisma-client-py
+        # drives; ~/.cache/prisma is where @prisma/engines stages its downloads.
+        path: |
+          ~/.cache/prisma-python
+          ~/.cache/prisma
+        key: ${{ runner.os }}-prisma-binaries-${{ steps.version.outputs.version }}

.github/ci-coverage-allowlist.yml

@@ -136,13 +136,6 @@ test_paths:
       - tests/litellm-proxy-extras/test_litellm_proxy_extras_utils.py
 
 dockerfiles:
-  - reason: >-
-      The componentized images the microservices chart deploys are built by no job; wiring both into
-      the scan workflow costs a full image build each and is deferred to a change that prices the
-      whole set
-    paths:
-      - backend/Dockerfile
-      - gateway/Dockerfile
   - reason: >-
       The dashboard container is a static Next.js export served by nginx, and the dashboard build
       and lint workflows already exercise that output, so building the image adds no signal about it

.github/pull_request_template.md

@@ -13,6 +13,33 @@ How it solves it:
 - <blah>
 - ...
 
+## User Flow
+
+<!-- Two ordered lists, Before and After, walking the same end user through the same task, written strictly from that user's seat
+     Read the linked issue, ticket, or customer thread first so the flow reflects the real application and the routes its users actually hit; don't invent a generic scenario
+     Lead each list with one plain sentence saying where the flow fails (Before) or succeeds (After), then number the steps
+     Every step is something the user does or observes: the HTTP method and full URL they hit, what they sent, and what visibly came back (status code, error text, the shape of an ID). UI steps name the page URL and what is on screen
+     No LiteLLM internals: never name functions, files, DB tables, config classes, hooks, callbacks, or code paths. "The upload hands back an ID that looks like OpenAI's own `file-abc123` instead of the scrambled one the gateway returned" is right, "no managed-file row was registered" is wrong
+     Keep the two lists step-for-step identical until they diverge, so the changed step is obvious
+     If the bug had a security or authorization consequence, end each list with what another user could or could no longer do
+     Regenerate this section whenever new commits change the PR's behavior, so it never describes an older revision
+
+Example:
+
+Before: a developer whose app streams chat completions gets no token counts back, so their cost dashboard reads zero
+
+1. They send POST https://litellm-domain/v1/chat/completions with `"stream": true` and no `stream_options`
+2. The last SSE chunk arrives with `"usage": null`, so their app records 0 prompt and 0 completion tokens
+3. They open https://litellm-domain/ui/?page=logs and see the request logged at $0 spend
+
+After: the same request comes back with real token counts, so the dashboard shows real spend
+
+1. The proxy admin sets `always_include_stream_usage: true` and restarts the proxy
+2. The developer sends the same POST https://litellm-domain/v1/chat/completions with `"stream": true` and no `stream_options`
+3. The last SSE chunk now carries a `usage` object with real prompt and completion token counts
+4. https://litellm-domain/ui/?page=logs shows that request at non-zero spend
+-->
+
 ## Relevant issues
 
 <!-- e.g., "Fixes #000" -->
@@ -37,12 +64,36 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
 ## Screenshots / Proof of Fix
 
 <!-- Include screenshots, screen recordings, or command (e.g., curl) + output demonstrating that your changes work as expected
-     The proof must be completely e2e with no mocks, using, for example, actual LLM calls costing real $. `pytest` commands are not enough
-     For bug fixes: show reproduction before the fix and passing behavior after
-     Include the commit hash each proof was captured at, for both the before and the after runs
-     If the change applies to all three LLM endpoints (/v1/responses, /v1/chat/completions, /v1/messages), include proof for every single one of them, not just one
-     For new features: show the feature working end-to-end
-     For UI changes: include before/after screenshots -->
+     The proof must be completely e2e with no mocks, using actual LLM calls costing real $$$ if applicable. `pytest` commands are not enough
+     Show ONLY the latest run: capture Before at the merge base and After at the PR's current tip, and when new commits change behavior, replace this whole section with the fresh run instead of stacking it on top of older ones. The run must be up to date. As soon as a new commit is made and it makes this PR description's after sha stale (it's no longer tip of PR), you must re-run the QA
+     Structure the section exactly as below: Before and After one heading level below this section, each naming the commit hash it was captured at, one lower-level heading per case inside each, the same case names in the same order on both sides, and numbered steps (command, observed output) under every case, never loose prose; shared setup (config, payloads) goes above Before, and with a single case, drop the case headings and number the steps directly
+
+### Before (<hash>)
+
+#### <case 1>
+
+1. ...
+2. ...
+
+#### <case 2>
+
+1. ...
+
+### After (<hash>)
+
+#### <case 1>
+
+1. ...
+2. ...
+
+#### <case 2>
+
+1. ...
+
+     For bug fixes: Before shows the reproduction, After shows the same steps passing
+     For new features: Before shows the capability missing, After shows it working end-to-end
+     If the change applies to all three LLM endpoints (/v1/responses, /v1/chat/completions, /v1/messages), make each endpoint its own case, not just one
+     For UI changes: before/after screenshots under the same headings -->
 
 ## Type
 
@@ -56,7 +107,11 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
 🚄 Infrastructure
 ✅ Test
 
-## Changes
+## Caveats (if any)
+
+<!-- Short bullet points, just like the TLDR: one line per bullet, roughly 10 words max
+     Call out known limitations, follow-up work, or anything a reviewer should watch out for
+     Leave this section empty if there are none -->
 
 ## QA runbook
 

.github/scripts/triage_with_llm.py

@@ -582,7 +582,9 @@ def build_issue_prompt(*, title: str, body: str) -> str:
                   Commands whose external dependencies (LLM provider, DB,
                   network) are mocked or stubbed do NOT count.
               Prose-only "steps to reproduce" with no run output, video, or
-              screenshot do NOT satisfy (1).
+              screenshot do NOT satisfy (1). An unfilled template scaffold
+              (bare headings such as "Version or commit:" with nothing under
+              them, empty numbered lists) counts as absent, not as evidence.
           (2) Expected vs. actual behavior (`has_expected_vs_actual`).
 
         FAIL the bug report if either (1) or (2) is missing. Do not bias
@@ -595,6 +597,13 @@ def build_issue_prompt(*, title: str, body: str) -> str:
             that it does not today).
           - Motivation / use case with a concrete example (config, API call,
             UI flow, or scenario showing what's blocked today).
+          - END-TO-END EVIDENCE OF THE DEAD-END (set
+            `has_dead_end_evidence=true` only when this is present): a video,
+            a screenshot, or the exact command(s) actually run paired with
+            their real output, showing the point where the flow stops today.
+            Mocked or stubbed dependencies do NOT count, and an unfilled
+            template scaffold (bare headings, empty numbered lists) counts as
+            absent.
 
         For an issue that is neither a bug report nor a feature request (a
         question, support request, or discussion), PASS as long as it has a
@@ -608,6 +617,7 @@ def build_issue_prompt(*, title: str, body: str) -> str:
           "has_repro": boolean,
           "has_expected_vs_actual": boolean,
           "has_motivation_example": boolean,
+          "has_dead_end_evidence": boolean,
           "missing": ["plain-english strings naming what is missing"],
           "explanation": "1-2 sentence reasoning for the team to skim"
         }}
@@ -705,6 +715,10 @@ _ISSUE_BUG_LABELS: tuple[tuple[str, str], ...] = (
 )
 _ISSUE_FEATURE_LABELS: tuple[tuple[str, str], ...] = (
     ("has_motivation_example", "Motivation and concrete example"),
+    (
+        "has_dead_end_evidence",
+        "End-to-end evidence of the dead-end (video, screenshot, or command + real output)",
+    ),
 )
 
 
@@ -836,8 +850,11 @@ def format_issue_close_comment(verdict: dict) -> str:
         "video, a screenshot, or the exact commands you ran with their real output / "
         "traceback) plus expected vs. actual behavior. Written steps with no run output, "
         "video, or screenshot don't count, and mocked or stubbed runs don't count.\n"
-        "   - For **feature requests**: a concrete description of what should change, plus a "
-        "use case and example (config / API call / UI flow).\n"
+        "   - For **feature requests**: a concrete description of what should change, a "
+        "use case and example (config / API call / UI flow), plus end-to-end evidence of "
+        "the dead-end (a video, a screenshot, or the exact commands you ran with their "
+        "real output showing where the flow stops today). Mocked or stubbed runs don't "
+        "count.\n"
         "2. Comment `@agent-shin reconsider`. I'll re-run triage and reopen the issue if it "
         "now meets the bar. (GitHub doesn't let external authors reopen an issue a maintainer "
         "or bot closed, so the comment-based reconsider is the reliable path.)\n"
@@ -943,8 +960,10 @@ def format_grace_warning_issue_comment(verdict: dict) -> str:
         "screenshot, or the exact commands you ran with their real output / traceback) plus "
         "expected vs. actual behavior. Written steps with no run output don't count, and "
         "mocked or stubbed runs don't count.\n"
-        "- For **feature requests**: a concrete description of what should change, plus a use "
-        "case and example (config / API call / UI flow).\n"
+        "- For **feature requests**: a concrete description of what should change, a use "
+        "case and example (config / API call / UI flow), plus end-to-end evidence of the "
+        "dead-end (a video, a screenshot, or the exact commands you ran with their real "
+        "output showing where the flow stops today). Mocked or stubbed runs don't count.\n"
         "\n"
         "**If the issue does get auto-closed in 2 hours**, comment `@agent-shin reconsider` "
         "and I'll re-evaluate. If it now meets the bar, I'll reopen the issue.\n"

.github/workflows/_test-unit-base.yml

@@ -18,10 +18,25 @@ on:
         type: number
         default: 2
       timeout-minutes:
-        description: "Job timeout in minutes"
+        description: >-
+          Timeout for the test step alone. Setup (checkout, dependency install,
+          Prisma client generation) gets its own allowance on top, so a slow
+          runner or a cold binary download can never cancel passing tests.
         required: false
         type: number
         default: 20
+      job-timeout-minutes:
+        description: >-
+          Backstop for the whole job. Keep it >= `timeout-minutes` plus 35: 30 for
+          the per-step ceilings on the setup steps below, and 5 for the runner
+          overhead the job clock charges but no step owns (job init, step
+          transitions, post-job cleanup). That headroom is what makes the test
+          budget a floor rather than a hope, since setup cannot overrun into it
+          without failing its own step first. GitHub expressions have no
+          arithmetic, so the sum is passed in rather than computed.
+        required: false
+        type: number
+        default: 55
       max-failures:
         description: "Stop after this many failures"
         required: false
@@ -44,30 +59,35 @@ jobs:
   run:
     name: Run tests
     runs-on: ubuntu-latest
-    timeout-minutes: ${{ inputs.timeout-minutes }}
+    timeout-minutes: ${{ inputs.job-timeout-minutes }}
     outputs:
       decision: ${{ steps.changes.outputs.decision }}
 
     steps:
       - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        timeout-minutes: 3
         with:
           persist-credentials: false
 
       - name: Detect backend-relevant changes
         id: changes
+        timeout-minutes: 2
         uses: ./.github/actions/detect-backend-changes
 
       - name: Set up Python
+        timeout-minutes: 3
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
           python-version: "3.12"
 
       - name: Set up uv
+        timeout-minutes: 3
         uses: ./.github/actions/setup-uv-with-retries
         with:
           version: "0.10.9"
 
       - name: Cache uv dependencies
+        timeout-minutes: 5
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:
           path: |
@@ -79,18 +99,24 @@ jobs:
 
       - name: Install dependencies
         if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 8
         run: |
           .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml
 
+      - name: Cache Prisma binaries
+        if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: 3
+        uses: ./.github/actions/cache-prisma-binaries
+
       - name: Generate Prisma client
         if: steps.changes.outputs.decision != 'skip'
-        env:
-          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
+        timeout-minutes: 3
         run: |
           uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 
       - name: Run tests
         if: steps.changes.outputs.decision != 'skip'
+        timeout-minutes: ${{ inputs.timeout-minutes }}
         env:
           TEST_PATH: ${{ inputs.test-path }}
           MAX_FAILURES: ${{ inputs.max-failures }}

.github/workflows/check-schema-sync.yml

@@ -10,6 +10,10 @@ on:
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   check-sync:
     name: Verify schema.prisma copies match root

.github/workflows/check-ui-api-types.yml

@@ -2,18 +2,19 @@ name: Check UI API Types Sync
 
 on:
   pull_request:
-    paths:
-      - "litellm/proxy/**"
-      - "litellm/types/**"
-      - "ui/litellm-dashboard/src/lib/http/schema.d.ts"
-      - "ui/litellm-dashboard/scripts/gen-api-types.mjs"
-      - "ui/litellm-dashboard/package.json"
-      - "ui/litellm-dashboard/package-lock.json"
-      - ".github/workflows/check-ui-api-types.yml"
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_staging
+      - "litellm_**"
 
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   check-sync:
     name: Verify schema.d.ts matches the proxy OpenAPI spec
@@ -24,18 +25,39 @@ jobs:
         uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
         with:
           persist-credentials: false
+          fetch-depth: 2
+
+      - name: Detect changes that can affect the generated types
+        id: changes
+        run: |
+          set -euo pipefail
+          if ! base="$(git rev-parse --verify --quiet HEAD^2 >/dev/null && git rev-parse HEAD^1)"; then
+            echo "Not a pull request merge commit, running the full check."
+            echo "relevant=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          files="$(git diff --name-only "$base" HEAD)"
+          if grep -Eq '^(litellm/(proxy|types)/|ui/litellm-dashboard/(src/lib/http/schema\.d\.ts|scripts/gen-api-types\.mjs|package(-lock)?\.json)$|\.github/workflows/check-ui-api-types\.yml$)' <<< "$files"; then
+            echo "relevant=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "No proxy, types or generator changes in this pull request, nothing to verify."
+            echo "relevant=false" >> "$GITHUB_OUTPUT"
+          fi
 
       - name: Set up Python
+        if: steps.changes.outputs.relevant == 'true'
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
           python-version: "3.12"
 
       - name: Set up uv
+        if: steps.changes.outputs.relevant == 'true'
         uses: ./.github/actions/setup-uv-with-retries
         with:
           version: "0.10.9"
 
       - name: Cache uv dependencies
+        if: steps.changes.outputs.relevant == 'true'
         uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
         with:
           path: |
@@ -46,14 +68,19 @@ jobs:
             ${{ runner.os }}-uv-
 
       - name: Install backend dependencies
+        if: steps.changes.outputs.relevant == 'true'
         run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
 
+      - name: Cache Prisma binaries
+        if: steps.changes.outputs.relevant == 'true'
+        uses: ./.github/actions/cache-prisma-binaries
+
       - name: Generate Prisma client
-        env:
-          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
+        if: steps.changes.outputs.relevant == 'true'
         run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 
       - name: Set up Node.js
+        if: steps.changes.outputs.relevant == 'true'
         uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
         with:
           node-version-file: ui/litellm-dashboard/.nvmrc
@@ -61,16 +88,19 @@ jobs:
           cache-dependency-path: ui/litellm-dashboard/package-lock.json
 
       - name: Install dashboard dependencies
+        if: steps.changes.outputs.relevant == 'true'
         working-directory: ui/litellm-dashboard
         run: npm ci
 
       - name: Regenerate types from the live spec
+        if: steps.changes.outputs.relevant == 'true'
         working-directory: ui/litellm-dashboard
         env:
           LITELLM_PYTHON: "uv run --no-sync python"
         run: npm run gen:api
 
       - name: Fail if types are stale
+        if: steps.changes.outputs.relevant == 'true'
         run: |
           if ! git diff --exit-code -- ui/litellm-dashboard/src/lib/http/schema.d.ts; then
             echo "::error file=ui/litellm-dashboard/src/lib/http/schema.d.ts::Generated API types are out of sync with the proxy OpenAPI spec."

.github/workflows/conventional-commits.yml

@@ -14,6 +14,10 @@ on:
 permissions:
   pull-requests: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   lint-pr-title:
     name: Validate PR title

.github/workflows/guard-fork-dependencies.yml

@@ -15,6 +15,10 @@ on:
 
 permissions: {}
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   guard:
     name: Block fork dependency changes

.github/workflows/helm_unit_test.yml

@@ -9,6 +9,10 @@ on:
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   unit-test:
     runs-on: ubuntu-latest

.github/workflows/image-scan.yml

@@ -12,6 +12,11 @@ on:
       - docker/Dockerfile.non_root
       - migrations/Dockerfile
       - migrations/run.py
+      - gateway/Dockerfile
+      - gateway/main.py
+      - backend/Dockerfile
+      - backend/main.py
+      - docker/component_entrypoint.sh
       - litellm-proxy-extras/**
       - tests/proxy_migration_tests/**
       - uv.lock
@@ -147,3 +152,63 @@ jobs:
         run: |
           python -m pip install "pytest==9.0.3"
           python -m pytest tests/proxy_migration_tests/test_offline_image_migration.py -v
+
+  gateway-image:
+    name: gateway-image
+    runs-on: ubuntu-latest
+    if: >-
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name == github.repository
+    timeout-minutes: 30
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Build gateway image
+        run: docker build -f gateway/Dockerfile -t litellm-gateway-scan:${{ github.sha }} .
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Verify the gateway serves offline as a non-root uid
+        env:
+          LITELLM_IMAGE: litellm-gateway-scan:${{ github.sha }}
+          LITELLM_COMPONENT_PORT: "4000"
+        run: |
+          python -m pip install "pytest==9.0.3"
+          python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py -v
+
+  backend-image:
+    name: backend-image
+    runs-on: ubuntu-latest
+    if: >-
+      github.event_name != 'pull_request' ||
+      github.event.pull_request.head.repo.full_name == github.repository
+    timeout-minutes: 30
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - name: Build backend image
+        run: docker build -f backend/Dockerfile -t litellm-backend-scan:${{ github.sha }} .
+
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Verify the backend serves offline as a non-root uid
+        env:
+          LITELLM_IMAGE: litellm-backend-scan:${{ github.sha }}
+          LITELLM_COMPONENT_PORT: "4001"
+        run: |
+          python -m pip install "pytest==9.0.3"
+          python -m pytest tests/proxy_migration_tests/test_component_image_serves_offline.py -v

.github/workflows/mutation-test.yml

@@ -57,9 +57,10 @@ jobs:
         run: |
           .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml
 
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
       - name: Generate Prisma client
-        env:
-          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
         run: |
           uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 

.github/workflows/publish-basedpyright-base-counts.yml

@@ -43,22 +43,15 @@ jobs:
         with:
           version: "0.10.9"
 
-      - name: Install dependencies
-        run: |
-          uv sync --frozen --group proxy-dev --group e2e-dev
-
-      # Mirrors test-linting.yml's lint job: basedpyright resolves Prisma's
-      # generated client only after `prisma generate`, and the published counts
-      # must match what that job would measure for the same tree.
-      - name: Generate Prisma client
-        env:
-          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
-        run: |
-          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
 
+      # The gate provisions its own measurement env (.venv-typecheck: a frozen
+      # uv sync of its canonical dependency groups plus a generated Prisma
+      # client), so no install step here can drift from what local runs measure.
       - name: Emit basedpyright counts for HEAD
         run: |
-          uv run --no-sync python scripts/type_check_gate.py --emit-counts-dir "$RUNNER_TEMP/basedpyright-counts"
+          python scripts/type_check_gate.py --emit-counts-dir "$RUNNER_TEMP/basedpyright-counts"
           counts_file=$(ls "$RUNNER_TEMP"/basedpyright-counts/basedpyright-counts-*.json)
           echo "COUNTS_ARTIFACT_NAME=$(basename "$counts_file" .json)" >> "$GITHUB_ENV"
 

.github/workflows/test-code-quality.yml

@@ -65,6 +65,12 @@ jobs:
       - name: check_provider_folders_documented
         run: uv run --no-sync python ./tests/code_coverage_tests/check_provider_folders_documented.py
 
+      - name: check_prisma_binary_cache
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_prisma_binary_cache.py
+
+      - name: check_workflow_startup_safety
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_workflow_startup_safety.py
+
       - name: router_code_coverage
         run: uv run --no-sync python ./tests/code_coverage_tests/router_code_coverage.py
 

.github/workflows/test-linting.yml

@@ -11,6 +11,10 @@ on:
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   lint:
     runs-on: ubuntu-latest
@@ -39,9 +43,10 @@ jobs:
           BASE_SHA: ${{ github.event.pull_request.base.sha }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: |
-          MERGE_BASE=$(gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
+          retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
+          MERGE_BASE=$(retry gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha')
           test -n "$MERGE_BASE"
-          git fetch --no-tags --depth=1 origin "$MERGE_BASE"
+          retry git fetch --no-tags --depth=1 origin "$MERGE_BASE"
           echo "GATE_BASE_SHA=$MERGE_BASE" >> "$GITHUB_ENV"
 
       - name: Set up Python
@@ -67,12 +72,13 @@ jobs:
         run: |
           uv sync --frozen --group proxy-dev --group e2e-dev
 
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
       # basedpyright resolves Prisma's generated client (litellm/proxy/schema.prisma)
       # only after `prisma generate` writes prisma/client.py et al. Without this the
       # DB wrappers typed against the generated client would degrade to Unknown.
       - name: Generate Prisma client
-        env:
-          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
         run: |
           uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 
@@ -156,7 +162,8 @@ jobs:
         env:
           BASE_SHA: ${{ github.event.pull_request.base.sha }}
         run: |
-          git fetch --no-tags --depth=1 origin "$BASE_SHA"
+          retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
+          retry git fetch --no-tags --depth=1 origin "$BASE_SHA"
 
       - name: Set up Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
@@ -200,7 +207,8 @@ jobs:
           GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }}
         run: |
           if [ -n "$GITGUARDIAN_API_KEY" ]; then
-            git fetch --no-tags --unshallow origin
+            retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; }
+            retry git fetch --no-tags --unshallow origin
             uv tool run --from 'ggshield==1.48.0' ggshield secret scan repo .
           else
             echo "GITGUARDIAN_API_KEY not set, skipping ggshield scan"

.github/workflows/test-litellm-ui-build.yml

@@ -10,6 +10,10 @@ on:
       - litellm_oss_staging
       - "litellm_**"
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   build-ui:
     runs-on: ubuntu-latest

.github/workflows/test-litellm-ui-lint.yml

@@ -10,6 +10,10 @@ on:
       - litellm_oss_staging
       - "litellm_**"
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   frontend-lint:
     runs-on: ubuntu-latest

.github/workflows/test-litellm-ui-unit.yml

@@ -42,6 +42,11 @@ jobs:
       - name: Install dependencies
         run: npm ci
 
+      - name: Run UI type tests (Vitest)
+        env:
+          CI: "true"
+        run: npm run test:types
+
       - name: Run UI unit tests (Vitest)
         env:
           CI: "true"

.github/workflows/test-mcp.yml

@@ -11,6 +11,10 @@ on:
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   test:
     runs-on: ubuntu-latest

.github/workflows/test-model-map.yaml

@@ -11,6 +11,10 @@ on:
 permissions:
   contents: read
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
   validate-model-prices-json:
     runs-on: ubuntu-latest

.github/workflows/test-terraform-modules.yml

@@ -0,0 +1,54 @@
+name: Terraform Modules
+
+on:
+  push:
+    paths:
+      - "terraform/litellm/aws/**"
+      - ".github/workflows/test-terraform-modules.yml"
+  pull_request:
+    branches:
+      - main
+      - litellm_internal_staging
+      - litellm_oss_staging
+      - "litellm_**"
+    paths:
+      - "terraform/litellm/aws/**"
+      - ".github/workflows/test-terraform-modules.yml"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  aws-module:
+    name: fmt, validate, test (aws)
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    defaults:
+      run:
+        working-directory: terraform/litellm/aws
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          persist-credentials: false
+
+      - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2
+        with:
+          terraform_version: 1.13.3
+          terraform_wrapper: false
+
+      - name: fmt
+        run: terraform fmt -recursive -check -diff
+
+      - name: init
+        run: terraform init -backend=false -input=false
+
+      - name: validate
+        run: terraform validate
+
+      # Plan-only, mock_provider-backed: no AWS credentials, no API calls.
+      - name: test
+        run: terraform test

.github/workflows/test-terraform-provider.yml

@@ -92,9 +92,10 @@ jobs:
         run: |
           .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
 
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
       - name: Generate Prisma client
-        env:
-          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
         run: |
           uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 

.github/workflows/test-unit-documentation.yml

@@ -65,10 +65,12 @@ jobs:
         run: |
           .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
 
+      - name: Cache Prisma binaries
+        if: steps.changes.outputs.decision != 'skip'
+        uses: ./.github/actions/cache-prisma-binaries
+
       - name: Generate Prisma client
         if: steps.changes.outputs.decision != 'skip'
-        env:
-          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
         run: |
           uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 

.github/workflows/test-unit-proxy-db.yml

@@ -28,6 +28,10 @@ concurrency:
 #     Most of a shard's time is pytest plugin load + xdist worker imports +
 #     pytest-cov instrumentation, not the tests themselves. Keeping per-shard
 #     work low and matching worker count to runner cores is what controls it.
+#   * `timeout` bounds the pytest step only. Checkout, dependency install, and
+#     Prisma client generation draw on a separate allowance in the base
+#     workflow, so slow setup shows up as a slow job rather than as a
+#     cancelled shard whose tests were passing.
 #   * workers: 4 matches the 4-core ubuntu-latest runner. -n 8 on 4 cores
 #     oversubscribes 2x and workers fight for CPU during their cold-start
 #     imports (measured ~441% CPU for -n 8 locally, i.e. ~55% effective).
@@ -131,8 +135,6 @@ jobs:
             test-path: >-
               tests/proxy_unit_tests/test_proxy_server.py
               tests/proxy_unit_tests/test_proxy_server_keys.py
-              tests/proxy_unit_tests/test_proxy_server_caching.py
-              tests/proxy_unit_tests/test_proxy_server_langfuse.py
               tests/proxy_unit_tests/test_proxy_server_spend.py
               tests/proxy_unit_tests/test_aproxy_startup.py
             workers: 4

.github/workflows/test-unit-proxy-endpoints.yml

@@ -38,12 +38,15 @@ jobs:
         tests/test_litellm/proxy/google_endpoints
         tests/test_litellm/proxy/openai_files_endpoint
         tests/test_litellm/proxy/batches_endpoints
+        tests/test_litellm/proxy/fine_tuning_endpoints
+        tests/test_litellm/proxy/vector_store_files_endpoints
         tests/test_litellm/proxy/video_endpoints
         tests/test_litellm/proxy/response_api_endpoints
         tests/test_litellm/proxy/image_endpoints
         tests/test_litellm/proxy/vector_store_endpoints
         tests/test_litellm/proxy/agent_endpoints
         tests/test_litellm/proxy/a2a
+        tests/test_litellm/proxy/credential_endpoints
         tests/test_litellm/proxy/discovery_endpoints
         tests/test_litellm/proxy/health_endpoints
         tests/test_litellm/proxy/shutdown
@@ -73,4 +76,5 @@ jobs:
       workers: 4
       reruns: 2
       timeout-minutes: 60
+      job-timeout-minutes: 95
       artifact-name: proxy-server

.github/workflows/test-unit-proxy-legacy.yml

@@ -1,104 +0,0 @@
-name: "Unit Tests: Proxy Legacy Tests"
-
-on:
-  pull_request:
-    branches:
-      - main
-      - litellm_internal_staging
-      - litellm_oss_staging
-      - "litellm_**"
-  push:
-    branches:
-      - main
-      - litellm_internal_staging
-
-permissions:
-  contents: read
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
-    strategy:
-      fail-fast: false
-      matrix:
-        test-group:
-          - name: "auth-and-jwt"
-            path: "tests/proxy_unit_tests/test_[a-j]*.py"
-          - name: "key-generation"
-            path: "tests/proxy_unit_tests/test_[k-o]*.py"
-          - name: "proxy-config"
-            path: "tests/proxy_unit_tests/test_prisma*.py tests/proxy_unit_tests/test_prompt*.py tests/proxy_unit_tests/test_proxy_[c-r]*.py"
-          - name: "proxy-server"
-            path: "tests/proxy_unit_tests/test_proxy_server.py"
-          - name: "proxy-server-extras"
-            path: "tests/proxy_unit_tests/test_proxy_server_*.py tests/proxy_unit_tests/test_proxy_setting_guardrails.py"
-          - name: "proxy-utils"
-            path: "tests/proxy_unit_tests/test_proxy_utils.py"
-          - name: "proxy-token-counter"
-            path: "tests/proxy_unit_tests/test_proxy_token_counter.py"
-          - name: "proxy-response-and-misc"
-            path: "tests/proxy_unit_tests/test_[r-t]*.py"
-          - name: "proxy-user-auth-and-spend"
-            path: "tests/proxy_unit_tests/test_[u-z]*.py"
-
-    name: ${{ matrix.test-group.name }}
-
-    steps:
-      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
-        with:
-          persist-credentials: false
-
-      - name: Detect backend-relevant changes
-        id: changes
-        uses: ./.github/actions/detect-backend-changes
-
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: "3.12"
-
-      - name: Set up uv
-        uses: ./.github/actions/setup-uv-with-retries
-        with:
-          version: "0.10.9"
-
-      - name: Cache uv dependencies
-        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
-        with:
-          path: |
-            ~/.cache/uv
-            .venv
-          key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-uv-
-
-      - name: Install dependencies
-        if: steps.changes.outputs.decision != 'skip'
-        run: |
-          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
-
-      - name: Generate Prisma client
-        if: steps.changes.outputs.decision != 'skip'
-        env:
-          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
-        run: |
-          uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
-
-      - name: Run tests - ${{ matrix.test-group.name }}
-        if: steps.changes.outputs.decision != 'skip'
-        env:
-          TEST_PATH: ${{ matrix.test-group.path }}
-        run: |
-          uv run --no-sync pytest ${TEST_PATH} \
-            --tb=short -vv \
-            --maxfail=10 \
-            -n 2 \
-            --reruns 1 \
-            --reruns-delay 1 \
-            --dist=loadscope \
-            --durations=20

.github/workflows/weekly_load_anomaly.yml

@@ -51,9 +51,10 @@ jobs:
         run: |
           .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy
 
+      - name: Cache Prisma binaries
+        uses: ./.github/actions/cache-prisma-binaries
+
       - name: Generate Prisma client
-        env:
-          PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache
         run: |
           uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma
 

.gitignore

@@ -1,5 +1,6 @@
 .python-version
 .venv
+.venv-typecheck
 .venv_policy_test
 .env
 .claude

CLAUDE.md

@@ -1,4 +1,12 @@
-Do not write any comments (existing comments can stay) unless explicitly asked to in a user (not system) prompt
+Do not write comments unless they are any of:
+- absolutely necessary to explain some very complex business logic (in which case, keep it concise and clear)
+- used as an input for tools to read and act on. For example:
+  - entries in `.git-blame-ignore-revs` saying which commit is excluded from git blame
+  - a lint or type checker suppression like `# mutable-ok` or `# pyright: ignore[reportArgumentType]  # <reason>` when introducing a truly unavoidable violation
+- a TODO or FIXME
+  - Not great to have those, but if it's unavoidable, make sure to include a strong, concise reason for why it's there or, better yet, link to a GitHub issue for the follow-up work
+
+Explanation: The point of this rule is to keep out AI slop comments. AI writes way too many and way too verbose comments. Code comments are, in a way, a violation of DRY code. You must update logic in two locations to change the code, and "hard to change" is literally the definition of tech debt. We should instead aim to write code that is intuitive and clear, even at a glance, to the reader, being both easy to maintain and high performance
 
 Don't assume that the existing code is correct or the right way of doing things / good coding patterns. In fact, there are a lot of bad coding practices, overly complex code, code smells, etc. If something doesn't look right, speak up. Feel free to break existing patterns or question weird existing code to make new code high quality, as in:
 
@@ -9,7 +17,7 @@ Don't assume that the existing code is correct or the right way of doing things
 - easy to maintain/change
 - modern
 
-In that order of importance
+In descending order of importance
 
 When adding new features, add meaningful tests. Don't add tests that don't check anything substantial and is there just to make the code coverage pass. Yes, code coverage is important, but I'd rather have no signal whether the code is working than tests that don't fail when code is broken. The goal is to have tests that would fail before the feature was added/if the code was mutated in a way that breaks the feature and succeed only when the feature is fully working. I should run mutation testing and see > 90% kill rate
 
@@ -21,7 +29,9 @@ End-to-end tests belong in `tests/e2e/` and must follow the harness conventions
 
 When creating PRs, don't set base to `main`. `litellm_internal_staging` is the default base branch and serves that purpose for both internal and external / OSS contributions
 
-When writing a PR body, treat the comments and imperative instructions inside @.github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule
+When writing a PR body, treat the comments and imperative instructions inside .github/pull_request_template.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/pull_request_template.md from disk before writing a PR body to make sure you see every comment rule
+
+Same applies for filing bug reports and feature requests, with .github/ISSUE_TEMPLATE/bug_report.yml and .github/ISSUE_TEMPLATE/feature_request.yml, respectively
 
 If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just leave the section blank
 
@@ -29,7 +39,7 @@ Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We pref
 
 If you ever make public-facing PR descriptions, comments, issues, commit messages, etc., always follow these guidelines to sound less AI-y:
 - don't use emojis
-- don't use "—". Instead, reach for ",", ".", conjunction words, ":", ";", etc. in descending order of preference: vary among them, weighted toward the front of the list, and skip "," where it would cause a comma splice or the sentence is getting long. Overusing any one of them, ";" especially, also feels AI-y
+- don't use "—". Instead, reach for ",", ".", conjunction words, ":", ";", etc. in descending order of preference: vary among them, weighted toward the front of the list, and skip "," where it would cause a comma splice or the sentence is getting long. Overusing any one of them, ";" especially, also feels AI-y. A word cap does not penalize you for adding more sentences: when writing under tight word budgets, prefer a period split or a conjunction over ";", and keep to at most one ";" per message
 - don't use the pattern "It's not X, it's Y", "You're not X, you're Y", etc.
 - don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose
 - don't add a trailing "." at the end of paragraphs (just like this file). That means every paragraph, not just the last one (of the markdown file, PR description, GitHub comment, etc.). Rule of thumb: if you're adding new line(s) before the next sentence, don't add a "."
@@ -41,11 +51,13 @@ Python max line length is 120, not 88
 
 When you fix violations gated by `ruff-strict-budget.json`, `type-discipline-budget.json`, or `basedpyright-code-budget.json`, run `make lint-budget-update` and commit the lowered limits so the ceilings ratchet down instead of leaving stale headroom. It measures the working tree, so it must contain exactly the fixes you're committing
 
-`make pre-commit` always saves its complete output to a per-worktree log file and prints that path as its first and last output lines. To inspect a run, read or grep that log instead of re-running the multi-minute checks just to see a different slice, and re-run only after the working tree actually changed
+`make check` (f.k.a. `make pre-commit`, which still works identically as an alias) saves its complete output to a log file in .git (overwriting previous logs) and prints that path as its first and last output lines. To inspect a run, read or grep that log instead of re-running the multi-minute checks just to see a different slice
+
+`make check`, `make lint`, `scripts/pre_commit_lint.sh`, and the standalone budget gates (`scripts/ruff_strict_gate.py`, `scripts/type_discipline_gate.py`, `scripts/type_check_gate.py`) each hold one of 2 machine-wide slots, so when other sessions or worktrees on the same box are already running heavy work, yours prints "all N machine-wide slots are busy; queueing" and then stays quiet until a slot frees. Give the command a long timeout and let it wait rather than killing it, retrying it, or assuming it hung. Don't change the # of machine-wide slots or make it unlimited by setting `LITELLM_GATE_SLOTS=0`
 
 If you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and pushing `reportAny` / `reportExplicitAny` closer to their basedpyright ceilings, just validate it in the caller with Pydantic (a model or `TypeAdapter` that returns the typed thing or raises will do) and then pass the now typed variable in
 
-If you get an LIT001 or LIT002 fail, refactor the code to follow functional programming best practices rather than introducing mutable data structures. For example, build values in one shot with comprehensions or generators wrapped in `tuple()` / `frozenset()` instead of seeding an empty `list`/`dict`/`set` and mutating it over time. Ideally, `# mutable-ok` is never used; reach for it only as a genuine last resort when an immutable rewrite is truly impossible, and always pair it with a real reason
+If you get an LIT001 or LIT002 fail, refactor the code to follow functional programming best practices rather than introducing mutable data structures. For example, build values in one shot with comprehensions or generators wrapped in `tuple()` / `MappingProxyType()` / `frozenset()` instead of seeding an empty `list`/`dict`/`set` and mutating it over time. Ideally, `# mutable-ok` is never used; reach for it only as a genuine last resort when an immutable rewrite is truly impossible, and always pair it with a real reason
 
 Every lint or type suppression must name the exact rule inside brackets and carry a reason comment, e.g. `# pyright: ignore[reportArgumentType]  # stubs lack async overload` or `# noqa: TID251  # <reason>`. `# type: ignore` is banned (LIT009): pyrightconfig.json sets `enableTypeIgnoreComments` to false, so it silently does nothing
 
@@ -59,7 +71,7 @@ Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages
 
 When working on a PR, keep the PR description in sync with new commits being made
 
-Replies/rebuttals to AI PR review bots must be 15-25 word human-readable replies
+All GitHub comments must be human-readable and 15-25 words max
 
 Monkeypatching attributes of a class to do testing is an anti-pattern. Prefer dependency-injecting things into classes. That way, at unit test time, you can pass a mocked dependency in
 
@@ -72,8 +84,9 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega
 - Composition over inheritance
 - Never-nester: early returns over deep nesting
 - Don't throw; model failures as values (One function (e.g., raise_public) maps error union to existing public exception contracts via exhaustive match + assert_never)
-- No mutation; don't reassign variables, global or local. Instead of mutable lists and dicts, prefer tuples, frozen dataclasses (with slots=True), etc.
-  - Annotate every variable with `: Final` (LIT010). Unpacking and walrus targets cannot carry the annotation, so they are implicitly final. Don't rebind them. Never rebind or mutate function parameters (LIT011); `self`/`cls` attribute stores are the exception. If rebinding or in-place mutation is truly unavoidable, suppress with `# rebind-ok: <reason>` explaining why
+- No mutation; don't reassign variables, global or local. Instead of mutable lists and dicts, prefer tuples, frozen dataclasses (with slots=True), `MappingProxyType`, etc.
+  - Annotate every variable with `: Final` (LIT010). Unpacking and walrus targets cannot carry the annotation, so they are implicitly final. Don't rebind them. Never rebind or mutate function parameters (LIT011); `self`/`cls` attribute stores are the exception. If rebinding or in-place mutation is truly unavoidable, suppress with `# rebind-ok: <reason>`
+  - Qualify every TypedDict field with `ReadOnly[...]` (LIT012), which nests freely with `Required` / `NotRequired` / `Annotated` in any order. If making the key writable is truly unavoidable, suppress with `# writable-ok: <reason>`
 - Use dependency injection
 - Fully typed; no `Any` or coarse types like `dict[str, Any]` or just `dict`. Every function parameter must be strongly typed
 - Use tagged unions + match

Makefile

@@ -4,11 +4,11 @@
 .PHONY: help test test-unit test-unit-llms test-unit-proxy-guardrails test-unit-proxy-core test-unit-proxy-misc \
 	test-unit-integrations test-unit-core-utils test-unit-other test-unit-root \
 	test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \
-	info lint lint-dev lint-checks format \
+	info lint lint-inner lint-dev lint-checks format \
 	lint-basedpyright lint-e2e-basedpyright lint-basedpyright-budget-update lint-type-discipline lint-type-discipline-budget-update \
 	lint-ruff-budget lint-ruff-budget-update lint-budget-update lint-gate \
 	install-dev install-proxy-dev install-test-deps install-hooks \
-	install-helm-unittest check-circular-imports check-import-safety pre-commit \
+	install-helm-unittest check-circular-imports check-import-safety check check-inner pre-commit \
 	lint-install lint-fetch-base bootstrap
 
 # Default target
@@ -22,7 +22,8 @@ help:
 	@echo "  make install-test-deps  - Install the full local test environment"
 	@echo "  make install-helm-unittest - Install helm unittest plugin"
 	@echo "  make install-hooks      - Install git hooks (Conventional Commits + Branches)"
-	@echo "  make pre-commit         - Run CI-equivalent lint on staged files (run before committing)"
+	@echo "  make check              - Run CI-equivalent lint on staged files, or on the diff vs the base branch when nothing is staged"
+	@echo "  make pre-commit         - Legacy alias for make check"
 	@echo "  make format             - Apply ruff format code formatting"
 	@echo "  make format-check       - Check ruff format code formatting (matches CI)"
 	@echo "  make lint               - Run all linting (Ruff, basedpyright, format check, circular imports, import safety)"
@@ -51,10 +52,17 @@ help:
 	@echo "  make test-proxy-unit-b  - Run proxy_unit_tests (p-z, ~28 files)"
 	@echo "  make test-integration   - Run integration tests"
 	@echo "  make test-unit-helm     - Run helm unit tests"
+	@echo ""
+	@echo "Heavy targets (check, lint) queue for LITELLM_GATE_SLOTS machine-wide"
+	@echo "slots (default 2; 0 disables) so parallel sessions don't thrash one machine."
 
 UV := uv
 UV_RUN := $(UV) run --no-sync
 
+# Machine-wide slot queue for the heavy targets below; python3 + stdlib only, so
+# it runs before any venv exists. See scripts/gate_slot_lock.py.
+GATE_SLOT_LOCK := python3 scripts/gate_slot_lock.py
+
 LINT_DEP_INSTALL ?= install-dev
 LINT_E2E_DEP_INSTALL ?= lint-install
 LINT_DEP_BASE ?= lint-fetch-base
@@ -72,6 +80,8 @@ info:
 install-dev:
 	$(UV) sync --inexact --frozen
 
+# Deliberately unqueued: provisioning is I/O bound, so it doesn't need one of the
+# machine-wide slots the CPU-bound gates below share.
 bootstrap:
 	$(UV) sync --inexact --frozen --extra proxy --group proxy-dev --group e2e-dev
 	$(UV_RUN) python scripts/prisma_generate_if_needed.py
@@ -124,10 +134,10 @@ lint-fetch-base:
 	git fetch origin litellm_internal_staging
 
 # Mirror test-linting.yml's lint job environment: the proxy-dev group plus a generated
-# Prisma client, so basedpyright resolves the same modules CI does (without the generated
-# client the DB wrappers typed against it degrade to Unknown, drifting the budget from
-# CI's). --inexact tops up the venv instead of pruning the proxy extras gen:api and the
-# running proxy need.
+# Prisma client, so `basedpyright tests/e2e` resolves the same modules CI does. The
+# budget gate itself no longer measures here (scripts/type_check_gate.py provisions its
+# own .venv-typecheck). --inexact tops up the venv instead of pruning the proxy extras
+# gen:api and the running proxy need.
 lint-install:
 	$(UV) sync --inexact --frozen --group proxy-dev --group e2e-dev
 	$(UV_RUN) python scripts/prisma_generate_if_needed.py
@@ -228,7 +238,10 @@ check-import-safety: $(LINT_DEP_INSTALL)
 # does (merge-base with origin/litellm_internal_staging). Setup (env sync, Prisma client,
 # base fetch) runs once up front; the checks themselves are independent, so a sub-make
 # fans them out with -j and the fast ones finish under basedpyright's shadow.
-lint: lint-install lint-fetch-base
+lint:
+	@$(GATE_SLOT_LOCK) $(MAKE) lint-inner
+
+lint-inner: lint-install lint-fetch-base
 	$(MAKE) -j $(LINT_JOBS) $(LINT_OUTPUT_SYNC) LINT_DEP_INSTALL= LINT_E2E_DEP_INSTALL= LINT_DEP_BASE= lint-checks
 
 lint-checks: lint-format-check-changed lint-ruff lint-gate lint-type-discipline lint-basedpyright lint-e2e-basedpyright check-circular-imports check-import-safety
@@ -236,13 +249,23 @@ lint-checks: lint-format-check-changed lint-ruff lint-gate lint-type-discipline
 # Faster linting for local development (only checks changed code)
 lint-dev: lint-format-changed check-circular-imports check-import-safety
 
-# Run the gating CI checks against your staged files right before committing. Mirrors
+# Run the gating CI checks against your changes. Scopes to staged files when anything
+# is staged (warning about changed files left unstaged); with nothing staged it falls
+# back to the working tree's diff against the merge base with the base branch, so a
+# fresh merge commit or an unstaged working tree still gets checked. Mirrors
 # test-linting.yml (Python), test-litellm-ui-build.yml's frontend-lint (dashboard), and
-# check-ui-api-types.yml (API-type drift), skipping any whose files you didn't stage.
+# check-ui-api-types.yml (API-type drift), skipping any whose files aren't in scope.
 # Not auto-installed as a git hook so it never slows an unrelated human commit.
-pre-commit: bootstrap
+check:
+	@$(GATE_SLOT_LOCK) $(MAKE) check-inner
+
+check-inner: bootstrap
 	./scripts/pre_commit_lint.sh
 
+pre-commit:
+	@echo "make pre-commit is a legacy alias; use make check" >&2
+	@$(MAKE) check
+
 # Testing targets
 test: install-test-deps
 	$(UV_RUN) pytest tests/

backend/routes/allowlist.py

@@ -146,11 +146,13 @@ BACKEND_EXACT_PATHS: frozenset[str] = frozenset(
         "/docs/oauth2-redirect",
         "/redoc",
         "/fallback/login",
+        "/mcp",  # bare spelling of the aggregate MCP endpoint; /mcp/ prefix covers the rest
     }
 )
 
 BACKEND_MOUNT_PATHS: frozenset[str] = frozenset(
     {
         "/swagger",  # API documentation static assets belong to the backend
+        "/mcp",  # lazily-mounted MCP sub-app serves on the backend component
     }
 )

basedpyright-code-budget.json

@@ -1,36 +1,36 @@
 {
   "reportAny": {
-    "limit": 29204
+    "limit": 22344
   },
   "reportArgumentType": {
-    "limit": 2635
+    "limit": 2578
   },
   "reportAssignmentType": {
-    "limit": 329
+    "limit": 323
   },
   "reportAttributeAccessIssue": {
-    "limit": 516
+    "limit": 488
   },
   "reportCallIssue": {
-    "limit": 123
+    "limit": 114
   },
   "reportConstantRedefinition": {
     "limit": 40
   },
   "reportDeprecated": {
-    "limit": 215
+    "limit": 213
   },
   "reportDuplicateImport": {
     "limit": 19
   },
   "reportExplicitAny": {
-    "limit": 9227
+    "limit": 6991
   },
   "reportFunctionMemberAccess": {
     "limit": 7
   },
   "reportGeneralTypeIssues": {
-    "limit": 157
+    "limit": 154
   },
   "reportIncompatibleMethodOverride": {
     "limit": 56
@@ -54,10 +54,10 @@
     "limit": 0
   },
   "reportMissingParameterType": {
-    "limit": 5850
+    "limit": 5681
   },
   "reportMissingTypeArgument": {
-    "limit": 15833
+    "limit": 15609
   },
   "reportMissingTypeStubs": {
     "limit": 40
@@ -72,7 +72,7 @@
     "limit": 0
   },
   "reportOptionalMemberAccess": {
-    "limit": 1078
+    "limit": 1061
   },
   "reportOptionalOperand": {
     "limit": 0
@@ -84,52 +84,52 @@
     "limit": 56
   },
   "reportPrivateUsage": {
-    "limit": 1825
+    "limit": 1824
   },
   "reportRedeclaration": {
     "limit": 8
   },
   "reportReturnType": {
-    "limit": 218
+    "limit": 213
   },
   "reportTypedDictNotRequiredAccess": {
-    "limit": 27
+    "limit": 26
   },
   "reportUndefinedVariable": {
     "limit": 0
   },
   "reportUnknownArgumentType": {
-    "limit": 45242
+    "limit": 44709
   },
   "reportUnknownLambdaType": {
-    "limit": 113
+    "limit": 112
   },
   "reportUnknownMemberType": {
-    "limit": 40340
+    "limit": 39154
   },
   "reportUnknownParameterType": {
-    "limit": 20293
+    "limit": 19947
   },
   "reportUnknownVariableType": {
-    "limit": 31796
+    "limit": 30772
   },
   "reportUnnecessaryCast": {
-    "limit": 122
+    "limit": 117
   },
   "reportUnnecessaryComparison": {
-    "limit": 703
+    "limit": 699
   },
   "reportUnnecessaryContains": {
     "limit": 5
   },
   "reportUnnecessaryIsInstance": {
-    "limit": 865
+    "limit": 851
   },
   "reportUntypedBaseClass": {
-    "limit": 72
+    "limit": 0
   },
   "reportUntypedFunctionDecorator": {
-    "limit": 33
+    "limit": 27
   },
   "reportUnusedClass": {
     "limit": 23
@@ -138,7 +138,7 @@
     "limit": 139
   },
   "reportUnusedImport": {
-    "limit": 555
+    "limit": 545
   },
   "reportUnusedVariable": {
     "limit": 146

ci_cd/generate_model_prices_schema.py

@@ -96,6 +96,7 @@ ARRAY_KEYS: dict[str, JsonSchema] = {
                 "output_cost_per_token": NONNEG_NUMBER,
                 "output_cost_per_reasoning_token": NONNEG_NUMBER,
                 "cache_read_input_token_cost": NONNEG_NUMBER,
+                "cache_creation_input_token_cost": NONNEG_NUMBER,
                 "input_cost_per_query": NONNEG_NUMBER,
             },
             "additionalProperties": False,

enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py

@@ -99,6 +99,7 @@ class BaseEmailLogger(CustomLogger):
         email_html_content = USER_INVITATION_EMAIL_TEMPLATE.format(
             email_logo_url=email_params.logo_url,
             recipient_email=email_params.recipient_email,
+            invitation_link=email_params.base_url,
             base_url=email_params.base_url,
             email_support_contact=email_params.support_contact,
             email_footer=email_params.signature,
@@ -826,10 +827,15 @@ class BaseEmailLogger(CustomLogger):
         """
         # Early validation
         if not user_id:
-            verbose_proxy_logger.debug("No user_id provided for invitation link")
+            verbose_proxy_logger.warning(
+                "No user_id provided for invitation link. Email will link to base URL instead of onboarding page"
+            )
             return base_url
 
         if not await self._is_prisma_client_available():
+            verbose_proxy_logger.warning(
+                "Prisma client not available. Email will link to base URL instead of onboarding page"
+            )
             return base_url
 
         # Wait for any concurrent invitation creation to complete
@@ -839,11 +845,15 @@ class BaseEmailLogger(CustomLogger):
         invitation = await self._get_or_create_invitation(user_id)
         if not invitation:
             verbose_proxy_logger.warning(
-                f"Failed to get/create invitation for user_id: {user_id}"
+                f"Failed to get/create invitation for user_id: {user_id}. Email will link to base URL instead of onboarding page"
             )
             return base_url
 
-        return self._construct_invitation_link(invitation.id, base_url)
+        invitation_link = self._construct_invitation_link(invitation.id, base_url)
+        verbose_proxy_logger.info(
+            f"Successfully created invitation link for user_id: {user_id}"
+        )
+        return invitation_link
 
     async def _is_prisma_client_available(self) -> bool:
         """Check if Prisma client is available"""
@@ -921,7 +931,9 @@ class BaseEmailLogger(CustomLogger):
 
         # http://localhost:4000/ui/onboarding?invitation_id=7a096b3a-37c6-440f-9dd1-ba22e8043f6b
         """
-        return f"{base_url}/ui/onboarding?invitation_id={invitation_id}"
+        base_url = base_url.rstrip("/")
+        invitation_link = f"{base_url}/ui/onboarding?invitation_id={invitation_id}"
+        return invitation_link
 
     async def send_email(
         self,

enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py

@@ -3,7 +3,8 @@ Polls LiteLLM_ManagedObjectTable to check if the batch job is complete, and if t
 """
 
 from datetime import datetime, timedelta, timezone
-from typing import TYPE_CHECKING, List, Optional, Tuple
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Any, Dict, Final, List, Optional, Tuple
 
 from litellm._logging import verbose_proxy_logger
 from litellm._uuid import uuid
@@ -23,6 +24,15 @@ if TYPE_CHECKING:
 
 CHECK_BATCH_COST_USER_AGENT = "LiteLLM Proxy/CheckBatchCost"
 
+TERMINAL_MANAGED_OBJECT_STATUSES: Final[Tuple[str, ...]] = (
+    "completed",
+    "complete",
+    "failed",
+    "expired",
+    "cancelled",
+    "stale_expired",
+)
+
 
 class CheckBatchCost:
     def __init__(
@@ -42,12 +52,43 @@ class CheckBatchCost:
         # Cached after the first poll cycle. Once we know the column is absent we skip
         # the guaranteed-failing primary query on every subsequent cycle.
         self._has_batch_processed_column: bool = True
+        self.batch_processed_support_confirmed: bool = False
 
-    async def _get_user_info(self, batch_id, user_id) -> dict:
+    @staticmethod
+    def _is_missing_batch_processed_column_error(err: Exception) -> bool:
+        message: Final = str(err).lower()
+        return "batch_processed" in message or "unknown column" in message or "does not exist" in message
+
+    async def confirm_batch_processed_support(self) -> None:
+        """
+        Probe the batch_processed column before the proxy serves traffic, so the retrieve
+        path never sees an unconfirmed poller on a schema that has the column and accounts
+        inline for a batch the first poll cycle then accounts again.
+        """
+        try:
+            await self.prisma_client.db.litellm_managedobjecttable.find_first(
+                where={"file_purpose": "batch", "batch_processed": False}
+            )
+        except Exception as probe_err:
+            if not self._is_missing_batch_processed_column_error(probe_err):
+                verbose_proxy_logger.debug(
+                    f"CheckBatchCost: batch_processed probe failed, the poll cycle will confirm support: {probe_err}"
+                )
+                return
+            self._has_batch_processed_column = False
+            verbose_proxy_logger.warning("CheckBatchCost: batch_processed column not found, querying without it")
+            return
+        self.batch_processed_support_confirmed = True
+
+    async def _get_user_info(self, batch_id: str, user_id: Optional[str]) -> Dict[str, Any]:
         """
         Look up user email and key alias by user_id for enriching the S3 callback metadata.
         Returns a dict with user_api_key_user_email and user_api_key_alias (both may be None).
+        Returns an empty dict when user_id is None: batches created by a team or service
+        account key carry no user id, and find_unique(where={"user_id": None}) raises.
         """
+        if not user_id:
+            return {}
         try:
             user_row = await self.prisma_client.db.litellm_usertable.find_unique(
                 where={"user_id": user_id}
@@ -62,17 +103,77 @@ class CheckBatchCost:
             verbose_proxy_logger.error(f"CheckBatchCost: could not look up user {user_id} for batch {batch_id}: {e}")
             return {}
 
+    async def _get_key_alias(self, batch_id: str, api_key: str | None) -> str | None:
+        """Resolve the creating virtual key's alias from its hashed token."""
+        if not api_key:
+            return None
+        try:
+            key_row = await self.prisma_client.db.litellm_verificationtoken.find_unique(
+                where={"token": api_key}
+            )
+            return getattr(key_row, "key_alias", None) if key_row is not None else None
+        except Exception as e:
+            verbose_proxy_logger.error(f"CheckBatchCost: could not look up key alias for batch {batch_id}: {e}")
+            return None
+
+    async def _get_team_alias(self, team_id: str | None) -> str | None:
+        """Resolve a team's alias from its id."""
+        if not team_id:
+            return None
+        try:
+            team_row = await self.prisma_client.db.litellm_teamtable.find_unique(
+                where={"team_id": team_id}
+            )
+            return getattr(team_row, "team_alias", None) if team_row is not None else None
+        except Exception as e:
+            verbose_proxy_logger.error(f"CheckBatchCost: could not look up team alias for team {team_id}: {e}")
+            return None
+
+    async def _build_creator_attribution_metadata(
+        self, job: "LiteLLM_ManagedObjectTable", batch_id: str
+    ) -> Dict[str, Any]:
+        """
+        Rebuild the spend-tracking metadata for the key, team, and tags that created the
+        batch so the batch-cost spend log is attributed the same way a non-batch request
+        is. Rows created before api_key and request_tags were persisted carry only
+        created_by and team_id, and fall back to those. A named creating key owns
+        user_api_key_alias; when it has no alias, or the key has since been rotated or
+        deleted, the field keeps the creating user's alias that _get_user_info filled in,
+        because a resolvable name is more useful on the spend row than a null.
+        """
+        api_key = getattr(job, "api_key", None)
+        team_id = getattr(job, "team_id", None)
+        request_tags = getattr(job, "request_tags", None)
+
+        metadata: Dict[str, Any] = {
+            "user_api_key_user_id": job.created_by,
+            "user_api_key": api_key,
+            "user_api_key_team_id": team_id,
+            **(await self._get_user_info(batch_id, job.created_by)),
+        }
+
+        key_alias = await self._get_key_alias(batch_id, api_key)
+        if key_alias is not None:
+            metadata["user_api_key_alias"] = key_alias
+        team_alias = await self._get_team_alias(team_id)
+        if team_alias is not None:
+            metadata["user_api_key_team_alias"] = team_alias
+        if isinstance(request_tags, list) and request_tags:
+            metadata["tags"] = [tag for tag in request_tags if isinstance(tag, str)]
+
+        return metadata
+
     async def _cleanup_stale_managed_objects(self) -> None:
         """
         Mark managed objects older than MANAGED_OBJECT_STALENESS_CUTOFF_DAYS days
         in non-terminal states as 'stale_expired'. These will never complete and
         should not be polled.
         """
-        cutoff = datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS)
-        result = await self.prisma_client.db.litellm_managedobjecttable.update_many(
+        cutoff: Final = datetime.now(timezone.utc) - timedelta(days=MANAGED_OBJECT_STALENESS_CUTOFF_DAYS)
+        result: Final = await self.prisma_client.db.litellm_managedobjecttable.update_many(
             where={
                 "file_purpose": "batch",
-                "status": {"not_in": ["completed", "complete", "failed", "expired", "cancelled", "stale_expired"]},
+                "status": {"not_in": list(TERMINAL_MANAGED_OBJECT_STATUSES)},
                 "created_at": {"lt": cutoff},
             },
             data={"status": "stale_expired"},
@@ -83,6 +184,26 @@ class CheckBatchCost:
                 f"(older than {MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days) as stale_expired"
             )
 
+        if not self._has_batch_processed_column:
+            return
+
+        # A row already in a terminal status is never rewritten by the sweep above, so
+        # without this it keeps a poll-page slot forever and starves newer batches.
+        retired: Final = await self.prisma_client.db.litellm_managedobjecttable.update_many(
+            where={
+                "file_purpose": "batch",
+                "batch_processed": False,
+                "status": {"in": ["complete", "completed"]},
+                "created_at": {"lt": cutoff},
+            },
+            data={"batch_processed": True},
+        )
+        if retired > 0:
+            verbose_proxy_logger.warning(
+                f"CheckBatchCost: gave up on {retired} completed managed objects older than "
+                f"{MANAGED_OBJECT_STALENESS_CUTOFF_DAYS} days that were never costed"
+            )
+
     async def _fallback_find_jobs(self) -> list:
         """Query batch jobs without the batch_processed filter (for older schemas)."""
         return await self.prisma_client.db.litellm_managedobjecttable.find_many(
@@ -103,6 +224,68 @@ class CheckBatchCost:
             order={"created_at": "asc"},
         )
 
+    async def _retire_job(self, job: "LiteLLM_ManagedObjectTable", reason: str) -> None:
+        """
+        Take a row that can never be costed out of the poll page. Leaving it selectable
+        would burn one of the MAX_OBJECTS_PER_POLL_CYCLE slots on every future cycle, and
+        once enough such rows accumulate no newer batch is ever reached. Older schemas
+        without batch_processed can only be excluded through the status filter.
+        """
+        data: Final = (
+            {"batch_processed": True}
+            if self._has_batch_processed_column
+            else {"status": "stale_expired"}
+        )
+        try:
+            await self.prisma_client.db.litellm_managedobjecttable.update(
+                where={"id": job.id},
+                data=data,
+            )
+        except Exception as db_err:
+            verbose_proxy_logger.error(
+                f"CheckBatchCost: failed to retire uncostable job {job.id} ({reason}): {db_err}"
+            )
+            return
+        verbose_proxy_logger.warning(
+            f"CheckBatchCost: job {job.id} can never be costed ({reason}), "
+            "so it will no longer be polled"
+        )
+
+    @staticmethod
+    def _has_unified_id_without_model(job: "LiteLLM_ManagedObjectTable") -> bool:
+        """A unified id that decodes but carries no model_id can never be routed."""
+        from litellm.proxy.openai_files_endpoints.common_utils import (
+            convert_b64_uid_to_unified_uid,
+            get_model_id_from_unified_batch_id,
+        )
+
+        decoded: Final = convert_b64_uid_to_unified_uid(job.unified_object_id)
+        return (
+            decoded != job.unified_object_id
+            and get_model_id_from_unified_batch_id(decoded) is None
+        )
+
+    @staticmethod
+    def _is_batch_gone_at_provider(error: Exception, batch_id: str) -> bool:
+        """
+        A 404 naming the batch means the provider dropped its record of it, so no later
+        retrieve can ever succeed. A 404 about anything else, a renamed Azure deployment
+        or a fallback deployment that never saw this batch, is still fixable in config, so
+        it keeps retrying.
+        """
+        import openai
+
+        from litellm.exceptions import NotFoundError
+
+        return isinstance(error, (NotFoundError, openai.NotFoundError)) and batch_id in str(error)
+
+    def _batch_deployment_exists(self, model_id: str) -> bool:
+        """A 404 only proves the batch is gone when it came from the batch's own
+        deployment. Once that deployment leaves the router, default fallbacks can
+        silently send the retrieve to a provider that never saw the batch, so its
+        404 must not retire the row; the staleness sweep bounds it instead."""
+        return self.llm_router.get_deployment(model_id=model_id) is not None
+
     @staticmethod
     def _record_error(
         prom_logger: Optional["PrometheusLogger"], error_type: str
@@ -382,6 +565,7 @@ class CheckBatchCost:
         credentials = self.llm_router.get_deployment_credentials_with_provider(model_id) or {}
         _file_content = await afile_content(
             file_id=raw_output_file_id,
+            _litellm_internal_model_credentials=MappingProxyType(dict(credentials)),
             **credentials,
         )
 
@@ -485,9 +669,6 @@ class CheckBatchCost:
             function_id=str(uuid.uuid4()),
         )
 
-        creator_user_id = job.created_by
-        user_info = await self._get_user_info(batch_id, job.created_by)
-
         logging_obj.update_environment_variables(
             litellm_params={
                 # set the user-agent header so that S3 callback consumers can easily identify CheckBatchCost callbacks
@@ -496,10 +677,7 @@ class CheckBatchCost:
                         "user-agent": CHECK_BATCH_COST_USER_AGENT,
                     }
                 },
-                "metadata": {
-                    "user_api_key_user_id": creator_user_id,
-                    **user_info,
-                },
+                "metadata": await self._build_creator_attribution_metadata(job, batch_id),
             },
             optional_params={},
         )
@@ -573,8 +751,9 @@ class CheckBatchCost:
                     take=MAX_OBJECTS_PER_POLL_CYCLE,
                     order={"created_at": "asc"},
                 )
+                self.batch_processed_support_confirmed = True
             except Exception as query_err:
-                if "batch_processed" not in str(query_err).lower() and "unknown column" not in str(query_err).lower() and "does not exist" not in str(query_err).lower():
+                if not self._is_missing_batch_processed_column_error(query_err):
                     raise
                 # Permanent schema gap — cache the result so future cycles skip straight to fallback
                 self._has_batch_processed_column = False
@@ -587,6 +766,8 @@ class CheckBatchCost:
         for job in jobs:
             routing = self._resolve_job_routing(job, prom_logger)
             if routing is None:
+                if self._has_unified_id_without_model(job):
+                    await self._retire_job(job, "unified object id has no model id")
                 continue
             model_id, batch_id = routing
 
@@ -609,11 +790,13 @@ class CheckBatchCost:
                 )
                 if prom_logger:
                     prom_logger.record_check_batch_cost_error("provider_retrieval_error")
+                if self._is_batch_gone_at_provider(e, batch_id) and self._batch_deployment_exists(model_id):
+                    await self._retire_job(job, f"batch {batch_id} no longer exists at the provider")
                 continue
 
             ## RETRIEVE THE BATCH JOB OUTPUT FILE
             if (
-                response.status == "completed"
+                response.status in ("completed", "complete", "expired")
                 and response.output_file_id is not None
             ):
                 try:
@@ -640,7 +823,7 @@ class CheckBatchCost:
                 # mark the job as complete
                 try:
                     update_data: dict = {
-                        "status": "complete",
+                        "status": response.status if response.status != "completed" else "complete",
                         "file_object": response.model_dump_json(),
                     }
                     if self._has_batch_processed_column:
@@ -654,8 +837,28 @@ class CheckBatchCost:
                         f"CheckBatchCost: failed to mark job {job.id} complete in DB: {db_err}"
                     )
 
-            elif response.status in ("failed", "expired", "cancelled"):
+            elif response.status in (
+                "completed",
+                "complete",
+                "failed",
+                "expired",
+                "cancelled",
+            ):
                 try:
+                    from litellm.proxy.openai_files_endpoints.common_utils import (
+                        _is_base64_encoded_unified_file_id,
+                        ensure_batch_response_managed_file_ids,
+                    )
+
+                    response.id = job.unified_object_id
+                    await ensure_batch_response_managed_file_ids(
+                        response=response,
+                        managed_files_obj=self.proxy_logging_obj.get_proxy_hook("managed_files"),
+                        prisma_client=self.prisma_client,
+                        verbose_proxy_logger=verbose_proxy_logger,
+                        db_batch_object=job,
+                        unified_batch_id=_is_base64_encoded_unified_file_id(job.unified_object_id),
+                    )
                     update_data = {
                         "status": response.status,
                         "file_object": response.model_dump_json(),

enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py

@@ -1,10 +1,10 @@
 """
 Polls LiteLLM_ManagedObjectTable to check if the response is complete.
-Cost tracking is handled automatically by litellm.aget_responses().
+Cost tracking is handled automatically by the get-responses call.
 """
 
 from datetime import datetime, timedelta, timezone
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Dict, Optional, cast
 
 import litellm
 from litellm._logging import verbose_proxy_logger
@@ -13,11 +13,15 @@ from litellm.constants import (
     MAX_OBJECTS_PER_POLL_CYCLE,
     STALE_OBJECT_CLEANUP_BATCH_SIZE,
 )
+from litellm.responses.utils import ResponsesAPIRequestUtils
+from litellm.types.llms.openai import ResponsesAPIResponse
 
 if TYPE_CHECKING:
     from litellm.proxy.utils import PrismaClient, ProxyLogging
     from litellm.router import Router
 
+TERMINAL_RESPONSE_STATUSES = frozenset({"completed", "failed", "cancelled", "incomplete"})
+
 
 class CheckResponsesCost:
     def __init__(
@@ -33,6 +37,28 @@ class CheckResponsesCost:
         self.prisma_client: PrismaClient = prisma_client
         self.llm_router: Router = llm_router
 
+    async def _get_response(
+        self,
+        response_id: str,
+        litellm_metadata: Dict[str, str],
+    ) -> ResponsesAPIResponse:
+        """Fetch the upstream response, using deployment credentials when available.
+
+        LiteLLM-encoded response IDs carry the ``model_id`` of the deployment that
+        served the original request, so routing through ``llm_router`` applies that
+        deployment's ``api_base`` / ``api_key`` / ``api_version``, exactly like
+        ``GET /v1/responses/{id}`` does. ``litellm.aget_responses`` on its own only
+        sees provider env vars, so it fails for every deployment whose credentials
+        live in the config; the row then never leaves ``queued``.
+        """
+        model_id: Optional[str] = ResponsesAPIRequestUtils.get_model_id_from_response_id(response_id)
+        if model_id is None or self.llm_router.get_deployment(model_id=model_id) is None:
+            return await litellm.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata)
+        router_response = await self.llm_router.aget_responses(
+            response_id=response_id, litellm_metadata=litellm_metadata
+        )
+        return cast(ResponsesAPIResponse, router_response)
+
     async def _expire_stale_rows(
         self, cutoff: datetime, batch_size: int
     ) -> int:
@@ -87,8 +113,8 @@ class CheckResponsesCost:
         Check if background responses are complete and track their cost.
         - Get all status="queued" or "in_progress" and file_purpose="response" jobs
         - Query the provider to check if response is complete
-        - Cost is automatically tracked by litellm.aget_responses()
-        - Mark completed/failed/cancelled responses as complete in the database
+        - Cost is automatically tracked by the get-responses call
+        - Mark responses in a terminal state as complete in the database
         """
         try:
             await self._cleanup_stale_managed_objects()
@@ -134,7 +160,7 @@ class CheckResponsesCost:
                     litellm_metadata["model"] = model_name
                     litellm_metadata["model_group"] = model_name  # Use same value for model_group
                 
-                response = await litellm.aget_responses(
+                response = await self._get_response(
                     response_id=responses_id_security,
                     litellm_metadata=litellm_metadata,
                 )
@@ -144,21 +170,14 @@ class CheckResponsesCost:
                 )
                 
             except Exception as e:
-                verbose_proxy_logger.info(
+                verbose_proxy_logger.warning(
                     f"Skipping job {unified_object_id} due to error: {e}"
                 )
                 continue
 
-            # Check if response is in a terminal state
-            if response.status == "completed":
+            if response.status in TERMINAL_RESPONSE_STATUSES:
                 verbose_proxy_logger.info(
-                    f"Response {unified_object_id} is complete. Cost automatically tracked by aget_responses."
-                )
-                completed_jobs.append(job)
-                
-            elif response.status in ["failed", "cancelled"]:
-                verbose_proxy_logger.info(
-                    f"Response {unified_object_id} has status {response.status}, marking as complete"
+                    f"Response {unified_object_id} has terminal status {response.status}, marking as complete"
                 )
                 completed_jobs.append(job)
 

enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py

@@ -11,7 +11,7 @@ Endpoints for /project operations
 #### PROJECT MANAGEMENT ####
 
 import json
-from collections.abc import Mapping, Sequence
+from collections.abc import Sequence
 from typing import TYPE_CHECKING
 
 from fastapi import APIRouter, Depends, HTTPException, Request
@@ -19,6 +19,7 @@ from fastapi import APIRouter, Depends, HTTPException, Request
 from litellm._logging import verbose_proxy_logger
 from litellm._uuid import uuid
 from litellm.proxy._types import *
+from litellm.proxy.auth.auth_checks import delete_cached_project_object
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.management_endpoints.common_utils import _set_object_metadata_field
 from litellm.proxy.management_helpers.utils import (
@@ -28,7 +29,11 @@ from litellm.proxy.utils import PrismaClient, handle_exception_on_proxy
 
 if TYPE_CHECKING:
     from prisma import models as prisma_models
-    from prisma.actions import LiteLLM_TeamTableActions
+    from prisma.actions import (
+        LiteLLM_ProjectTableActions,
+        LiteLLM_TeamTableActions,
+        LiteLLM_VerificationTokenActions,
+    )
 
 router = APIRouter()
 
@@ -38,6 +43,27 @@ def _team_table(prisma_client: PrismaClient) -> "LiteLLM_TeamTableActions[prisma
     return team_table
 
 
+def _project_table(prisma_client: PrismaClient) -> "LiteLLM_ProjectTableActions[prisma_models.LiteLLM_ProjectTable]":
+    project_table: LiteLLM_ProjectTableActions[prisma_models.LiteLLM_ProjectTable] = (
+        prisma_client.db.litellm_projecttable
+    )
+    return project_table
+
+
+def _verification_token_table(
+    prisma_client: PrismaClient,
+) -> "LiteLLM_VerificationTokenActions[prisma_models.LiteLLM_VerificationToken]":
+    verification_token_table: LiteLLM_VerificationTokenActions[prisma_models.LiteLLM_VerificationToken] = (
+        prisma_client.db.litellm_verificationtoken
+    )
+    return verification_token_table
+
+
+def _jsonified(prisma_client: PrismaClient, payload: dict[str, object]) -> dict[str, object]:
+    jsonified: dict[str, object] = prisma_client.jsonify_object(payload)
+    return jsonified
+
+
 async def _check_user_permission_for_project(
     user_api_key_dict: UserAPIKeyAuth,
     team_id: str | None,
@@ -136,7 +162,7 @@ def _check_team_project_limits(
 
     # --- Validate project models are a subset of team models ---
     project_models = data.models
-    team_models = team_object.models or []
+    team_models: list[str] = team_object.models or []
     if project_models and len(team_models) > 0:
         # If team has 'all-proxy-models', skip validation as it allows all models
         if SpecialModelNames.all_proxy_models.value not in team_models:
@@ -187,11 +213,11 @@ async def _create_budget_for_project(
 ) -> str:
     """Create a budget for the project and return budget_id."""
     budget_params = LiteLLM_BudgetTable.model_fields.keys()
-    _json_data: Mapping[str, object] = data.json(exclude_none=True)
+    _json_data: dict[str, object] = data.model_dump(exclude_none=True)
     _budget_data = {k: v for k, v in _json_data.items() if k in budget_params}
     budget_row = LiteLLM_BudgetTable.model_validate(_budget_data)
 
-    new_budget = prisma_client.jsonify_object(budget_row.json(exclude_none=True))
+    new_budget = _jsonified(prisma_client, budget_row.model_dump(exclude_none=True))
 
     _budget: prisma_models.LiteLLM_BudgetTable = await prisma_client.db.litellm_budgettable.create(
         data={
@@ -226,7 +252,7 @@ async def _set_project_object_permission(
     return None
 
 
-def _remove_budget_fields_from_project_data(project_data: dict) -> dict:
+def _remove_budget_fields_from_project_data(project_data: dict[str, object]) -> dict[str, object]:
     """
     Remove budget fields from project data.
     Budget fields belong to LiteLLM_BudgetTable, not LiteLLM_ProjectTable.
@@ -395,9 +421,7 @@ async def new_project(
             data.project_id = str(uuid.uuid4())
         else:
             # Check if project_id already exists
-            existing_project = await prisma_client.db.litellm_projecttable.find_unique(
-                where={"project_id": data.project_id}
-            )
+            existing_project = await _project_table(prisma_client).find_unique(where={"project_id": data.project_id})
             if existing_project is not None:
                 raise ProxyException(
                     message=f"Project id = {data.project_id} already exists. Please use a different project id.",
@@ -422,11 +446,14 @@ async def new_project(
         )
 
         # Create project row (following organization_endpoints.py pattern)
-        project_row = LiteLLM_ProjectTable(
-            **data.json(exclude_none=True),
-            object_permission_id=object_permission_id,
-            created_by=user_api_key_dict.user_id or litellm_proxy_admin_name,
-            updated_by=user_api_key_dict.user_id or litellm_proxy_admin_name,
+        project_row_payload: dict[str, object] = data.model_dump(exclude_none=True)
+        project_row = LiteLLM_ProjectTable.model_validate(
+            {
+                **project_row_payload,
+                "object_permission_id": object_permission_id,
+                "created_by": user_api_key_dict.user_id or litellm_proxy_admin_name,
+                "updated_by": user_api_key_dict.user_id or litellm_proxy_admin_name,
+            }
         )
 
         for field in LiteLLM_ManagementEndpoint_MetadataFields:
@@ -437,7 +464,7 @@ async def new_project(
                     value=getattr(data, field),
                 )
 
-        new_project_row = prisma_client.jsonify_object(project_row.json(exclude_none=True))
+        new_project_row = _jsonified(prisma_client, project_row.model_dump(exclude_none=True))
 
         # Remove budget fields (following organization_endpoints.py pattern)
         new_project_row = _remove_budget_fields_from_project_data(new_project_row)
@@ -514,6 +541,7 @@ async def update_project(
         litellm_proxy_admin_name,
         premium_user,
         prisma_client,
+        user_api_key_cache,
     )
 
     try:
@@ -558,7 +586,7 @@ async def update_project(
         # Fetch existing project
         existing_project: (
             prisma_models.LiteLLM_ProjectTable | None
-        ) = await prisma_client.db.litellm_projecttable.find_unique(where={"project_id": data.project_id})
+        ) = await _project_table(prisma_client).find_unique(where={"project_id": data.project_id})
 
         if existing_project is None:
             raise ProxyException(
@@ -615,8 +643,7 @@ async def update_project(
             )
 
         # Prepare update data
-        update_data = data.json(exclude_none=True, exclude={"project_id"})
-        update_data = prisma_client.jsonify_object(update_data)
+        update_data = _jsonified(prisma_client, data.model_dump(exclude_none=True, exclude={"project_id"}))
         update_data["updated_by"] = user_api_key_dict.user_id or litellm_proxy_admin_name
 
         # Handle budget updates
@@ -658,9 +685,10 @@ async def update_project(
         # Handle metadata fields
         for field in LiteLLM_ManagementEndpoint_MetadataFields:
             if field in update_data:
-                if update_data.get("metadata") is None:
-                    update_data["metadata"] = {}
-                update_data["metadata"][field] = update_data.pop(field)
+                existing_metadata = update_data.get("metadata")
+                metadata_dict: dict[str, object] = existing_metadata if isinstance(existing_metadata, dict) else {}
+                metadata_dict[field] = update_data.pop(field)
+                update_data["metadata"] = metadata_dict
 
         # Remove budget fields (following organization_endpoints.py pattern)
         update_data = _remove_budget_fields_from_project_data(update_data)
@@ -672,6 +700,11 @@ async def update_project(
             include={"litellm_budget_table": True, "object_permission": True},
         )
 
+        await delete_cached_project_object(
+            project_id=data.project_id,
+            user_api_key_cache=user_api_key_cache,
+        )
+
         return updated_project
     except Exception as e:
         verbose_proxy_logger.exception(
@@ -710,7 +743,7 @@ async def delete_project(
     }'
     ```
     """
-    from litellm.proxy.proxy_server import premium_user, prisma_client
+    from litellm.proxy.proxy_server import premium_user, prisma_client, user_api_key_cache
 
     try:
         if not premium_user:
@@ -741,11 +774,11 @@ async def delete_project(
                 detail={"error": "Only admins can delete projects"},
             )
 
-        deleted_projects = []
+        deleted_projects: list[prisma_models.LiteLLM_ProjectTable | None] = []
 
         for project_id in data.project_ids:
             # Check if project exists
-            existing_project = await prisma_client.db.litellm_projecttable.find_unique(where={"project_id": project_id})
+            existing_project = await _project_table(prisma_client).find_unique(where={"project_id": project_id})
 
             if existing_project is None:
                 raise ProxyException(
@@ -758,7 +791,7 @@ async def delete_project(
             # Check if there are any keys associated with this project
             associated_keys: Sequence[
                 prisma_models.LiteLLM_VerificationToken
-            ] = await prisma_client.db.litellm_verificationtoken.find_many(where={"project_id": project_id})
+            ] = await _verification_token_table(prisma_client).find_many(where={"project_id": project_id})
 
             if len(associated_keys) > 0:
                 raise ProxyException(
@@ -771,7 +804,12 @@ async def delete_project(
             # Delete the project
             deleted_project: (
                 prisma_models.LiteLLM_ProjectTable | None
-            ) = await prisma_client.db.litellm_projecttable.delete(where={"project_id": project_id})
+            ) = await _project_table(prisma_client).delete(where={"project_id": project_id})
+
+            await delete_cached_project_object(
+                project_id=project_id,
+                user_api_key_cache=user_api_key_cache,
+            )
 
             deleted_projects.append(deleted_project)
 
@@ -817,7 +855,7 @@ async def project_info(
             )
 
         # Fetch project
-        project: prisma_models.LiteLLM_ProjectTable | None = await prisma_client.db.litellm_projecttable.find_unique(
+        project: prisma_models.LiteLLM_ProjectTable | None = await _project_table(prisma_client).find_unique(
             where={"project_id": project_id},
             include={"litellm_budget_table": True, "object_permission": True},
         )
@@ -889,7 +927,7 @@ async def list_projects(
         if user_api_key_has_admin_view(user_api_key_dict):
             projects: Sequence[
                 prisma_models.LiteLLM_ProjectTable
-            ] = await prisma_client.db.litellm_projecttable.find_many(
+            ] = await _project_table(prisma_client).find_many(
                 include={"litellm_budget_table": True, "object_permission": True}
             )
         else:
@@ -899,9 +937,9 @@ async def list_projects(
             user_record: prisma_models.LiteLLM_UserTable | None = await prisma_client.db.litellm_usertable.find_unique(
                 where={"user_id": user_api_key_dict.user_id},
             )
-            user_team_ids: Sequence[str] = user_record.teams if user_record is not None and user_record.teams else []
+            user_team_ids: list[str] = user_record.teams if user_record is not None and user_record.teams else []
 
-            projects = await prisma_client.db.litellm_projecttable.find_many(
+            projects = await _project_table(prisma_client).find_many(
                 where={"team_id": {"in": user_team_ids}},
                 include={"litellm_budget_table": True, "object_permission": True},
             )

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.53"
+version = "0.1.56"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.53"
+version = "0.1.56"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

helm/litellm-helm/templates/migrations-job.yaml

@@ -105,6 +105,10 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       restartPolicy: OnFailure
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       {{- with .Values.affinity }}
       affinity:
         {{- toYaml . | nindent 8 }}

helm/litellm-helm/tests/migrations-job_tests.yaml

@@ -290,3 +290,27 @@ tests:
           value:
             allowPrivilegeEscalation: false
             readOnlyRootFilesystem: true
+  - it: should schedule onto the same nodes as the gateway
+    template: migrations-job.yaml
+    set:
+      migrationJob:
+        enabled: true
+      nodeSelector:
+        karpenter.sh/nodepool: litellm-e2e
+      tolerations:
+        - key: workload
+          operator: Equal
+          value: litellm-e2e
+          effect: NoSchedule
+    asserts:
+      - equal:
+          path: spec.template.spec.nodeSelector
+          value:
+            karpenter.sh/nodepool: litellm-e2e
+      - equal:
+          path: spec.template.spec.tolerations
+          value:
+            - key: workload
+              operator: Equal
+              value: litellm-e2e
+              effect: NoSchedule

helm/litellm/templates/backend/deployment.yaml

@@ -81,6 +81,10 @@ spec:
           readinessProbe:
             {{- toYaml . | nindent 12 }}
           {{- end }}
+          {{- with .Values.backend.startupProbe }}
+          startupProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
           {{- with .Values.backend.lifecycle }}
           lifecycle:
             {{- toYaml . | nindent 12 }}

helm/litellm/templates/backend/hpa.yaml

@@ -30,4 +30,8 @@ spec:
           type: Utilization
           averageUtilization: {{ .Values.backend.hpa.targetMemoryUtilizationPercentage }}
     {{- end }}
+  {{- with .Values.backend.hpa.behavior }}
+  behavior:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
 {{- end }}

helm/litellm/templates/gateway/deployment.yaml

@@ -83,6 +83,10 @@ spec:
           readinessProbe:
             {{- toYaml . | nindent 12 }}
           {{- end }}
+          {{- with .Values.gateway.startupProbe }}
+          startupProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
           {{- with .Values.gateway.lifecycle }}
           lifecycle:
             {{- toYaml . | nindent 12 }}

helm/litellm/templates/gateway/hpa.yaml

@@ -30,4 +30,8 @@ spec:
           type: Utilization
           averageUtilization: {{ .Values.gateway.hpa.targetMemoryUtilizationPercentage }}
     {{- end }}
+  {{- with .Values.gateway.hpa.behavior }}
+  behavior:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
 {{- end }}

helm/litellm/templates/ui/deployment.yaml

@@ -69,6 +69,10 @@ spec:
           readinessProbe:
             {{- toYaml . | nindent 12 }}
           {{- end }}
+          {{- with .Values.ui.startupProbe }}
+          startupProbe:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
           {{- with .Values.ui.lifecycle }}
           lifecycle:
             {{- toYaml . | nindent 12 }}

helm/litellm/templates/ui/hpa.yaml

@@ -30,4 +30,8 @@ spec:
           type: Utilization
           averageUtilization: {{ .Values.ui.hpa.targetMemoryUtilizationPercentage }}
     {{- end }}
+  {{- with .Values.ui.hpa.behavior }}
+  behavior:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
 {{- end }}

helm/litellm/tests/hpa_behavior_tests.yaml

@@ -0,0 +1,58 @@
+suite: test HPA scaling behavior passthrough
+templates:
+  - gateway/hpa.yaml
+  - backend/hpa.yaml
+  - ui/hpa.yaml
+values:
+  - ./values/required.yaml
+tests:
+  - it: HPA omits spec.behavior by default, so Kubernetes' default scaling applies
+    templates:
+      - gateway/hpa.yaml
+      - backend/hpa.yaml
+    asserts:
+      - isKind:
+          of: HorizontalPodAutoscaler
+      - notExists:
+          path: spec.behavior
+
+  - it: gateway HPA renders spec.behavior verbatim when configured
+    template: gateway/hpa.yaml
+    set:
+      gateway.hpa.behavior:
+        scaleDown:
+          stabilizationWindowSeconds: 300
+          policies:
+            - { type: Percent, value: 50, periodSeconds: 60 }
+        scaleUp:
+          stabilizationWindowSeconds: 0
+          selectPolicy: Max
+          policies:
+            - { type: Percent, value: 100, periodSeconds: 30 }
+            - { type: Pods, value: 2, periodSeconds: 30 }
+    asserts:
+      - equal:
+          path: spec.behavior
+          value:
+            scaleDown:
+              stabilizationWindowSeconds: 300
+              policies:
+                - { type: Percent, value: 50, periodSeconds: 60 }
+            scaleUp:
+              stabilizationWindowSeconds: 0
+              selectPolicy: Max
+              policies:
+                - { type: Percent, value: 100, periodSeconds: 30 }
+                - { type: Pods, value: 2, periodSeconds: 30 }
+
+  - it: behavior passthrough works on every autoscaled component (ui parity)
+    template: ui/hpa.yaml
+    set:
+      ui.hpa.enabled: true
+      ui.hpa.behavior:
+        scaleUp:
+          stabilizationWindowSeconds: 0
+    asserts:
+      - equal:
+          path: spec.behavior.scaleUp.stabilizationWindowSeconds
+          value: 0

helm/litellm/tests/probe_tests.yaml

@@ -104,3 +104,30 @@ tests:
             periodSeconds: 15
             timeoutSeconds: 4
             failureThreshold: 3
+
+  - it: no startupProbe by default, so existing installs are unchanged
+    templates:
+      - gateway/deployment.yaml
+      - backend/deployment.yaml
+    asserts:
+      - notExists:
+          path: spec.template.spec.containers[0].startupProbe
+
+  - it: startupProbe renders verbatim when configured, gating a slow cold start
+    template: gateway/deployment.yaml
+    set:
+      gateway.startupProbe:
+        httpGet: { path: /health/readiness, port: http }
+        failureThreshold: 30
+        periodSeconds: 10
+        timeoutSeconds: 5
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[0].startupProbe
+          value:
+            httpGet:
+              path: /health/readiness
+              port: http
+            failureThreshold: 30
+            periodSeconds: 10
+            timeoutSeconds: 5

helm/litellm/values.yaml

@@ -223,12 +223,28 @@ gateway:
     initialDelaySeconds: 5
     periodSeconds: 10
     timeoutSeconds: 10
+  # Optional startupProbe. Empty by default, so existing installs are unchanged
+  # and liveness/readiness apply from container start. Set it to gate
+  # liveness/readiness until a slow cold start finishes — a high failureThreshold
+  # tolerates long first-boot times without a liveness-kill loop, e.g.:
+  #   httpGet: { path: /health/readiness, port: http }
+  #   failureThreshold: 30
+  #   periodSeconds: 10
+  startupProbe: {}
   hpa:
     enabled: true
     minReplicas: 1
     maxReplicas: 10
     targetCPUUtilizationPercentage: 70
     targetMemoryUtilizationPercentage: 80
+    # Optional autoscaling/v2 scaling behavior (scaleUp / scaleDown policies and
+    # stabilization windows). Empty by default -> Kubernetes' default behavior.
+    # Rendered verbatim under spec.behavior, e.g.:
+    #   scaleUp:
+    #     stabilizationWindowSeconds: 0
+    #     policies:
+    #       - { type: Percent, value: 100, periodSeconds: 30 }
+    behavior: {}
   # PodDisruptionBudget for the gateway pods. Set exactly one of
   # `minAvailable` / `maxUnavailable` (minAvailable wins if both are set;
   # enabling without either falls back to `maxUnavailable: 1`). Disabled by
@@ -319,11 +335,15 @@ backend:
     initialDelaySeconds: 5
     periodSeconds: 10
     timeoutSeconds: 10
+  # Optional startupProbe; same shape as gateway.startupProbe. Empty by default.
+  startupProbe: {}
   hpa:
     enabled: true
     minReplicas: 1
     maxReplicas: 4
     targetCPUUtilizationPercentage: 70
+    # Optional autoscaling/v2 scaling behavior; same shape as gateway.hpa.behavior.
+    behavior: {}
   # Same shape as gateway.pdb.
   pdb:
     enabled: false
@@ -379,11 +399,15 @@ ui:
     httpGet: { path: /, port: http }
     initialDelaySeconds: 2
     periodSeconds: 10
+  # Optional startupProbe; same shape as gateway.startupProbe. Empty by default.
+  startupProbe: {}
   hpa:
     enabled: false
     minReplicas: 1
     maxReplicas: 3
     targetCPUUtilizationPercentage: 80
+    # Optional autoscaling/v2 scaling behavior; same shape as gateway.hpa.behavior.
+    behavior: {}
   # Same shape as gateway.pdb.
   pdb:
     enabled: false

litellm-proxy-extras/litellm_proxy_extras/migrations/20260713010000_add_ptu_columns_to_daily_team_spend/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_DailyTeamSpend" ADD COLUMN IF NOT EXISTS "ptu_flat_cost" DOUBLE PRECISION NOT NULL DEFAULT 0.0;

litellm-proxy-extras/litellm_proxy_extras/migrations/20260730000000_add_api_key_and_request_tags_to_managed_object_table/migration.sql

@@ -0,0 +1,5 @@
+-- Add api_key and request_tags columns to LiteLLM_ManagedObjectTable
+-- Captured at batch-create time so CheckBatchCost can attribute batch-cost spend
+-- back to the creating virtual key (and its tags) even when created_by is null.
+ALTER TABLE "LiteLLM_ManagedObjectTable" ADD COLUMN IF NOT EXISTS "api_key" TEXT;
+ALTER TABLE "LiteLLM_ManagedObjectTable" ADD COLUMN IF NOT EXISTS "request_tags" JSONB DEFAULT '[]';

litellm-proxy-extras/litellm_proxy_extras/migrations/20260807000000_add_autorouter_session_tier_turns/migration.sql

@@ -0,0 +1 @@
+ALTER TABLE "LiteLLM_AutoRouterSession" ADD COLUMN IF NOT EXISTS "tier_turns" JSONB NOT NULL DEFAULT '{}';

litellm-proxy-extras/litellm_proxy_extras/migrations/20260810000000_add_verificationtoken_settings_updated_at/migration.sql

@@ -0,0 +1,5 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN IF NOT EXISTS "settings_updated_at" TIMESTAMP(3);
+
+-- AlterTable
+ALTER TABLE "LiteLLM_DeletedVerificationToken" ADD COLUMN IF NOT EXISTS "settings_updated_at" TIMESTAMP(3);

litellm-proxy-extras/litellm_proxy_extras/migrations/20260811172448_add_shadow_eval/migration.sql

@@ -0,0 +1,49 @@
+-- CreateTable
+CREATE TABLE "LiteLLM_ShadowEvalJob" (
+    "id" TEXT NOT NULL,
+    "api_key_id" TEXT NOT NULL,
+    "router_name" TEXT NOT NULL,
+    "judge_model" TEXT NOT NULL,
+    "shadow_percentage" DOUBLE PRECISION NOT NULL,
+    "max_turns" INTEGER NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "created_by" TEXT,
+    "ends_at" TIMESTAMP(3) NOT NULL,
+    "stopped_at" TIMESTAMP(3),
+
+    CONSTRAINT "LiteLLM_ShadowEvalJob_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "LiteLLM_ShadowEvalAttempt" (
+    "id" TEXT NOT NULL,
+    "job_id" TEXT NOT NULL,
+    "request_id" TEXT NOT NULL,
+    "outcome" TEXT NOT NULL,
+    "tier" TEXT,
+    "real_model" TEXT,
+    "shadow_model" TEXT,
+    "confidence" DOUBLE PRECISION,
+    "judge_cost" DOUBLE PRECISION NOT NULL DEFAULT 0,
+    "error" TEXT,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "LiteLLM_ShadowEvalAttempt_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_ShadowEvalJob_api_key_id_idx" ON "LiteLLM_ShadowEvalJob"("api_key_id");
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_ShadowEvalJob_created_at_idx" ON "LiteLLM_ShadowEvalJob"("created_at");
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_ShadowEvalAttempt_job_id_idx" ON "LiteLLM_ShadowEvalAttempt"("job_id");
+
+
+-- One active job per key, enforced by the database rather than a read-then-create in the
+-- start endpoint, which races against a concurrent start on another pod. Partial indexes
+-- are not expressible in schema.prisma, so this lives here only. Active means not yet
+-- stopped; the start endpoint stamps stopped_at on expired jobs before creating.
+CREATE UNIQUE INDEX "LiteLLM_ShadowEvalJob_one_active_per_key"
+    ON "LiteLLM_ShadowEvalJob"("api_key_id") WHERE "stopped_at" IS NULL;

litellm-proxy-extras/litellm_proxy_extras/migrations/20260813180408_add_shadow_eval_direction/migration.sql

@@ -0,0 +1,8 @@
+-- AlterTable
+ALTER TABLE "LiteLLM_ShadowEvalJob" ADD COLUMN     "baseline_model" TEXT,
+ADD COLUMN     "direction" TEXT NOT NULL DEFAULT 'forward';
+
+DROP INDEX IF EXISTS "LiteLLM_ShadowEvalJob_one_active_per_key";
+
+CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_ShadowEvalJob_one_active_per_key_direction"
+    ON "LiteLLM_ShadowEvalJob"("api_key_id", "direction") WHERE "stopped_at" IS NULL;

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -30,7 +30,7 @@ model LiteLLM_BudgetTable {
   end_users LiteLLM_EndUserTable[] // multiple end-users can have the same budget
   tags LiteLLM_TagTable[] // multiple tags can have the same budget
   team_membership LiteLLM_TeamMembership[] // budgets of Users within a Team 
-  organization_membership LiteLLM_OrganizationMembership[] // budgets of Users within a Organization 
+  organization_membership LiteLLM_OrganizationMembership[] // budgets of Users within a Organization
 }
 
 // Models on proxy
@@ -452,6 +452,7 @@ model LiteLLM_VerificationToken {
     created_by String?
     updated_at      DateTime?               @default(now()) @updatedAt @map("updated_at")
     updated_by String?
+    settings_updated_at DateTime?             @map("settings_updated_at")
     last_active     DateTime?  // When this key was last used
     rotation_count   Int?       @default(0) // Number of times key has been rotated
     auto_rotate      Boolean?   @default(false) // Whether this key should be auto-rotated
@@ -548,6 +549,7 @@ model LiteLLM_DeletedVerificationToken {
     created_by              String?   // Original creator
     updated_at              DateTime? // Last update timestamp before deletion
     updated_by              String?   // Last user who updated before deletion
+    settings_updated_at       DateTime? // Last configuration change before deletion
     last_active             DateTime? // When this key was last used before deletion
     rotation_count          Int?      @default(0)
     auto_rotate             Boolean?  @default(false)
@@ -893,6 +895,7 @@ model LiteLLM_DailyTeamSpend {
   api_requests        BigInt      @default(0)
   successful_requests BigInt      @default(0)
   failed_requests     BigInt      @default(0)
+  ptu_flat_cost       Float    @default(0.0)
   created_at          DateTime @default(now())
   updated_at          DateTime @updatedAt
 
@@ -985,6 +988,8 @@ model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use t
   created_at DateTime @default(now())
   created_by String?
   team_id String?
+  api_key String?
+  request_tags Json? @default("[]")
   updated_at DateTime @updatedAt
   updated_by String?
 
@@ -1439,11 +1444,55 @@ model LiteLLM_AutoRouterSession {
   total_tokens          BigInt   @default(0)
   spend                 Float    @default(0)
   saved_spend           Float    @default(0)
+  tier_turns            Json     @default("{}")
 
   @@id([api_key, session_id, router_name])
   @@index([last_turn_at], map: "idx_autorouter_session_last_turn")
 }
 
+// Shadow eval: evaluation of an auto-router against a key's live traffic, in either
+// direction. forward duplicates the requests the key did not route through the router
+// through it, answering whether the key should adopt it; reverse duplicates the requests
+// the router did serve against a fixed baseline model, answering whether a key already on
+// it still benefits. Either way a sampled slice runs in a detached task and an LLM judge
+// compares real vs shadow responses blind. The job row is immutable config plus
+// stopped_at; every count, status, and spend figure is derived from the append-only
+// attempt rows, so nothing can disagree across pods or stop races.
+model LiteLLM_ShadowEvalJob {
+  id                String   @id @default(cuid())
+  api_key_id        String   // hashed virtual key whose traffic is shadowed
+  router_name       String   // the auto-router under evaluation, in either direction
+  direction         String   @default("forward") // forward | reverse
+  baseline_model    String?  // reverse only: the fixed model the router is judged against
+  judge_model       String
+  shadow_percentage Float
+  max_turns         Int      // sample budget: judge at most this many turns
+  created_at        DateTime @default(now())
+  created_by        String?
+  ends_at           DateTime
+  stopped_at        DateTime?
+
+  @@index([api_key_id])
+  @@index([created_at])
+}
+
+// One row per sampled pipeline: a blind verdict (real | shadow | tie) or an error.
+model LiteLLM_ShadowEvalAttempt {
+  id           String   @id @default(cuid())
+  job_id       String
+  request_id   String   // the judged real request
+  outcome      String   // real | shadow | tie | error
+  tier         String?  // router's tier for the prompt, when classified
+  real_model   String?
+  shadow_model String?
+  confidence   Float?
+  judge_cost   Float    @default(0)
+  error        String?
+  created_at   DateTime @default(now())
+
+  @@index([job_id])
+}
+
 // ---------------------------------------------------------------------------
 // Workflow Run Tracking
 //

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.83"
+version = "0.4.86"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.83"
+version = "0.4.86"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm/__init__.py

@@ -26,6 +26,7 @@ def _dev_env_hot_reload_enabled() -> bool:
 if os.getenv("LITELLM_MODE", "DEV") == "DEV":
     _dotenv.load_dotenv(override=_dev_env_hot_reload_enabled())
 
+from collections.abc import Sequence
 from typing import (
     Any,
     Callable,
@@ -172,6 +173,7 @@ callbacks: List[
 callback_settings: Dict[str, Dict[str, Any]] = {}
 initialized_langfuse_clients: int = 0
 langfuse_default_tags: Optional[List[str]] = None
+langfuse_enable_update_trace_keys: bool = False
 langsmith_batch_size: Optional[int] = None
 prometheus_initialize_budget_metrics: Optional[bool] = False
 prometheus_latency_buckets: Optional[List[float]] = None
@@ -197,6 +199,7 @@ standard_logging_payload_excluded_fields: Optional[List[str]] = (
     None  # Fields to exclude from StandardLoggingPayload before callbacks receive it
 )
 log_raw_request_response: bool = False
+request_correlation_in_logs: bool = False
 redact_messages_in_exceptions: Optional[bool] = False
 redact_user_api_key_info: Optional[bool] = False
 # When True (default — preserves historical behavior), the Router appends
@@ -215,6 +218,9 @@ add_user_information_to_llm_headers: Optional[bool] = (
 overwrite_user_with_key_hash: bool = (
     False  # force the outgoing `user` param to the hashed api key, so providers see a stable, tamper-proof id
 )
+bedrock_request_metadata_fields: Optional[Sequence[str]] = (
+    None  # allow-list of `user_api_key_*` fields (+ `spend_logs_metadata`) sent as Bedrock `requestMetadata`
+)
 store_audit_logs = False  # Enterprise feature, allow users to see audit logs
 skip_system_message_in_guardrail: bool = False
 skip_tool_message_in_guardrail: bool = False
@@ -244,6 +250,8 @@ use_chat_completions_url_for_anthropic_messages: bool = bool(
 # Or via `litellm_settings.strip_anthropic_total_tokens: true` in
 # config.yaml.
 strip_anthropic_total_tokens: bool = False
+anthropic_sse_ping_interval_seconds: float = 15.0
+sse_keepalive_ping_interval_seconds: float | None = None
 route_all_chat_openai_to_responses: bool = (
     os.getenv("LITELLM_ROUTE_ALL_CHAT_OPENAI_TO_RESPONSES", "false").lower() == "true"
 )  # When True, routes all OpenAI /chat/completions requests through the Responses API bridge

litellm/_logging.py

@@ -1,4 +1,5 @@
 import ast
+import contextvars
 import logging
 import os
 import sys
@@ -6,12 +7,44 @@ from datetime import datetime
 from logging import Formatter
 from typing import Any, Final
 
+import litellm
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
 from litellm.litellm_core_utils.secret_redaction import redact_string
 
 set_verbose = False
 
+session_id_var: Final[contextvars.ContextVar[str]] = contextvars.ContextVar("session_id", default="")
+trace_id_var: Final[contextvars.ContextVar[str]] = contextvars.ContextVar("trace_id", default="")
+
+_MAX_CORRELATION_ID_LENGTH: Final = 256
+
+
+def _sanitize_correlation_id(value: str) -> str:
+    """Strip control characters, bound length, and redact credential-shaped
+    content before a caller-controlled trace_id/session_id (e.g.
+    litellm_session_id, x-litellm-trace-id) is stamped into log lines.
+
+    Without the first two, a caller could embed \\r/\\n or terminal escape
+    sequences to forge fake log entries, or submit an oversized value repeated
+    across every log line for the request. Without the redaction, a caller
+    could smuggle a real credential (e.g. an sk-... key) through this field:
+    CorrelationContextFilter stamps trace_id/session_id onto the record after
+    SecretRedactionFilter has already run, so those two fields never otherwise
+    pass through credential redaction.
+    """
+    stripped: Final = "".join(ch for ch in value if ch.isprintable())
+    return _redact_string(stripped[:_MAX_CORRELATION_ID_LENGTH])
+
+
+def set_session_id(session_id: str) -> "contextvars.Token[str]":
+    return session_id_var.set(_sanitize_correlation_id(session_id))
+
+
+def set_trace_id(trace_id: str) -> "contextvars.Token[str]":
+    return trace_id_var.set(_sanitize_correlation_id(trace_id))
+
+
 if set_verbose is True:
     logging.warning(
         "`litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs."
@@ -77,6 +110,28 @@ class SecretRedactionFilter(logging.Filter):
 _secret_filter: Final = SecretRedactionFilter()
 
 
+class CorrelationContextFilter(logging.Filter):
+    """Stamps each log record with the current request's trace_id and session_id from contextvars.
+
+    Works in tandem with JsonFormatter: the formatter's record.__dict__ loop picks up these
+    attributes as first-class JSON fields without any formatter-level code.
+    """
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        if not litellm.request_correlation_in_logs:
+            return True
+        trace_id: Final = trace_id_var.get()
+        if trace_id:
+            record.trace_id = trace_id  # rebind-ok: stamping the LogRecord is the Filter interface's contract
+        session_id: Final = session_id_var.get()
+        if session_id:
+            record.session_id = session_id  # rebind-ok: stamping the LogRecord is the Filter interface's contract
+        return True
+
+
+_correlation_filter: Final = CorrelationContextFilter()
+
+
 json_logs = bool(os.getenv("JSON_LOGS", False))
 # Create a handler for the logger (you may need to adapt this based on your needs)
 log_level: Final = os.getenv("LITELLM_LOG", "DEBUG")
@@ -84,6 +139,7 @@ numeric_level: Final[str] = getattr(logging, log_level.upper())
 handler: Final = logging.StreamHandler()
 handler.setLevel(numeric_level)
 handler.addFilter(_secret_filter)
+handler.addFilter(_correlation_filter)
 
 
 def _try_parse_json_message(message: str) -> dict[str, Any] | None:
@@ -146,6 +202,11 @@ def _get_standard_record_attrs() -> frozenset:
 
 _STANDARD_RECORD_ATTRS: Final = _get_standard_record_attrs()
 
+# CorrelationContextFilter is the only legitimate source for these two JSON fields;
+# see JsonFormatter.format() for why they're excluded from the generic message-content
+# and extra-attribute promotion paths.
+_RESERVED_CORRELATION_FIELDS: Final = frozenset(("trace_id", "session_id"))
+
 
 class JsonFormatter(Formatter):
     def __init__(self):
@@ -164,13 +225,18 @@ class JsonFormatter(Formatter):
             "timestamp": self.formatTime(record),
         }
 
-        # Parse embedded JSON or Python dict repr in message so sub-fields become first-class properties
+        # Parse embedded JSON or Python dict repr in message so sub-fields become first-class properties.
+        # trace_id/session_id are excluded here unconditionally (not just "if not already
+        # set") - CorrelationContextFilter is the only legitimate source for these two
+        # fields, and a message that merely happens to parse as JSON/dict (e.g. a proxy
+        # log line dumping raw request headers) must never be able to claim them, even on
+        # a record the filter hasn't stamped yet (no correlation context active for it).
         parsed = _try_parse_json_message(message_str)
         if parsed is None:
             parsed = _try_parse_embedded_python_dict(message_str)
         if parsed is not None:
             for key, value in parsed.items():
-                if key not in json_record:
+                if key not in json_record and key not in _RESERVED_CORRELATION_FIELDS:
                     json_record[key] = value
 
         # Include extra attributes passed via logger.debug("msg", extra={...})
@@ -178,6 +244,18 @@ class JsonFormatter(Formatter):
             if key not in _STANDARD_RECORD_ATTRS and key not in json_record:
                 json_record[key] = value
 
+        # trace_id/session_id are reserved: CorrelationContextFilter is the only
+        # legitimate source for these two fields. Without this, a message string
+        # that happens to parse as JSON/dict (e.g. a proxy log line dumping raw
+        # request headers) with a "trace_id"/"session_id" key would have already
+        # claimed the key at the parsed-message step above, and the extra-attributes
+        # loop's "key not in json_record" guard would then skip the real value -
+        # letting a caller-supplied header spoof another request's correlation ids.
+        for reserved_key in _RESERVED_CORRELATION_FIELDS:
+            value = getattr(record, reserved_key, None)
+            if value:
+                json_record[reserved_key] = value
+
         # Set component/logger only if not already supplied via extra={...}
         if "component" not in json_record:
             json_record["component"] = record.name
@@ -190,12 +268,34 @@ class JsonFormatter(Formatter):
         return safe_dumps(json_record)
 
 
+class CorrelationPlainFormatter(logging.Formatter):
+    """Appends trace_id/session_id to plain-text log lines stamped by CorrelationContextFilter.
+
+    Mirrors JsonFormatter's handling of these two fields so request_correlation_in_logs
+    behaves the same whether or not json_logs is enabled.
+    """
+
+    def format(self, record: logging.LogRecord) -> str:
+        formatted: Final = super().format(record)
+        trace_id: Final = getattr(record, "trace_id", None)
+        session_id: Final = getattr(record, "session_id", None)
+        if not trace_id and not session_id:
+            return formatted
+        parts: Final = tuple(
+            p
+            for p in (f"trace_id={trace_id}" if trace_id else None, f"session_id={session_id}" if session_id else None)
+            if p
+        )
+        return f"{formatted} [{' '.join(parts)}]"
+
+
 # Function to set up exception handlers for JSON logging
 def _setup_json_exception_handlers(formatter):
     # Create a handler with JSON formatting for exceptions
     error_handler: Final = logging.StreamHandler()
     error_handler.setFormatter(formatter)
     error_handler.addFilter(_secret_filter)
+    error_handler.addFilter(_correlation_filter)
 
     # Setup excepthook for uncaught exceptions
     def json_excepthook(exc_type, exc_value, exc_traceback):
@@ -243,7 +343,7 @@ if json_logs:
     handler.setFormatter(JsonFormatter())
     _setup_json_exception_handlers(JsonFormatter())
 else:
-    formatter: Final = logging.Formatter(
+    formatter: Final = CorrelationPlainFormatter(
         "\033[92m%(asctime)s - %(name)s:%(levelname)s\033[0m: %(filename)s:%(lineno)s - %(message)s",
         datefmt="%H:%M:%S",
     )
@@ -346,6 +446,7 @@ def _initialize_loggers_with_handler(handler: logging.Handler):
     - Prevents bubbling to parent/root (critical to prevent duplicate JSON logs)
     """
     handler.addFilter(_secret_filter)
+    handler.addFilter(_correlation_filter)
     for lg in _get_loggers_to_initialize():
         lg.handlers.clear()  # remove any existing handlers
         lg.addHandler(handler)  # add JSON formatter handler

litellm/_redis.py

@@ -67,12 +67,20 @@ def _init_arg_names(cls: type) -> frozenset[str]:
 
     Keyword-only parameters are included, and the MRO is walked because redis-py splits a
     connection's parameters between ``AbstractConnection`` and its concrete subclasses.
+
+    Each ``__init__`` is unwrapped before introspection: redis-py >= 7.4 decorates
+    ``AbstractConnection.__init__`` with ``@deprecated_args``, whose wrapper is declared
+    ``(self, *args, **kwargs)`` — introspecting the wrapper directly loses every real
+    parameter (``socket_timeout`` included), which silently emptied this allowlist and
+    dropped the socket timeouts from url-configured connections. ``inspect.unwrap``
+    follows the ``__wrapped__`` chain to the true signature and is a no-op on
+    undecorated ``__init__``s.
     """
     return frozenset(
         name
         for klass in inspect.getmro(cls)
         if klass is not object
-        for spec in (inspect.getfullargspec(klass.__init__),)
+        for spec in (inspect.getfullargspec(inspect.unwrap(klass.__init__)),)
         for name in spec.args + spec.kwonlyargs
     )
 

litellm/a2a_protocol/exception_mapping_utils.py

@@ -202,9 +202,9 @@ async def handle_a2a_localhost_retry(
     # Fix the agent card URL
     set_agent_card_url(agent_card, error.base_url)
 
-    # Reuse the httpx client LiteLLM attached at creation. It carries this agent's
-    # trace-id and auth headers, so a fresh client would drop them. Only clients built
-    # by ``create_a2a_client`` have it; an externally-supplied client cannot be retried.
+    # Reuse the httpx client and call context LiteLLM attached at creation, since the
+    # context carries this agent's trace-id/auth headers. Only clients built by
+    # ``create_a2a_client`` have them; an externally-supplied client cannot be retried.
     httpx_client: Final = getattr(a2a_client, "_litellm_httpx_client", None)
     if httpx_client is None:
         raise RuntimeError(
@@ -220,5 +220,8 @@ async def handle_a2a_localhost_retry(
         ),
     )
     new_client._litellm_httpx_client = httpx_client
+    new_client._litellm_call_context = getattr(  # pyright: ignore[reportAttributeAccessIssue]  # LiteLLM-owned stash
+        a2a_client, "_litellm_call_context", None
+    )
     new_client._litellm_agent_card = agent_card
     return new_client

litellm/a2a_protocol/litellm_completion_bridge/handler.py

@@ -10,7 +10,7 @@ A2A Streaming Events (in order):
 4. Status update (kind: "status-update") - Final status "completed" with final=true
 """
 
-from collections.abc import AsyncIterator, Mapping
+from collections.abc import AsyncIterator, Callable, Coroutine, Mapping
 from typing import Any, Final
 
 import litellm
@@ -54,7 +54,7 @@ class A2ACompletionBridgeHandler:
         agent_extra_headers: Mapping[str, str] | None,
         *,
         stream: bool,
-    ) -> Mapping[str, Any]:
+    ) -> Mapping[str, object]:
         # Extract message from params
         message: Final = params.get("message", {})
 
@@ -63,7 +63,7 @@ class A2ACompletionBridgeHandler:
 
         # Get completion params
         custom_llm_provider: Final = litellm_params.get("custom_llm_provider")
-        model: Final = litellm_params.get("model", "agent")
+        model: Final[str] = litellm_params.get("model", "agent")
 
         # Build full model string if provider specified
         # Skip prepending if model already starts with the provider prefix
@@ -109,13 +109,16 @@ class A2ACompletionBridgeHandler:
         return completion_params
 
     @staticmethod
-    async def _acompletion(completion_params: Mapping[str, Any]) -> ModelResponse | CustomStreamWrapper:
-        return await litellm.acompletion(**completion_params)
+    async def _acompletion(completion_params: Mapping[str, object]) -> ModelResponse | CustomStreamWrapper:
+        acompletion_fn: Final[Callable[..., Coroutine[object, object, ModelResponse | CustomStreamWrapper]]] = vars(
+            litellm
+        )["acompletion"]
+        return await acompletion_fn(**completion_params)
 
     @staticmethod
     async def handle_non_streaming(
         request_id: str,
-        params: dict[str, Any],
+        params: dict[str, object],
         litellm_params: dict[str, Any],
         api_base: str | None = None,
         agent_extra_headers: dict[str, str] | None = None,
@@ -296,8 +299,8 @@ class A2ACompletionBridgeHandler:
 # Convenience functions that delegate to the class methods
 async def handle_a2a_completion(
     request_id: str,
-    params: dict[str, Any],
-    litellm_params: dict[str, Any],
+    params: dict[str, object],
+    litellm_params: dict[str, object],
     api_base: str | None = None,
     agent_extra_headers: dict[str, str] | None = None,
 ) -> dict[str, object]:
@@ -313,8 +316,8 @@ async def handle_a2a_completion(
 
 async def handle_a2a_completion_streaming(
     request_id: str,
-    params: dict[str, Any],
-    litellm_params: dict[str, Any],
+    params: dict[str, object],
+    litellm_params: dict[str, object],
     api_base: str | None = None,
     agent_extra_headers: dict[str, str] | None = None,
 ) -> AsyncIterator[dict[str, object]]:

litellm/a2a_protocol/main.py

@@ -12,7 +12,8 @@ Provides standalone functions with @client decorator for LiteLLM logging integra
 import asyncio
 import datetime
 import uuid
-from collections.abc import AsyncIterator, Coroutine
+from collections.abc import AsyncIterator, Coroutine, Mapping
+from types import ModuleType
 from typing import TYPE_CHECKING, Any, Final, Optional, cast
 
 import litellm
@@ -30,6 +31,7 @@ from litellm.utils import client
 
 if TYPE_CHECKING:
     from a2a.client import Client as A2AClientType
+    from a2a.client import ClientCallContext as A2ACallContextType
     from a2a.compat.v0_3.types import (
         AgentCard,
         Message,
@@ -37,15 +39,18 @@ if TYPE_CHECKING:
         SendMessageResponse,
         SendStreamingMessageRequest,
         SendStreamingMessageResponse,
+        SendStreamingMessageSuccessResponse,
         Task,
     )
+    from a2a.types.a2a_pb2 import SendMessageRequest as CoreSendMessageRequest
+    from a2a.types.a2a_pb2 import StreamResponse as CoreStreamResponse
 
 # Runtime imports — requires a2a-sdk>=1.1.0
 A2A_SDK_AVAILABLE = False
-_a2a_conversions: Any = None
+_a2a_conversions: ModuleType | None = None
 
 try:
-    from a2a.client import Client, ClientConfig, create_client
+    from a2a.client import Client, ClientCallContext, ClientConfig, create_client
     from a2a.compat.v0_3 import conversions as _a2a_conversions
     from a2a.compat.v0_3.types import (
         Message,
@@ -60,6 +65,7 @@ try:
     A2A_SDK_AVAILABLE = True
 except ImportError:
     Client = None
+    ClientCallContext = None
     ClientConfig = None
     create_client = None
 
@@ -126,7 +132,7 @@ _A2A_COST_PARAM_KEYS: Final = ("cost_per_query", "input_cost_per_token", "output
 
 def _set_litellm_params_on_logging_obj(
     kwargs: dict[str, Any],
-    litellm_params: dict[str, Any],
+    litellm_params: Mapping[str, object],
 ) -> None:
     """
     Merge the agent's pricing params into model_call_details["litellm_params"]
@@ -148,7 +154,7 @@ def _set_litellm_params_on_logging_obj(
     logging_obj.model_call_details["litellm_params"] = {**existing, **cost_params}
 
 
-def _get_a2a_model_info(a2a_client: Any, kwargs: dict[str, Any]) -> str:
+def _get_a2a_model_info(a2a_client: "A2AClientType", kwargs: dict[str, Any]) -> str:
     """
     Extract agent info and set model/custom_llm_provider for cost tracking.
 
@@ -177,7 +183,7 @@ def _get_a2a_model_info(a2a_client: Any, kwargs: dict[str, Any]) -> str:
     return agent_name
 
 
-def _get_a2a_client_agent_card(a2a_client: Any) -> Optional["AgentCard"]:
+def _get_a2a_client_agent_card(a2a_client: "A2AClientType") -> Optional["AgentCard"]:
     agent_card = cast(Optional["AgentCard"], getattr(a2a_client, "_litellm_agent_card", None))
     if agent_card is not None:
         return agent_card
@@ -189,9 +195,9 @@ def _get_a2a_client_agent_card(a2a_client: Any) -> Optional["AgentCard"]:
 
 async def _send_message_via_completion_bridge(
     request: "SendMessageRequest",
-    custom_llm_provider: str,
+    custom_llm_provider: object,
     api_base: str | None,
-    litellm_params: dict[str, Any],
+    litellm_params: dict[str, object],
     agent_extra_headers: dict[str, str] | None = None,
 ) -> LiteLLMSendMessageResponse:
     """
@@ -218,6 +224,24 @@ async def _send_message_via_completion_bridge(
     return LiteLLMSendMessageResponse.from_dict(response_dict, request_id=str(request.id))
 
 
+def _get_a2a_call_context(a2a_client: "A2AClientType") -> Optional["A2ACallContextType"]:
+    return getattr(a2a_client, "_litellm_call_context", None)
+
+
+def _to_core_send_message_request(request: "SendMessageRequest") -> "CoreSendMessageRequest":
+    from a2a.compat.v0_3 import conversions
+
+    return conversions.to_core_send_message_request(request)
+
+
+def _to_compat_stream_response(
+    event: "CoreStreamResponse", request_id: str | int
+) -> "SendStreamingMessageSuccessResponse":
+    from a2a.compat.v0_3 import conversions
+
+    return conversions.to_compat_stream_response(event, request_id=request_id)
+
+
 async def _send_message(a2a_client: "A2AClientType", request: "SendMessageRequest") -> "SendMessageResponse":
     """Send a non-streaming message via a2a-sdk 1.x and return JSON-RPC response."""
     if _a2a_conversions is None:
@@ -225,17 +249,14 @@ async def _send_message(a2a_client: "A2AClientType", request: "SendMessageReques
             "The 'a2a' package is required for A2A agent invocation. Install it with: pip install a2a-sdk"
         )
 
-    pb_request: Final = _a2a_conversions.to_core_send_message_request(request)
+    pb_request: Final = _to_core_send_message_request(request)
     last_event = None
-    async for event in a2a_client.send_message(pb_request):
+    async for event in a2a_client.send_message(pb_request, context=_get_a2a_call_context(a2a_client)):
         last_event = event
     if last_event is None:
         raise RuntimeError("A2A send_message failed: no response received from agent.")
 
-    stream_compat: Final = _a2a_conversions.to_compat_stream_response(
-        last_event,
-        request_id=request.id,
-    )
+    stream_compat: Final = _to_compat_stream_response(last_event, request_id=request.id)
     result: Final = stream_compat.result
     if not isinstance(result, (Message, Task)):
         raise RuntimeError(
@@ -300,12 +321,9 @@ async def _stream_messages(
             "The 'a2a' package is required for A2A agent invocation. Install it with: pip install a2a-sdk"
         )
 
-    pb_request: Final = _a2a_conversions.to_core_send_message_request(request)
-    async for event in a2a_client.send_message(pb_request):
-        compat_chunk = _a2a_conversions.to_compat_stream_response(
-            event,
-            request_id=request.id,
-        )
+    pb_request: Final[CoreSendMessageRequest] = _a2a_conversions.to_core_send_message_request(request)
+    async for event in a2a_client.send_message(pb_request, context=_get_a2a_call_context(a2a_client)):
+        compat_chunk = _to_compat_stream_response(event, request_id=request.id)
         yield SendStreamingMessageResponse(root=compat_chunk)
 
 
@@ -362,10 +380,10 @@ async def asend_message(
     a2a_client: Optional["A2AClientType"] = None,
     request: Optional["SendMessageRequest"] = None,
     api_base: str | None = None,
-    litellm_params: dict[str, Any] | None = None,
+    litellm_params: dict[str, object] | None = None,
     agent_id: str | None = None,
     agent_extra_headers: dict[str, str] | None = None,
-    **kwargs: Any,
+    **kwargs: object,
 ) -> LiteLLMSendMessageResponse:
     """
     Async: Send a message to an A2A agent.
@@ -479,7 +497,7 @@ async def asend_message(
     response: Final = LiteLLMSendMessageResponse.from_a2a_response(a2a_response, request_id=str(request.id))
 
     # Calculate token usage from request and response
-    response_dict: Final = a2a_response.model_dump(mode="json", exclude_none=True)
+    response_dict: Final[dict[str, object]] = a2a_response.model_dump(mode="json", exclude_none=True)
     (
         prompt_tokens,
         completion_tokens,
@@ -510,7 +528,7 @@ def send_message(
     a2a_client: "A2AClientType",
     request: "SendMessageRequest",
     **kwargs: Any,
-) -> LiteLLMSendMessageResponse | Coroutine[Any, Any, LiteLLMSendMessageResponse]:
+) -> LiteLLMSendMessageResponse | Coroutine[object, object, LiteLLMSendMessageResponse]:
     """
     Sync: Send a message to an A2A agent.
 
@@ -539,9 +557,9 @@ def _build_streaming_logging_obj(
     request: "SendStreamingMessageRequest",
     agent_name: str,
     agent_id: str | None,
-    litellm_params: dict[str, Any] | None,
-    metadata: dict[str, Any] | None,
-    proxy_server_request: dict[str, Any] | None,
+    litellm_params: dict[str, object] | None,
+    metadata: dict[str, object] | None,
+    proxy_server_request: dict[str, object] | None,
 ) -> Logging:
     """Build logging object for streaming A2A requests."""
     start_time: Final = datetime.datetime.now()
@@ -582,10 +600,10 @@ async def asend_message_streaming(
     a2a_client: Optional["A2AClientType"] = None,
     request: Optional["SendStreamingMessageRequest"] = None,
     api_base: str | None = None,
-    litellm_params: dict[str, Any] | None = None,
+    litellm_params: dict[str, object] | None = None,
     agent_id: str | None = None,
-    metadata: dict[str, Any] | None = None,
-    proxy_server_request: dict[str, Any] | None = None,
+    metadata: dict[str, object] | None = None,
+    proxy_server_request: dict[str, object] | None = None,
     agent_extra_headers: dict[str, str] | None = None,
     **kwargs: object,
 ) -> AsyncIterator[Any]:
@@ -756,26 +774,12 @@ async def create_a2a_client(
 
     verbose_logger.info("Creating A2A client for %s", base_url)
 
-    # Use get_async_httpx_client with per-agent params so that different agents
-    # (with different extra_headers) get separate cached clients.  The params
-    # dict is hashed into the cache key, keeping agent auth isolated while
-    # still reusing connections within the same agent.
-    #
-    # Only pass params that AsyncHTTPHandler.__init__ accepts (e.g. timeout).
-    # Use "disable_aiohttp_transport" key for cache-key-only data (it's
-    # filtered out before reaching the constructor).
-    _client_params: Final[dict] = {"timeout": timeout}
-    if extra_headers:
-        # Encode headers into a cache-key-only param so each unique header
-        # set produces a distinct cache key.
-        _client_params["disable_aiohttp_transport"] = str(sorted(extra_headers.items()))
     _async_handler: Final = get_async_httpx_client(
         llm_provider=httpxSpecialProvider.A2AProvider,
-        params=_client_params,
+        params={"timeout": timeout},
     )
     httpx_client: Final = _async_handler.client
     if extra_headers:
-        httpx_client.headers.update(extra_headers)
         verbose_proxy_logger.debug("A2A client created with extra_headers=%s", list(extra_headers.keys()))
 
     a2a_client: Final = await create_client(  # pyright: ignore[reportOptionalCall]
@@ -784,11 +788,17 @@ async def create_a2a_client(
             httpx_client=httpx_client,
             streaming=streaming,
         ),
+        resolver_http_kwargs={"headers": extra_headers} if extra_headers else None,
     )
     # Stash LiteLLM-owned handles on the client so the localhost-retry path can reuse
-    # the configured httpx client (with this agent's trace-id/auth headers) without
-    # excavating a2a-sdk private internals.
+    # the configured httpx client and this agent's headers without excavating
+    # a2a-sdk private internals.
     a2a_client._litellm_httpx_client = httpx_client
+    a2a_client._litellm_call_context = (  # pyright: ignore[reportAttributeAccessIssue]  # LiteLLM-owned stash
+        ClientCallContext(service_parameters=extra_headers)  # pyright: ignore[reportOptionalCall]  # SDK checked above
+        if extra_headers
+        else None
+    )
     agent_card: Final = getattr(a2a_client, "_card", None)
     if agent_card is not None:
         a2a_client._litellm_agent_card = agent_card

litellm/a2a_protocol/providers/pydantic_ai_agents/transformation.py

@@ -6,16 +6,33 @@ This module provides fake streaming by converting non-streaming responses into s
 """
 
 import asyncio
-from collections.abc import AsyncIterator
-from typing import Any, Final, cast
+from collections.abc import AsyncIterator, Mapping, Sequence
+from typing import Any, Final, Protocol, cast, runtime_checkable
 from uuid import uuid4
 
+from pydantic import TypeAdapter
+
 from litellm._logging import verbose_logger
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
     get_async_httpx_client,
 )
 
+_ANY_KEY_DICT_ADAPTER: Final = TypeAdapter(dict[object, object])
+_STR_KEY_DICT_ADAPTER: Final = TypeAdapter(dict[str, object])
+_LIST_ADAPTER: Final = TypeAdapter(list[object])
+_TEXT_ADAPTER: Final = TypeAdapter(str)
+
+
+@runtime_checkable
+class _SupportsModelDump(Protocol):
+    def model_dump(self, *, mode: str, exclude_none: bool) -> Mapping[str, object]: ...
+
+
+@runtime_checkable
+class _SupportsPydanticDict(Protocol):
+    def dict(self, *, exclude_none: bool) -> Mapping[str, object]: ...
+
 
 class PydanticAITransformation:
     """
@@ -28,7 +45,7 @@ class PydanticAITransformation:
     """
 
     @staticmethod
-    def _remove_none_values(obj: Any) -> Any:
+    def _remove_none_values(obj: object) -> object:
         """
         Recursively remove None values from a dict/list structure.
 
@@ -42,14 +59,18 @@ class PydanticAITransformation:
             Cleaned object with None values removed
         """
         if isinstance(obj, dict):
-            return {k: PydanticAITransformation._remove_none_values(v) for k, v in obj.items() if v is not None}
+            typed_dict: Final = _ANY_KEY_DICT_ADAPTER.validate_python(obj)
+            return {k: PydanticAITransformation._remove_none_values(v) for k, v in typed_dict.items() if v is not None}
         elif isinstance(obj, list):
-            return [PydanticAITransformation._remove_none_values(item) for item in obj if item is not None]
+            typed_list: Final = _LIST_ADAPTER.validate_python(obj)
+            return [PydanticAITransformation._remove_none_values(item) for item in typed_list if item is not None]
         else:
             return obj
 
     @staticmethod
-    def _params_to_dict(params: Any) -> dict[str, Any]:
+    def _params_to_dict(
+        params: "_SupportsModelDump | _SupportsPydanticDict | Mapping[str, object]",
+    ) -> Mapping[str, object]:
         """
         Convert params to a dict, handling Pydantic models.
 
@@ -59,10 +80,10 @@ class PydanticAITransformation:
         Returns:
             Dict representation of params
         """
-        if hasattr(params, "model_dump"):
+        if isinstance(params, _SupportsModelDump):
             # Pydantic v2 model
             return params.model_dump(mode="python", exclude_none=True)
-        elif hasattr(params, "dict"):
+        elif isinstance(params, _SupportsPydanticDict):
             # Pydantic v1 model
             return params.dict(exclude_none=True)
         elif isinstance(params, dict):
@@ -75,12 +96,12 @@ class PydanticAITransformation:
     async def _poll_for_completion(
         client: AsyncHTTPHandler,
         endpoint: str,
-        task_id: str,
+        task_id: object,
         request_id: str,
         max_attempts: int = 30,
         poll_interval: float = 0.5,
         agent_extra_headers: dict[str, str] | None = None,
-    ) -> dict[str, Any]:
+    ) -> dict[str, object]:
         """
         Poll for task completion using tasks/get method.
 
@@ -112,10 +133,10 @@ class PydanticAITransformation:
                 },
             )
             response.raise_for_status()
-            poll_data = response.json()
+            poll_data = _STR_KEY_DICT_ADAPTER.validate_python(response.json())
 
-            result = poll_data.get("result", {})
-            status = result.get("status", {})
+            result = _STR_KEY_DICT_ADAPTER.validate_python(poll_data.get("result", {}))
+            status = _STR_KEY_DICT_ADAPTER.validate_python(result.get("status", {}))
             state = status.get("state", "")
 
             verbose_logger.debug("Pydantic AI: Poll attempt %s/%s, state=%s", attempt + 1, max_attempts, state)
@@ -133,10 +154,10 @@ class PydanticAITransformation:
     async def _send_and_poll_raw(
         api_base: str,
         request_id: str,
-        params: Any,
+        params: "_SupportsModelDump | _SupportsPydanticDict | Mapping[str, object]",
         timeout: float = 60.0,
         agent_extra_headers: dict[str, str] | None = None,
-    ) -> dict[str, Any]:
+    ) -> dict[str, object]:
         """
         Send a request to Pydantic AI agent and return the raw task response.
 
@@ -153,14 +174,16 @@ class PydanticAITransformation:
             Raw Pydantic AI task response (with history/artifacts)
         """
         # Convert params to dict if it's a Pydantic model
-        params_dict = PydanticAITransformation._params_to_dict(params)
-
         # Remove None values - FastA2A doesn't accept null for optional fields
-        params_dict = PydanticAITransformation._remove_none_values(params_dict)
+        params_dict: Final = _ANY_KEY_DICT_ADAPTER.validate_python(
+            PydanticAITransformation._remove_none_values(PydanticAITransformation._params_to_dict(params))
+        )
 
         # Ensure the message has 'kind': 'message' as required by FastA2A/Pydantic AI
         if "message" in params_dict:
-            params_dict["message"]["kind"] = "message"
+            message_value: Final = _ANY_KEY_DICT_ADAPTER.validate_python(params_dict["message"])
+            message_value["kind"] = "message"
+            params_dict["message"] = message_value
 
         # Build A2A JSON-RPC request using message/send method for FastA2A compatibility
         a2a_request: Final = {
@@ -189,11 +212,11 @@ class PydanticAITransformation:
             },
         )
         response.raise_for_status()
-        response_data = response.json()
+        response_data = _STR_KEY_DICT_ADAPTER.validate_python(response.json())
 
         # Check if task is already completed
-        result: Final = response_data.get("result", {})
-        status: Final = result.get("status", {})
+        result: Final = _STR_KEY_DICT_ADAPTER.validate_python(response_data.get("result", {}))
+        status: Final = _STR_KEY_DICT_ADAPTER.validate_python(result.get("status", {}))
         state: Final = status.get("state", "")
 
         if state != "completed":
@@ -217,10 +240,10 @@ class PydanticAITransformation:
     async def send_non_streaming_request(
         api_base: str,
         request_id: str,
-        params: Any,
+        params: "_SupportsModelDump | _SupportsPydanticDict | Mapping[str, object]",
         timeout: float = 60.0,
         agent_extra_headers: dict[str, str] | None = None,
-    ) -> dict[str, Any]:
+    ) -> dict[str, object]:
         """
         Send a non-streaming A2A request to Pydantic AI agent and wait for completion.
 
@@ -253,10 +276,10 @@ class PydanticAITransformation:
     async def send_and_get_raw_response(
         api_base: str,
         request_id: str,
-        params: Any,
+        params: "_SupportsModelDump | _SupportsPydanticDict | Mapping[str, object]",
         timeout: float = 60.0,
         agent_extra_headers: dict[str, str] | None = None,
-    ) -> dict[str, Any]:
+    ) -> dict[str, object]:
         """
         Send a request to Pydantic AI agent and return the raw task response.
 
@@ -282,9 +305,9 @@ class PydanticAITransformation:
 
     @staticmethod
     def _transform_to_a2a_response(
-        response_data: dict[str, Any],
+        response_data: Mapping[str, object],
         request_id: str,
-    ) -> dict[str, Any]:
+    ) -> dict[str, object]:
         """
         Transform Pydantic AI task response to standard A2A non-streaming format.
 
@@ -328,7 +351,7 @@ class PydanticAITransformation:
         }
 
     @staticmethod
-    def _extract_response_text(response_data: dict[str, Any]) -> tuple[str, str, list]:
+    def _extract_response_text(response_data: Mapping[str, object]) -> tuple[object, object, Sequence[object]]:
         """
         Extract response text from completed task response.
 
@@ -342,52 +365,53 @@ class PydanticAITransformation:
         Returns:
             Tuple of (full_text, message_id, parts)
         """
-        result: Final = response_data.get("result", {})
+        result: Final = _STR_KEY_DICT_ADAPTER.validate_python(response_data.get("result", {}))
 
         # Try to extract from artifacts first (preferred for results)
         artifacts: Final = result.get("artifacts", [])
         if artifacts:
-            for artifact in artifacts:
-                parts = artifact.get("parts", [])
+            for artifact in _LIST_ADAPTER.validate_python(artifacts):
+                parts = _LIST_ADAPTER.validate_python(_STR_KEY_DICT_ADAPTER.validate_python(artifact).get("parts", []))
                 for part in parts:
-                    if part.get("kind") == "text":
-                        text = part.get("text", "")
+                    if (part_dict := _STR_KEY_DICT_ADAPTER.validate_python(part)).get("kind") == "text":
+                        text = part_dict.get("text", "")
                         if text:
                             return text, str(uuid4()), parts
 
         # Fall back to history - get the last agent message
-        history: Final = result.get("history", [])
+        history: Final = _LIST_ADAPTER.validate_python(result.get("history", []))
         for msg in reversed(history):
-            if msg.get("role") == "agent":
-                parts = msg.get("parts", [])
-                message_id = msg.get("messageId", str(uuid4()))
+            if (msg_dict := _STR_KEY_DICT_ADAPTER.validate_python(msg)).get("role") == "agent":
+                parts = _LIST_ADAPTER.validate_python(msg_dict.get("parts", []))
+                message_id = msg_dict.get("messageId", str(uuid4()))
                 full_text = ""
                 for part in parts:
-                    if part.get("kind") == "text":
-                        full_text += part.get("text", "")
+                    if (part_dict := _STR_KEY_DICT_ADAPTER.validate_python(part)).get("kind") == "text":
+                        full_text += _TEXT_ADAPTER.validate_python(part_dict.get("text", ""))
                 if full_text:
                     return full_text, message_id, parts
 
         # Fall back to message field (original format)
         message: Final = result.get("message", {})
         if message:
-            parts = message.get("parts", [])
-            message_id = message.get("messageId", str(uuid4()))
+            message_dict: Final = _STR_KEY_DICT_ADAPTER.validate_python(message)
+            parts = _LIST_ADAPTER.validate_python(message_dict.get("parts", []))
+            message_id = message_dict.get("messageId", str(uuid4()))
             full_text = ""
             for part in parts:
-                if part.get("kind") == "text":
-                    full_text += part.get("text", "")
+                if (part_dict := _STR_KEY_DICT_ADAPTER.validate_python(part)).get("kind") == "text":
+                    full_text += _TEXT_ADAPTER.validate_python(part_dict.get("text", ""))
             return full_text, message_id, parts
 
         return "", str(uuid4()), []
 
     @staticmethod
     async def fake_streaming_from_response(
-        response_data: dict[str, Any],
+        response_data: Mapping[str, object],
         request_id: str,
         chunk_size: int = 50,
         delay_ms: int = 10,
-    ) -> AsyncIterator[dict[str, Any]]:
+    ) -> AsyncIterator[dict[str, object]]:
         """
         Convert a non-streaming A2A response into fake streaming chunks.
 
@@ -410,12 +434,12 @@ class PydanticAITransformation:
         full_text, message_id, parts = PydanticAITransformation._extract_response_text(response_data)
 
         # Extract input message from raw response for history
-        result: Final = response_data.get("result", {})
-        history: Final = result.get("history", [])
-        input_message = {}
+        result: Final = _STR_KEY_DICT_ADAPTER.validate_python(response_data.get("result", {}))
+        history: Final = _LIST_ADAPTER.validate_python(result.get("history", []))
+        input_message = _STR_KEY_DICT_ADAPTER.validate_python({})
         for msg in history:
-            if msg.get("role") == "user":
-                input_message = msg
+            if (msg_dict := _STR_KEY_DICT_ADAPTER.validate_python(msg)).get("role") == "user":
+                input_message = msg_dict
                 break
 
         # Generate IDs for streaming events
@@ -426,45 +450,49 @@ class PydanticAITransformation:
 
         # 1. Emit initial task event (kind: "task", status: "submitted")
         # Format matches A2ACompletionBridgeTransformation.create_task_event
-        task_event: Final = {
-            "jsonrpc": "2.0",
-            "id": request_id,
-            "result": {
-                "contextId": context_id,
-                "history": [
-                    {
-                        "contextId": context_id,
-                        "kind": "message",
-                        "messageId": input_message_id,
-                        "parts": input_message.get("parts", [{"kind": "text", "text": ""}]),
-                        "role": "user",
-                        "taskId": task_id,
-                    }
-                ],
-                "id": task_id,
-                "kind": "task",
-                "status": {
-                    "state": "submitted",
+        task_event: Final = _STR_KEY_DICT_ADAPTER.validate_python(
+            {
+                "jsonrpc": "2.0",
+                "id": request_id,
+                "result": {
+                    "contextId": context_id,
+                    "history": [
+                        {
+                            "contextId": context_id,
+                            "kind": "message",
+                            "messageId": input_message_id,
+                            "parts": input_message.get("parts", [{"kind": "text", "text": ""}]),
+                            "role": "user",
+                            "taskId": task_id,
+                        }
+                    ],
+                    "id": task_id,
+                    "kind": "task",
+                    "status": {
+                        "state": "submitted",
+                    },
                 },
-            },
-        }
+            }
+        )
         yield task_event
 
         # 2. Emit status update (kind: "status-update", status: "working")
         # Format matches A2ACompletionBridgeTransformation.create_status_update_event
-        working_event: Final = {
-            "jsonrpc": "2.0",
-            "id": request_id,
-            "result": {
-                "contextId": context_id,
-                "final": False,
-                "kind": "status-update",
-                "status": {
-                    "state": "working",
+        working_event: Final = _STR_KEY_DICT_ADAPTER.validate_python(
+            {
+                "jsonrpc": "2.0",
+                "id": request_id,
+                "result": {
+                    "contextId": context_id,
+                    "final": False,
+                    "kind": "status-update",
+                    "status": {
+                        "state": "working",
+                    },
+                    "taskId": task_id,
                 },
-                "taskId": task_id,
-            },
-        }
+            }
+        )
         yield working_event
 
         # Small delay to simulate processing
@@ -473,29 +501,32 @@ class PydanticAITransformation:
         # 3. Emit artifact update chunks (kind: "artifact-update")
         # Format matches A2ACompletionBridgeTransformation.create_artifact_update_event
         if full_text:
+            full_text_str: Final = _TEXT_ADAPTER.validate_python(full_text)
             # Split text into chunks
-            for i in range(0, len(full_text), chunk_size):
-                chunk_text = full_text[i : i + chunk_size]
-                is_last_chunk = (i + chunk_size) >= len(full_text)
+            for i in range(0, len(full_text_str), chunk_size):
+                chunk_text = full_text_str[i : i + chunk_size]
+                is_last_chunk = (i + chunk_size) >= len(full_text_str)
 
-                artifact_event = {
-                    "jsonrpc": "2.0",
-                    "id": request_id,
-                    "result": {
-                        "contextId": context_id,
-                        "kind": "artifact-update",
-                        "taskId": task_id,
-                        "artifact": {
-                            "artifactId": artifact_id,
-                            "parts": [
-                                {
-                                    "kind": "text",
-                                    "text": chunk_text,
-                                }
-                            ],
+                artifact_event = _STR_KEY_DICT_ADAPTER.validate_python(
+                    {
+                        "jsonrpc": "2.0",
+                        "id": request_id,
+                        "result": {
+                            "contextId": context_id,
+                            "kind": "artifact-update",
+                            "taskId": task_id,
+                            "artifact": {
+                                "artifactId": artifact_id,
+                                "parts": [
+                                    {
+                                        "kind": "text",
+                                        "text": chunk_text,
+                                    }
+                                ],
+                            },
                         },
-                    },
-                }
+                    }
+                )
                 yield artifact_event
 
                 # Add delay between chunks (except for last chunk)
@@ -503,19 +534,21 @@ class PydanticAITransformation:
                     await asyncio.sleep(delay_ms / 1000.0)
 
         # 4. Emit final status update (kind: "status-update", status: "completed", final: true)
-        completed_event: Final = {
-            "jsonrpc": "2.0",
-            "id": request_id,
-            "result": {
-                "contextId": context_id,
-                "final": True,
-                "kind": "status-update",
-                "status": {
-                    "state": "completed",
+        completed_event: Final = _STR_KEY_DICT_ADAPTER.validate_python(
+            {
+                "jsonrpc": "2.0",
+                "id": request_id,
+                "result": {
+                    "contextId": context_id,
+                    "final": True,
+                    "kind": "status-update",
+                    "status": {
+                        "state": "completed",
+                    },
+                    "taskId": task_id,
                 },
-                "taskId": task_id,
-            },
-        }
+            }
+        )
         yield completed_event
 
         verbose_logger.info("Pydantic AI: Fake streaming completed for request_id=%s", request_id)

litellm/batches/batch_utils.py

@@ -5,7 +5,8 @@ from typing import Any, Final, Literal
 
 import litellm
 from litellm._logging import verbose_logger
-from litellm.litellm_core_utils.llm_cost_calc.utils import _parse_prompt_tokens_details
+from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS
+from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details
 from litellm.types.llms.openai import Batch
 from litellm.types.utils import CallTypes, ModelInfo, Usage
 from litellm.utils import token_counter
@@ -58,6 +59,17 @@ async def _handle_completed_batch(
         model_name: Optional model name
         litellm_params: Optional litellm parameters containing credentials (api_key, api_base, etc.)
     """
+    # A completed batch whose request lines all failed has no output file - the
+    # results are written to a separate error_file_id and output_file_id is None.
+    # There is nothing to price or measure, so report an empty result set instead
+    # of calling _fetch_batch_output_file_content, which raises on a missing
+    # output file. Without this guard the logging worker crashes on every
+    # aretrieve_batch poll and the completed batch's zero-cost accounting is lost.
+    # The generic retrieval helper keeps raising for callers that explicitly ask
+    # for a missing output file.
+    if batch.output_file_id is None:
+        return 0.0, Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0), []
+
     file_content = await _fetch_batch_output_file_content(batch, custom_llm_provider, litellm_params=litellm_params)
 
     if (
@@ -101,7 +113,7 @@ def _iter_successful_output_line_stats(
             continue
         response_body = _get_response_from_batch_job_output_file(entry, custom_llm_provider)
         usage = _get_batch_job_usage_from_response_body(response_body, custom_llm_provider)
-        prompt_details = _parse_prompt_tokens_details(usage)
+        prompt_details = parse_prompt_tokens_details(usage)
         raw_model = response_body.get("model")
         response_model = raw_model if isinstance(raw_model, str) and raw_model else None
         if model_info is not None or custom_llm_provider in ("anthropic", "bedrock"):
@@ -295,7 +307,7 @@ def _extract_file_access_credentials(litellm_params: dict | None) -> dict:
 
     if litellm_params:
         # List of credential keys that should be passed to file operations
-        credential_keys: Final = [
+        credential_keys: Final = (
             "api_key",
             "api_base",
             "api_version",
@@ -309,7 +321,9 @@ def _extract_file_access_credentials(litellm_params: dict | None) -> dict:
             "bucket_name",
             "timeout",
             "max_retries",
-        ]
+            "_litellm_internal_model_credentials",
+            *AWS_CREDENTIAL_KWARGS_KEYS,
+        )
         for key in credential_keys:
             if key in litellm_params:
                 credentials[key] = litellm_params[key]

litellm/batches/main.py

@@ -22,6 +22,7 @@ from openai.types.batch import BatchRequestCounts
 
 import litellm
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.get_litellm_params import add_trusted_model_credentials_to_litellm_params
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.anthropic.batches.handler import AnthropicBatchesHandler
 from litellm.llms.azure.batches.handler import AzureBatchesAPI
@@ -527,6 +528,7 @@ def retrieve_batch(
             custom_llm_provider=custom_llm_provider,
             **kwargs,
         )
+        add_trusted_model_credentials_to_litellm_params(litellm_params, kwargs)
         if litellm_logging_obj is not None:
             litellm_logging_obj.update_from_kwargs(
                 kwargs=kwargs,
@@ -824,7 +826,7 @@ def list_batches(
 async def acancel_batch(
     batch_id: str,
     model: str | None = None,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] = "openai",
     metadata: dict[str, str] | None = None,
     extra_headers: dict[str, str] | None = None,
     extra_body: dict[str, str] | None = None,
@@ -870,7 +872,7 @@ async def acancel_batch(
 def cancel_batch(
     batch_id: str,
     model: str | None = None,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai"] | str = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock"] | str = "openai",
     metadata: dict[str, str] | None = None,
     extra_headers: dict[str, str] | None = None,
     extra_body: dict[str, str] | None = None,
@@ -991,9 +993,14 @@ def cancel_batch(
                 timeout=timeout,
                 max_retries=optional_params.max_retries,
             )
+        elif custom_llm_provider == "bedrock":
+            response = BedrockBatchesHandler.cancel_batch(
+                batch_id=batch_id,
+                **kwargs,
+            )
         else:
             raise litellm.exceptions.BadRequestError(
-                message=f"LiteLLM doesn't support {custom_llm_provider} for 'cancel_batch'. Only 'openai', 'azure', and 'vertex_ai' are supported.",
+                message=f"LiteLLM doesn't support {custom_llm_provider} for 'cancel_batch'. Only 'openai', 'azure', 'vertex_ai', and 'bedrock' are supported.",
                 model="n/a",
                 llm_provider=custom_llm_provider,
                 response=httpx.Response(

litellm/caching/caching.py

@@ -66,20 +66,7 @@ class Cache:
         default_in_memory_ttl: float | None = None,
         default_in_redis_ttl: float | None = None,
         similarity_threshold: float | None = None,
-        supported_call_types: list[CachingSupportedCallTypes] | None = [
-            "completion",
-            "acompletion",
-            "embedding",
-            "aembedding",
-            "atranscription",
-            "transcription",
-            "atext_completion",
-            "text_completion",
-            "arerank",
-            "rerank",
-            "responses",
-            "aresponses",
-        ],
+        supported_call_types: list[CachingSupportedCallTypes] | None = list(DEFAULT_CACHING_SUPPORTED_CALL_TYPES),
         # s3 Bucket, boto3 configuration
         azure_account_url: str | None = None,
         azure_blob_container: str | None = None,
@@ -927,20 +914,7 @@ def enable_cache(
     host: str | None = None,
     port: str | None = None,
     password: str | None = None,
-    supported_call_types: list[CachingSupportedCallTypes] | None = [
-        "completion",
-        "acompletion",
-        "embedding",
-        "aembedding",
-        "atranscription",
-        "transcription",
-        "atext_completion",
-        "text_completion",
-        "arerank",
-        "rerank",
-        "responses",
-        "aresponses",
-    ],
+    supported_call_types: list[CachingSupportedCallTypes] | None = list(DEFAULT_CACHING_SUPPORTED_CALL_TYPES),
     **kwargs,
 ):
     """
@@ -987,20 +961,7 @@ def update_cache(
     host: str | None = None,
     port: str | None = None,
     password: str | None = None,
-    supported_call_types: list[CachingSupportedCallTypes] | None = [
-        "completion",
-        "acompletion",
-        "embedding",
-        "aembedding",
-        "atranscription",
-        "transcription",
-        "atext_completion",
-        "text_completion",
-        "arerank",
-        "rerank",
-        "responses",
-        "aresponses",
-    ],
+    supported_call_types: list[CachingSupportedCallTypes] | None = list(DEFAULT_CACHING_SUPPORTED_CALL_TYPES),
     **kwargs,
 ):
     """

litellm/caching/caching_handler.py

@@ -18,8 +18,8 @@ import asyncio
 import datetime
 import inspect
 import time
-from collections.abc import AsyncGenerator, Callable, Generator
-from typing import TYPE_CHECKING, Any, Final, Optional
+from collections.abc import AsyncGenerator, AsyncIterator, Callable, Generator
+from typing import TYPE_CHECKING, Any, Final, Optional, TypeVar
 
 from pydantic import BaseModel
 
@@ -49,10 +49,15 @@ from litellm.types.utils import (
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+    from litellm.llms.anthropic.experimental_pass_through.messages.response_cache import (
+        AnthropicMessagesStreamCacheWriter,
+    )
     from litellm.types.utils import PromptTokensDetailsWrapper
 else:
     LiteLLMLoggingObj = Any
 
+_StreamResultT = TypeVar("_StreamResultT")
+
 
 from litellm.litellm_core_utils.core_helpers import (
     _get_parent_otel_span_from_kwargs,
@@ -106,7 +111,8 @@ def _should_defer_streaming_cache_hit_callbacks(*, kwargs: dict[str, Any]) -> bo
     When stream=True, do not run success callbacks at cache-hit time.
 
     Cached chat/text completion replay uses CustomStreamWrapper; cached Responses
-    replay uses CachedResponsesAPIStreamingIterator. Both invoke logging success
+    replay uses CachedResponsesAPIStreamingIterator; cached Anthropic Messages
+    replay uses CachedAnthropicMessagesStreamIterator. All invoke logging success
     handlers when the stream finishes; firing them here too would double-count
     spend and callback records.
     """
@@ -835,6 +841,18 @@ class LLMCachingHandler:
                 response_type="audio_transcription",
                 hidden_params=hidden_params,
             )
+        elif (
+            call_type == CallTypes.anthropic_messages.value or call_type == CallTypes.aanthropic_messages.value
+        ) and isinstance(cached_result, dict):
+            from litellm.llms.anthropic.experimental_pass_through.messages.response_cache import (
+                convert_cached_anthropic_messages_result,
+            )
+
+            cached_result = convert_cached_anthropic_messages_result(
+                cached_result=cached_result,
+                logging_obj=logging_obj,
+                kwargs=kwargs,
+            )
         elif (call_type == "aresponses" or call_type == "responses") and isinstance(cached_result, dict):
             use_chat_completion_cache: Final = _is_chat_completion_cached_dict(cached_result)
             if use_chat_completion_cache:
@@ -1031,6 +1049,26 @@ class LLMCachingHandler:
             and (kwargs.get("cache", {}).get("no-store", False) is not True)
         )
 
+    def wrap_streaming_result_for_cache(
+        self, result: _StreamResultT, call_type: str
+    ) -> "_StreamResultT | AnthropicMessagesStreamCacheWriter":
+        if call_type not in (
+            CallTypes.anthropic_messages.value,
+            CallTypes.aanthropic_messages.value,
+        ):
+            return result
+        if litellm.cache is None or not self._should_store_result_in_cache(
+            original_function=self.original_function, kwargs=self.request_kwargs
+        ):
+            return result
+        if not isinstance(result, AsyncIterator):
+            return result
+        from litellm.llms.anthropic.experimental_pass_through.messages.response_cache import (
+            AnthropicMessagesStreamCacheWriter,
+        )
+
+        return AnthropicMessagesStreamCacheWriter(stream=result, caching_handler=self)
+
     def _is_call_type_supported_by_cache(
         self,
         original_function: Callable,

litellm/caching/redis_cache.py

@@ -49,7 +49,7 @@ if TYPE_CHECKING:
     cluster_pipeline = ClusterPipeline
     async_redis_client = Redis
     async_redis_cluster_client = RedisCluster
-    Span = _Span | Any
+    Span = _Span
 else:
     pipeline = Any
     cluster_pipeline = Any
@@ -625,7 +625,11 @@ class RedisCache(BaseCache):
             f"{self.namespace}-{hashlib.sha256(script.encode()).hexdigest()[:16]}"
         )
 
-        async def run_script(keys: Sequence[str], args: Sequence[Any], client: Any = None) -> Any:
+        async def run_script(
+            keys: Sequence[str],
+            args: Sequence[str | bytes | int | float],
+            client: object = None,
+        ) -> object:
             async def execute() -> object:
                 executor: Callable[..., Awaitable[Any]] | None = litellm.in_memory_llm_clients_cache.get_cache(
                     key=script_cache_key
@@ -650,7 +654,11 @@ class RedisCache(BaseCache):
         if hasattr(_redis_client, "register_script"):
             registered_script: Final = _redis_client.register_script(script)
 
-            async def standalone_executor(keys: Sequence[str], args: Sequence[Any], client: Any = None) -> Any:
+            async def standalone_executor(
+                keys: Sequence[str],
+                args: Sequence[str | bytes | int | float],
+                client: object = None,
+            ) -> object:
                 namespaced_keys: Final = tuple(self.check_and_fix_namespace(key=key) for key in keys)
                 return await registered_script(keys=namespaced_keys, args=args, client=client)
 
@@ -659,7 +667,11 @@ class RedisCache(BaseCache):
         if hasattr(_redis_client, "script_load"):
             script_sha: Final = _redis_client.script_load(script)
 
-            async def cluster_executor(keys: Sequence[str], args: Sequence[Any], client: Any = None) -> Any:
+            async def cluster_executor(
+                keys: Sequence[str],
+                args: Sequence[str | bytes | int | float],
+                client: object = None,
+            ) -> object:
                 namespaced_keys: Final = tuple(self.check_and_fix_namespace(key=key) for key in keys)
                 return await _redis_client.evalsha(script_sha, len(namespaced_keys), *namespaced_keys, *args)
 
@@ -757,7 +769,7 @@ class RedisCache(BaseCache):
     async def _pipeline_helper(
         self,
         pipe: pipeline | cluster_pipeline,
-        cache_list: list[tuple[Any, Any]],
+        cache_list: Sequence[tuple[str, object]],
         ttl: float | None,
     ) -> list:
         """
@@ -783,7 +795,9 @@ class RedisCache(BaseCache):
         return results
 
     @_redis_circuit_breaker_guard
-    async def async_set_cache_pipeline(self, cache_list: list[tuple[Any, Any]], ttl: float | None = None, **kwargs):
+    async def async_set_cache_pipeline(
+        self, cache_list: Sequence[tuple[str, object]], ttl: float | None = None, **kwargs
+    ):
         """
         Use Redis Pipelines for bulk write operations
         """
@@ -795,7 +809,7 @@ class RedisCache(BaseCache):
         start_time: Final = time.time()
 
         print_verbose(f"Set Async Redis Cache: key list: {cache_list}\nttl={ttl}, redis_version={self.redis_version}")
-        cache_value: Final[Any] = None
+        cache_value: Final = None
         try:
             async with _redis_client.pipeline(transaction=False) as pipe:
                 results: Final = await self._pipeline_helper(pipe, cache_list, ttl)
@@ -1074,7 +1088,7 @@ class RedisCache(BaseCache):
             # NON blocking - notify users Redis is throwing an exception
             verbose_logger.error("litellm.caching.caching: get() - Got exception from REDIS: ", e)
 
-    def _run_redis_mget_operation(self, keys: list[str]) -> list[Any]:
+    def _run_redis_mget_operation(self, keys: list[str]) -> Sequence[bytes | str | None]:
         """
         Wrapper to call `mget` on the redis client
 
@@ -1082,7 +1096,7 @@ class RedisCache(BaseCache):
         """
         return self.redis_client.mget(keys=keys)
 
-    async def _async_run_redis_mget_operation(self, keys: list[str]) -> list[Any]:
+    async def _async_run_redis_mget_operation(self, keys: list[str]) -> Sequence[bytes | str | None]:
         """
         Wrapper to call `mget` on the redis client
 
@@ -1115,7 +1129,7 @@ class RedisCache(BaseCache):
                 cache_key = self.check_and_fix_namespace(key=cache_key or "")
                 _keys.append(cache_key)
             start_time: Final = time.time()
-            results: Final[list] = self._run_redis_mget_operation(keys=_keys)
+            results: Final = self._run_redis_mget_operation(keys=_keys)
             end_time: Final = time.time()
             _duration: Final = end_time - start_time
             self.service_logger_obj.service_success_hook(
@@ -1522,7 +1536,7 @@ class RedisCache(BaseCache):
     async def async_rpush(
         self,
         key: str,
-        values: list[Any],
+        values: Sequence[str | bytes | int | float],
         parent_otel_span: Span | None = None,
         **kwargs,
     ) -> int:
@@ -1572,7 +1586,7 @@ class RedisCache(BaseCache):
     async def _pipeline_rpush_helper(
         self,
         pipe: pipeline,
-        rpush_list: list[RedisPipelineRpushOperation],
+        rpush_list: Sequence[RedisPipelineRpushOperation],
     ) -> list[int]:
         """Helper function for pipeline rpush operations"""
         for rpush_op in rpush_list:
@@ -1588,7 +1602,7 @@ class RedisCache(BaseCache):
     @_redis_circuit_breaker_guard
     async def async_rpush_pipeline(
         self,
-        rpush_list: list[RedisPipelineRpushOperation],
+        rpush_list: Sequence[RedisPipelineRpushOperation],
     ) -> list[int]:
         """
         Use Redis Pipelines for bulk RPUSH operations

litellm/caching/redis_semantic_cache.py

@@ -13,6 +13,7 @@ import ast
 import asyncio
 import json
 import os
+from collections.abc import Callable, Mapping
 from typing import Any, Final, cast
 
 import litellm
@@ -47,7 +48,7 @@ class RedisSemanticCache(BaseCache):
         similarity_threshold: float | None = None,
         embedding_model: str = "text-embedding-ada-002",
         index_name: str | None = None,
-        **kwargs,
+        **kwargs: object,
     ):
         """
         Initialize the Redis Semantic Cache.
@@ -150,11 +151,11 @@ class RedisSemanticCache(BaseCache):
 
     def _init_semantic_cache(
         self,
-        semantic_cache_cls: Any,
+        semantic_cache_cls: Callable[..., object],
         index_name: str,
         redis_url: str,
-        cache_vectorizer: Any,
-    ) -> Any:
+        cache_vectorizer: object,
+    ) -> object:
         def _is_schema_mismatch(exc: ValueError) -> bool:
             error_message: Final = str(exc).lower()
             return any(phrase in error_message for phrase in ("schema does not match", "index schema"))
@@ -206,12 +207,12 @@ class RedisSemanticCache(BaseCache):
     def _get_cache_filters(self, key: str) -> dict[str, str]:
         return {self.CACHE_KEY_FIELD_NAME: str(key)}
 
-    def _get_cache_key_filter_expression(self, key: str) -> Any:
+    def _get_cache_key_filter_expression(self, key: str) -> object:
         from redisvl.query.filter import Tag
 
         return Tag(self.CACHE_KEY_FIELD_NAME) == str(key)
 
-    def _cache_hit_matches_key(self, cache_hit: dict[str, Any], key: str) -> bool:
+    def _cache_hit_matches_key(self, cache_hit: Mapping[str, object], key: str) -> bool:
         # Pre-isolation entries with no ``litellm_cache_key`` field cannot be
         # safely reassigned to a caller's scope and are treated as misses.
         cached_key = cache_hit.get(self.CACHE_KEY_FIELD_NAME)
@@ -297,7 +298,7 @@ class RedisSemanticCache(BaseCache):
                     return
 
     @staticmethod
-    def _coerce_response_input_value(value: Any) -> Any:
+    def _coerce_response_input_value(value: object) -> object:
         model_dump: Final = getattr(value, "model_dump", None)
         if callable(model_dump):
             return model_dump()
@@ -340,7 +341,7 @@ class RedisSemanticCache(BaseCache):
             )
         return embedding_response["data"][0]["embedding"]
 
-    def _get_cache_logic(self, cached_response: Any) -> Any:
+    def _get_cache_logic(self, cached_response: Any) -> object:
         """
         Process the cached response to prepare it for use.
 
@@ -369,7 +370,7 @@ class RedisSemanticCache(BaseCache):
 
         return cached_response
 
-    def set_cache(self, key: str, value: Any, **kwargs) -> None:
+    def set_cache(self, key: str, value: object, **kwargs) -> None:
         """
         Store a value in the semantic cache.
 
@@ -405,7 +406,7 @@ class RedisSemanticCache(BaseCache):
         except Exception as e:
             print_verbose(f"Error setting {value_str or value} in the Redis semantic cache: {e}")
 
-    def get_cache(self, key: str, **kwargs) -> Any:
+    def get_cache(self, key: str, **kwargs) -> object:
         """
         Retrieve a semantically similar cached response.
 
@@ -428,7 +429,7 @@ class RedisSemanticCache(BaseCache):
             # Check the cache for semantically similar prompts in this exact
             # LiteLLM cache-key scope.
             prompt_embedding: Final = self._get_embedding(prompt, metadata=kwargs.get("metadata"))
-            check_kwargs: Final[dict[str, Any]] = {
+            check_kwargs: Final[Mapping[str, object]] = {
                 "prompt": prompt,
                 "vector": prompt_embedding,
                 "filter_expression": self._get_cache_key_filter_expression(key),
@@ -508,7 +509,7 @@ class RedisSemanticCache(BaseCache):
             print_verbose(f"Error generating async embedding: {e}")
             raise ValueError(f"Failed to generate embedding: {e}") from e
 
-    async def async_set_cache(self, key: str, value: Any, **kwargs) -> None:
+    async def async_set_cache(self, key: str, value: object, **kwargs) -> None:
         """
         Asynchronously store a value in the semantic cache.
 
@@ -548,7 +549,7 @@ class RedisSemanticCache(BaseCache):
         except Exception as e:
             print_verbose(f"Error in async_set_cache: {e}")
 
-    async def async_get_cache(self, key: str, **kwargs) -> Any:
+    async def async_get_cache(self, key: str, **kwargs) -> object:
         """
         Asynchronously retrieve a semantically similar cached response.
 
@@ -573,7 +574,7 @@ class RedisSemanticCache(BaseCache):
 
             # Check the cache for semantically similar prompts in this exact
             # LiteLLM cache-key scope.
-            check_kwargs: Final[dict[str, Any]] = {
+            check_kwargs: Final[Mapping[str, object]] = {
                 "prompt": prompt,
                 "vector": prompt_embedding,
                 "filter_expression": self._get_cache_key_filter_expression(key),
@@ -615,7 +616,7 @@ class RedisSemanticCache(BaseCache):
             print_verbose(f"Error in async_get_cache: {e}")
             kwargs.setdefault("metadata", {})["semantic-similarity"] = 0.0
 
-    async def _index_info(self) -> dict[str, Any]:
+    async def _index_info(self) -> Mapping[str, object]:
         """
         Get information about the Redis index.
 
@@ -625,7 +626,7 @@ class RedisSemanticCache(BaseCache):
         aindex: Final = await self.llmcache._get_async_index()
         return await aindex.info()
 
-    async def async_set_cache_pipeline(self, cache_list: list[tuple[str, Any]], **kwargs) -> None:
+    async def async_set_cache_pipeline(self, cache_list: list[tuple[str, Any]], **kwargs: object) -> None:
         """
         Asynchronously store multiple values in the semantic cache.
 

litellm/completion_extras/litellm_responses_transformation/handler.py

@@ -22,6 +22,7 @@ class ResponsesToCompletionBridgeHandlerInputKwargs(TypedDict):
     model_response: "ModelResponse"
     logging_obj: "LiteLLMLoggingObj"
     custom_llm_provider: str
+    encoding: object
 
 
 class ResponsesToCompletionBridgeHandler:
@@ -102,35 +103,37 @@ class ResponsesToCompletionBridgeHandler:
         from litellm import LiteLLMLoggingObj
         from litellm.types.utils import ModelResponse
 
-        model: Final = kwargs.get("model")
+        typed_kwargs: Final[dict[str, object]] = kwargs
+
+        model: Final = typed_kwargs.get("model")
         if model is None or not isinstance(model, str):
             raise ValueError("model is required")
 
-        custom_llm_provider: Final = kwargs.get("custom_llm_provider")
+        custom_llm_provider: Final = typed_kwargs.get("custom_llm_provider")
         if custom_llm_provider is None or not isinstance(custom_llm_provider, str):
             raise ValueError("custom_llm_provider is required")
 
-        messages: Final = kwargs.get("messages")
+        messages: Final = typed_kwargs.get("messages")
         if messages is None or not isinstance(messages, list):
             raise ValueError("messages is required")
 
-        optional_params: Final = kwargs.get("optional_params")
+        optional_params: Final = typed_kwargs.get("optional_params")
         if optional_params is None or not isinstance(optional_params, dict):
             raise ValueError("optional_params is required")
 
-        litellm_params: Final = kwargs.get("litellm_params")
+        litellm_params: Final = typed_kwargs.get("litellm_params")
         if litellm_params is None or not isinstance(litellm_params, dict):
             raise ValueError("litellm_params is required")
 
-        headers: Final = kwargs.get("headers")
+        headers: Final = typed_kwargs.get("headers")
         if headers is None or not isinstance(headers, dict):
             raise ValueError("headers is required")
 
-        model_response: Final = kwargs.get("model_response")
+        model_response: Final = typed_kwargs.get("model_response")
         if model_response is None or not isinstance(model_response, ModelResponse):
             raise ValueError("model_response is required")
 
-        logging_obj: Final = kwargs.get("logging_obj")
+        logging_obj: Final = typed_kwargs.get("logging_obj")
         if logging_obj is None or not isinstance(logging_obj, LiteLLMLoggingObj):
             raise ValueError("logging_obj is required")
 
@@ -143,6 +146,7 @@ class ResponsesToCompletionBridgeHandler:
             model_response=model_response,
             logging_obj=logging_obj,
             custom_llm_provider=custom_llm_provider,
+            encoding=typed_kwargs.get("encoding"),
         )
 
     def completion(
@@ -205,7 +209,7 @@ class ResponsesToCompletionBridgeHandler:
                 messages=messages,
                 optional_params=optional_params,
                 litellm_params=litellm_params,
-                encoding=kwargs.get("encoding"),
+                encoding=validated_kwargs["encoding"],
                 api_key=kwargs.get("api_key"),
                 json_mode=kwargs.get("json_mode"),
             )
@@ -230,7 +234,7 @@ class ResponsesToCompletionBridgeHandler:
                 messages=messages,
                 optional_params=optional_params,
                 litellm_params=litellm_params,
-                encoding=kwargs.get("encoding"),
+                encoding=validated_kwargs["encoding"],
                 api_key=kwargs.get("api_key"),
                 json_mode=kwargs.get("json_mode"),
             )
@@ -303,7 +307,7 @@ class ResponsesToCompletionBridgeHandler:
                 messages=messages,
                 optional_params=optional_params,
                 litellm_params=litellm_params,
-                encoding=kwargs.get("encoding"),
+                encoding=validated_kwargs["encoding"],
                 api_key=kwargs.get("api_key"),
                 json_mode=kwargs.get("json_mode"),
             )
@@ -328,7 +332,7 @@ class ResponsesToCompletionBridgeHandler:
                 messages=messages,
                 optional_params=optional_params,
                 litellm_params=litellm_params,
-                encoding=kwargs.get("encoding"),
+                encoding=validated_kwargs["encoding"],
                 api_key=kwargs.get("api_key"),
                 json_mode=kwargs.get("json_mode"),
             )

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -4,8 +4,8 @@ Handler for transforming /chat/completions api requests to litellm.responses req
 
 import json
 import os
-from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping
-from typing import TYPE_CHECKING, Any, Final, Literal, Union, cast
+from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
+from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, Union, cast
 
 from openai.types.responses.custom_tool_param import CustomToolParam
 from openai.types.responses.response_input_param import (
@@ -45,6 +45,9 @@ from litellm.types.utils import GenericStreamingChunk, ModelResponseStream
 
 if TYPE_CHECKING:
     from openai.types.responses import ResponseInputImageParam
+    from openai.types.responses.response_text_config_param import (
+        ResponseTextConfigParam as ResponseText,
+    )
     from pydantic import BaseModel
 
     from litellm import LiteLLMLoggingObj, ModelResponse
@@ -57,6 +60,19 @@ if TYPE_CHECKING:
         ChatCompletionThinkingBlock,
         OpenAIMessageContentListBlock,
     )
+    from litellm.types.utils import Choices
+
+
+class _ReasoningSummaryText(TypedDict):
+    type: str
+    text: str
+
+
+class _BuiltReasoningItem(TypedDict):
+    type: Literal["reasoning"]
+    id: str
+    encrypted_content: str | None
+    summary: Sequence[_ReasoningSummaryText]
 
 
 def _get_reasoning_items(
@@ -72,13 +88,13 @@ def _get_reasoning_items(
 def _build_reasoning_item(
     item_id: str,
     encrypted_content: str | None,
-    summary_raw: Any,
-) -> dict[str, Any]:
+    summary_raw: Iterable[object] | None,
+) -> _BuiltReasoningItem:
     """Build a ChatCompletionReasoningItem-shaped dict from raw response data.
 
     Handles both pydantic objects (attribute access) and plain dicts.
     """
-    summary: Final[list[dict[str, Any]]] = []
+    summary: Final[list[_ReasoningSummaryText]] = []
     for s in summary_raw or []:
         if isinstance(s, dict):
             summary.append({"type": s.get("type", "summary_text"), "text": s.get("text", "")})
@@ -98,7 +114,7 @@ def _build_reasoning_item(
 
 
 class _ChatToolCallDict(ChatCompletionToolCallChunk, total=False):
-    provider_specific_fields: Mapping[str, Any]
+    provider_specific_fields: Mapping[str, object]
 
 
 def _tool_call_dict_from_output_item(item: Mapping[str, Any], index: int) -> _ChatToolCallDict:
@@ -142,10 +158,10 @@ def _flat_responses_tool_choice(choice_type: str, name: str) -> ToolChoiceFuncti
 
 
 def _reasoning_item_to_response_input(
-    r_item: ChatCompletionReasoningItem | dict[str, Any],
-) -> dict[str, Any]:
+    r_item: ChatCompletionReasoningItem,
+) -> dict[str, object]:
     """Convert a stored ChatCompletionReasoningItem back to a Responses API input item."""
-    r_input: Final[dict[str, Any]] = {
+    r_input: Final[dict[str, object]] = {
         "type": "reasoning",
         "id": r_item.get("id") or f"rs_{id(r_item)}",
         # summary is always required by the Responses API, even when empty
@@ -181,7 +197,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                 return _flat_responses_tool_choice(choice_type, nested_name)
         return tool_choice
 
-    def _handle_raw_dict_response_item(self, item: dict[str, Any], index: int) -> tuple[Any | None, int]:
+    def _handle_raw_dict_response_item(self, item: dict[str, Any], index: int) -> tuple["Choices | None", int]:
         """
         Handle raw dict response items from Responses API (e.g., GPT-5 Codex format).
 
@@ -228,8 +244,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
 
     def convert_chat_completion_messages_to_responses_api(
         self, messages: list["AllMessageValues"]
-    ) -> tuple[list[Any], str | None]:
-        input_items: Final[list[Any]] = []
+    ) -> tuple[list[object], str | None]:
+        input_items: Final[list[object]] = []
         instructions: str | None = None
         custom_tool_call_ids: Final = frozenset(
             tool_call["id"]
@@ -270,7 +286,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                 # Convert tool message to function call output format
                 # The Responses API expects 'output' to be a list with input_text/input_image types
                 # Using list format for consistency across text and multimodal content
-                tool_output: list[dict[str, Any]]
+                tool_output: list[dict[str, object]]
                 if content is None:
                     tool_output = []
                 elif isinstance(content, str):
@@ -308,7 +324,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                     function = tool_call.get("function")
                     custom = tool_call.get("custom")
                     if function:
-                        input_tool_call: dict[str, Any] = {
+                        input_tool_call: dict[str, object] = {
                             "type": "function_call",
                             "call_id": tool_call["id"],
                         }
@@ -376,15 +392,15 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             elif key == "web_search_options":
                 self._add_web_search_tool(responses_api_request, value)
 
-    def _build_sanitized_litellm_params(self, litellm_params: dict) -> dict[str, Any]:
+    def _build_sanitized_litellm_params(self, litellm_params: dict) -> dict[str, object]:
         """Build sanitized litellm_params with merged metadata."""
         responses_optional_param_keys: Final = set(ResponsesAPIOptionalRequestParams.__annotations__.keys())
-        sanitized: Final[dict[str, Any]] = {
+        sanitized: Final[dict[str, object]] = {
             key: value for key, value in litellm_params.items() if key not in responses_optional_param_keys
         }
         legacy_metadata: Final = litellm_params.get("metadata")
         existing_litellm_metadata: Final = litellm_params.get("litellm_metadata")
-        merged_litellm_metadata: Final[dict[str, Any]] = {}
+        merged_litellm_metadata: Final[dict[str, object]] = {}
         if isinstance(legacy_metadata, dict):
             merged_litellm_metadata.update(legacy_metadata)
         if isinstance(existing_litellm_metadata, dict):
@@ -424,7 +440,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         litellm_params: dict,
         headers: dict,
         litellm_logging_obj: "LiteLLMLoggingObj",
-        client: Any | None = None,
+        client: object | None = None,
     ) -> dict:
         (
             input_items,
@@ -498,9 +514,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
 
     @staticmethod
     def _convert_response_output_to_choices(
-        output_items: list[Any],
-        handle_raw_dict_callback: Callable | None = None,
-    ) -> list[Any]:
+        output_items: Sequence[object],
+        handle_raw_dict_callback: Callable[..., tuple["Choices | None", int]] | None = None,
+    ) -> list["Choices"]:
         """
         Convert Responses API output items to chat completion choices.
 
@@ -529,11 +545,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         choices: Final[list[Choices]] = []
         index = 0
         reasoning_content: str | None = None
-        pending_reasoning_item: dict[str, Any] | None = None
+        pending_reasoning_item: _BuiltReasoningItem | None = None
 
         # Collect all tool calls to put them in a single choice
         # (Chat Completions API expects all tool calls in one message)
-        accumulated_tool_calls: Final[list[dict[str, Any]]] = []
+        accumulated_tool_calls: Final[list[Mapping[str, object]]] = []
         tool_call_index = 0
 
         for item in output_items:
@@ -640,7 +656,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         return choices
 
     @classmethod
-    def _extract_output_from_completed_event(cls, parsed_chunk: dict[str, Any]) -> list[dict[str, Any]] | None:
+    def _extract_output_from_completed_event(cls, parsed_chunk: Mapping[str, object]) -> list[dict[str, object]] | None:
         response_payload: Final = parsed_chunk.get("response")
         if not isinstance(response_payload, dict):
             return None
@@ -650,12 +666,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         return cast(list[dict[str, Any]], response_output)
 
     @classmethod
-    def _recover_output_items_from_raw_sse(cls, raw_sse: str | None) -> list[dict[str, Any]]:
+    def _recover_output_items_from_raw_sse(cls, raw_sse: str | None) -> list[dict[str, object]]:
         if not raw_sse or not isinstance(raw_sse, str):
             return []
 
-        recovered_output_items: Final[dict[int, dict[str, Any]]] = {}
-        recovered_text_only_items: Final[dict[int, dict[str, Any]]] = {}
+        recovered_output_items: Final[dict[int, dict[str, object]]] = {}
+        recovered_text_only_items: Final[dict[int, dict[str, object]]] = {}
 
         for chunk in raw_sse.splitlines():
             parsed_chunk = parse_sse_json_chunk(chunk)
@@ -690,7 +706,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         # but text-only items at indices without a matching OUTPUT_ITEM_DONE
         # must still be preserved (e.g. multi-output responses where some
         # indices only emitted OUTPUT_TEXT_DONE).
-        merged_items: Final[dict[int, dict[str, Any]]] = {**recovered_text_only_items}
+        merged_items: Final[dict[int, dict[str, object]]] = {**recovered_text_only_items}
         merged_items.update(recovered_output_items)
 
         if merged_items:
@@ -699,7 +715,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         return []
 
     @classmethod
-    def _recover_output_items_from_logging(cls, logging_obj: "LiteLLMLoggingObj") -> list[dict[str, Any]]:
+    def _recover_output_items_from_logging(cls, logging_obj: "LiteLLMLoggingObj") -> list[dict[str, object]]:
         model_call_details: Final = getattr(logging_obj, "model_call_details", {}) or {}
         original_response: Final = model_call_details.get("original_response")
         return cls._recover_output_items_from_raw_sse(original_response)
@@ -714,7 +730,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         messages: list["AllMessageValues"],
         optional_params: dict,
         litellm_params: dict,
-        encoding: Any,
+        encoding: object,
         api_key: str | None = None,
         json_mode: bool | None = None,
     ) -> "ModelResponse":
@@ -788,7 +804,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
     ) -> BaseModelResponseIterator:
         return OpenAiResponsesToChatCompletionStreamIterator(streaming_response, sync_stream, json_mode)
 
-    def _convert_content_str_to_input_text(self, content: str, role: str) -> dict[str, Any]:
+    def _convert_content_str_to_input_text(self, content: str, role: str) -> dict[str, object]:
         if role == "user" or role == "system" or role == "tool":
             return {"type": "input_text", "text": content}
         else:
@@ -825,13 +841,13 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
     def _convert_content_to_responses_format(
         self,
         content: str
-        | list[Any]
+        | list[object]
         | Iterable[
             Union["OpenAIMessageContentListBlock", "ChatCompletionThinkingBlock", "ChatCompletionRedactedThinkingBlock"]
         ]
         | None,
         role: str,
-    ) -> list[dict[str, Any]]:
+    ) -> list[dict[str, object]]:
         """Convert chat completion content to responses API format"""
         from litellm.types.llms.openai import ChatCompletionImageObject
 
@@ -973,7 +989,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
 
         return optional_params
 
-    def _map_reasoning_effort(self, reasoning_effort: str | dict[str, Any]) -> Reasoning | None:
+    def _map_reasoning_effort(self, reasoning_effort: str | Reasoning) -> Reasoning | None:
         # If dict is passed, convert it directly to Reasoning object
         if isinstance(reasoning_effort, dict):
             return Reasoning(**reasoning_effort)
@@ -1006,7 +1022,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
     def _add_web_search_tool(
         self,
         responses_api_request: ResponsesAPIOptionalRequestParams,
-        web_search_options: Any,
+        web_search_options: object,
     ) -> None:
         """
         Add web search tool to responses API request.
@@ -1024,14 +1040,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             tools = []
             responses_api_request["tools"] = tools
 
-        web_search_tool: Final[dict[str, Any]] = {"type": "web_search"}
+        web_search_tool: Final[dict[str, object]] = {"type": "web_search"}
         if isinstance(web_search_options, dict):
             web_search_tool.update(web_search_options)
 
         # Cast to Any to match the expected union type for tools list items
         tools.append(cast(Any, web_search_tool))
 
-    def _transform_response_format_to_text_format(self, response_format: dict[str, Any] | Any) -> dict[str, Any] | None:
+    def _transform_response_format_to_text_format(self, response_format: object) -> "ResponseText | None":
         """
         Transform Chat Completion response_format parameter to Responses API text.format parameter.
 
@@ -1130,7 +1146,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
 
 
 class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
-    def __init__(self, streaming_response, sync_stream: bool, json_mode: bool | None = False):
+    def __init__(
+        self,
+        streaming_response: Union[Iterator[str], AsyncIterator[str], "ModelResponse", "BaseModel"],
+        sync_stream: bool,
+        json_mode: bool | None = False,
+    ):
         super().__init__(streaming_response, sync_stream, json_mode)
         self._chat_completion_id: str | None = None
         self._tool_call_index_map: dict[int, int] = {}  # mutable-ok: per-stream accumulator state
@@ -1387,7 +1408,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
             finish_reason: Final = "tool_calls" if has_function_calls else "stop"
 
             # Extract reasoning items with encrypted_content for round-tripping
-            completed_reasoning_items: list[dict[str, Any]] | None = None
+            completed_reasoning_items: list[_BuiltReasoningItem] | None = None
             for item in output_items:
                 if not isinstance(item, dict) or item.get("type") != "reasoning":
                     continue
@@ -1439,7 +1460,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
             ]
         )
 
-    def chunk_parser(self, chunk: dict) -> "ModelResponseStream":
+    def chunk_parser(self, chunk: dict[str, object]) -> "ModelResponseStream":
         """
         Parse a Responses API streaming chunk and convert to OpenAI format.
 

litellm/constants.py

@@ -7,6 +7,7 @@ from litellm.litellm_core_utils.env_utils import get_env_int, get_env_int_or_non
 DEFAULT_HEALTH_CHECK_PROMPT: Final = str(os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm"))
 AZURE_DEFAULT_RESPONSES_API_VERSION: Final = str(os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview"))
 ROUTER_MAX_FALLBACKS: Final = int(os.getenv("ROUTER_MAX_FALLBACKS", 5))
+ROUTER_FALLBACK_ERROR_DETAIL_MAX_CHARS: Final = 2000
 DEFAULT_BATCH_SIZE: Final = int(os.getenv("DEFAULT_BATCH_SIZE", 512))
 DEFAULT_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_FLUSH_INTERVAL_SECONDS", 5))
 DEFAULT_S3_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_S3_FLUSH_INTERVAL_SECONDS", 10))
@@ -140,6 +141,8 @@ LITELLM_UI_ALLOW_HEADERS: Final = [
     "x-litellm-semantic-filter",
     "x-litellm-semantic-filter-tools",
     "x-litellm-adaptive-router-model",
+    "x-litellm-applied-guardrails",
+    "x-litellm-guardrail-scan-id",
 ]
 
 # Gemini model-specific minimal thinking budget constants
@@ -279,6 +282,7 @@ TOOL_POLICY_CACHE_TTL_SECONDS: Final = int(os.getenv("TOOL_POLICY_CACHE_TTL_SECO
 GUARDRAIL_SCANNED_MESSAGES_CACHE_TTL_SECONDS: Final = int(
     os.getenv("GUARDRAIL_SCANNED_MESSAGES_CACHE_TTL_SECONDS", 24 * 60 * 60)
 )
+BEDROCK_APPLY_GUARDRAIL_CHUNK_BUDGET_CHARS: Final = 25_000
 # Aggregation threshold: default to 80% of the asyncio queue maxsize so the check can always trigger.
 # Must be < LITELLM_ASYNCIO_QUEUE_MAXSIZE; if set higher the aggregation logic will never fire.
 MAX_SIZE_IN_MEMORY_QUEUE: Final = int(os.getenv("MAX_SIZE_IN_MEMORY_QUEUE", int(LITELLM_ASYNCIO_QUEUE_MAXSIZE * 0.8)))
@@ -470,6 +474,8 @@ EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE: Final = float(
 ### ANTHROPIC CONSTANTS ###
 ANTHROPIC_TOKEN_COUNTING_BETA_VERSION = os.getenv("ANTHROPIC_TOKEN_COUNTING_BETA_VERSION", "token-counting-2024-11-01")
 ANTHROPIC_SKILLS_API_BETA_VERSION: Final = "skills-2025-10-02"
+ANTHROPIC_BATCHES_ROUTE: Final = "/v1/messages/batches"
+VERTEX_BATCH_PREDICTION_JOBS_ROUTE: Final = "batchPredictionJobs"
 ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES: Final = {
     "low": 1,
     "medium": 5,
@@ -1320,6 +1326,8 @@ X_LITELLM_DISABLE_CALLBACKS: Final = "x-litellm-disable-callbacks"
 LITELLM_METADATA_FIELD: Final = "litellm_metadata"
 OLD_LITELLM_METADATA_FIELD: Final = "metadata"
 RETURN_RAW_MODEL_NAME_METADATA_KEY: Final = "_complexity_router_return_raw_model_name"
+SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY: Final = "_session_deployment_affinity_ttl"
+CONSUMED_REQUEST_TAGS_METADATA_KEY: Final = "_consumed_request_tags"
 INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
 LITELLM_TRUNCATED_PAYLOAD_FIELD: Final = "litellm_truncated"
 LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE: Final = (
@@ -1475,21 +1483,31 @@ CLOUDZERO_MAX_FETCHED_DATA_RECORDS: Final = int(os.getenv("CLOUDZERO_MAX_FETCHED
 SPEND_LOG_CLEANUP_JOB_NAME: Final = "spend_log_cleanup"
 KEY_ROTATION_JOB_NAME: Final = "litellm_key_rotation_job"
 EXPIRED_UI_SESSION_KEY_CLEANUP_JOB_NAME: Final = "litellm_expired_ui_session_key_cleanup_job"
+WEEKLY_SPEND_REPORT_JOB_ID: Final = "weekly_spend_report_job"
+MONTHLY_SPEND_REPORT_JOB_ID: Final = "monthly_spend_report_job"
+PROMETHEUS_FALLBACK_STATS_JOB_ID: Final = "prometheus_fallback_stats_job"
+SLACK_DAILY_REPORT_LOCK_ID: Final = "slack_daily_report"
 SPEND_LOG_RUN_LOOPS: Final = int(os.getenv("SPEND_LOG_RUN_LOOPS", 500))
 SPEND_LOG_CLEANUP_BATCH_SIZE: Final = int(os.getenv("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000))
 SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES = int(os.getenv("SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3))
 SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS: Final = float(
     os.getenv("SPEND_LOG_CLEANUP_BATCH_FAILURE_BACKOFF_SECONDS", 0.5)
 )
+SPEND_LOG_CLEANUP_RUN_BUDGET_SECONDS: Final = float(os.getenv("SPEND_LOG_CLEANUP_RUN_BUDGET_SECONDS", "300"))
+SPEND_LOG_CLEANUP_BATCH_TIMEOUT_SECONDS: Final = float(os.getenv("SPEND_LOG_CLEANUP_BATCH_TIMEOUT_SECONDS", "30"))
+SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP: Final = int(os.getenv("SPEND_LOG_CLEANUP_REMAINING_COUNT_CAP", "100000"))
 TOOL_SPEND_TOP_TOOLS: Final = 100
 SPEND_LOG_PARTITION_INTERVAL: Final = os.getenv("SPEND_LOG_PARTITION_INTERVAL", "day")
 SPEND_LOG_PARTITION_PRECREATE_AHEAD: Final = int(os.getenv("SPEND_LOG_PARTITION_PRECREATE_AHEAD", 7))
 SPEND_LOG_WRITE_BATCH_MAX_BYTES: Final = max(1, int(os.getenv("SPEND_LOG_WRITE_BATCH_MAX_BYTES", 2_000_000)))
 SPEND_LOG_QUEUE_SIZE_THRESHOLD: Final = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100))
+SPEND_LOG_QUEUE_MAX_BYTES: Final = max(1, int(os.getenv("SPEND_LOG_QUEUE_MAX_BYTES", "64000000")))
 SPEND_LOG_QUEUE_POLL_INTERVAL: Final = float(os.getenv("SPEND_LOG_QUEUE_POLL_INTERVAL", 2.0))
 SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE: Final = int(os.getenv("SPEND_COUNTER_RESEED_LOCKS_MAX_SIZE", 10000))
 DEFAULT_CRON_JOB_LOCK_TTL_SECONDS: Final = int(os.getenv("DEFAULT_CRON_JOB_LOCK_TTL_SECONDS", 60))  # 1 minute
 PROXY_BUDGET_RESCHEDULER_MIN_TIME: Final = int(os.getenv("PROXY_BUDGET_RESCHEDULER_MIN_TIME", 597))
+RESET_BUDGET_JOB_BATCH_SIZE: Final = max(1, int(os.getenv("RESET_BUDGET_JOB_BATCH_SIZE", "500")))
+RESET_BUDGET_JOB_MAX_CHUNKS_PER_RUN: Final = max(1, int(os.getenv("RESET_BUDGET_JOB_MAX_CHUNKS_PER_RUN", "100")))
 PROXY_BATCH_POLLING_INTERVAL: Final = int(os.getenv("PROXY_BATCH_POLLING_INTERVAL", 3600))
 MAX_OBJECTS_PER_POLL_CYCLE: Final = max(1, int(os.getenv("MAX_OBJECTS_PER_POLL_CYCLE", 50)))
 MANAGED_OBJECT_STALENESS_CUTOFF_DAYS: Final = max(1, int(os.getenv("MANAGED_OBJECT_STALENESS_CUTOFF_DAYS", 7)))
@@ -1518,6 +1536,10 @@ APSCHEDULER_REPLACE_EXISTING: Final = os.getenv("APSCHEDULER_REPLACE_EXISTING",
     "1",
 ]  # always replace existing jobs
 
+# Width of the window scheduled background jobs are spread across, so they do not all fire
+# on one instant on every replica. Tunable per deployment via general_settings.
+DEFAULT_STAGGER_WINDOW_SECONDS: Final = 300
+
 # The number of tag entries are higher than number of user, team entries. This leads to a higher QPS.
 # This will run tag spcific tasks at a later time to smooth QPS
 DAILY_TAG_SPEND_BATCH_MULTIPLIER: Final = 2.3
@@ -1571,6 +1593,13 @@ DEFAULT_MCP_ACCESS_GROUP_NEGATIVE_CACHE_TTL: Final = 10
 # in a single ``/{name1,name2,...}/mcp`` URL. Bounds the per-request DB / cache
 # fan-out an authenticated caller can trigger by stuffing the path with tokens.
 DEFAULT_MCP_NAMESPACE_CSV_MAX_TOKENS: Final = 16
+# Ceilings on the cached auth registries; larger tables fall back to per-row lookups
+# instead of holding an unbounded id set in every worker.
+TAG_REGISTRY_MAX_SIZE: Final = 5000
+END_USER_RESTRICTED_REGISTRY_MAX_SIZE: Final = 5000
+# How long a failed registry load is remembered as "unusable", so a degraded Postgres
+# is not re-scanned on every request on top of the per-id lookups it falls back to.
+REGISTRY_ERROR_NEGATIVE_CACHE_TTL: Final = 30
 
 # Sentry Scrubbing Configuration
 SENTRY_DENYLIST: Final = [
@@ -1716,3 +1745,21 @@ BROWSER_SECURITY_HEADERS: Final[frozenset[str]] = frozenset(
 )
 
 UNSAFE_PROXY_RESPONSE_HEADERS: Final[frozenset[str]] = HTTP_FRAMING_HEADERS | BROWSER_SECURITY_HEADERS
+
+# PTU reservation rollup writes rows to LiteLLM_DailyTeamSpend with this
+# sentinel api_key so PTU flat cost stays distinguishable from real per-request
+# spend under the table's composite unique constraint.
+PTU_SENTINEL_API_KEY: Final[str] = "__ptu_flat_cost__"
+PTU_ROLLUP_JOB_ID: Final[str] = "ptu_flat_cost_rollup_job"
+PTU_ROLLUP_LOCK_TTL_SECONDS: Final[int] = 900
+# Furthest back the catch-up pass looks for unpriced PTU days when a deployment
+# declares no ptu_effective_from, bounding the scan for an open-ended window.
+PTU_ROLLUP_MAX_BACKFILL_DAYS: Final[int] = 90
+# Deployments named in the lapsed-window alert before it is truncated, so a fleet-wide
+# expiry cannot produce an alert too large for the channel delivering it.
+PTU_LAPSED_ALERT_LIMIT: Final[int] = 10
+# Slack allowed when deciding a sentinel row is stale. The row's updated_at and the
+# run's cutoff are stamped by different hosts, so clock skew between them must not let
+# one run delete a charge another just wrote. A stale row is hours old and a concurrent
+# one is seconds old, so a few minutes separates them.
+PTU_PRUNE_SKEW_GRACE_SECONDS: Final[int] = 300

litellm/containers/main.py

@@ -1,9 +1,11 @@
 import asyncio
 import contextvars
 import json
-from collections.abc import Coroutine
+from collections.abc import Coroutine, Mapping
 from functools import partial
-from typing import Any, Final, Literal, overload
+from typing import Final, Literal, overload
+
+import httpx
 
 import litellm
 from litellm.constants import request_timeout as DEFAULT_REQUEST_TIMEOUT
@@ -48,16 +50,16 @@ __all__ = [
 @client
 async def acreate_container(
     name: str,
-    expires_after: dict[str, Any] | None = None,
+    expires_after: Mapping[str, object] | None = None,
     file_ids: list[str] | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     # LiteLLM specific params,
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
     # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
     # The extra values given here take precedence over values defined on the client or passed to this method.
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
 ) -> ContainerObject:
     """Asynchronously calls the `create_container` function with the given arguments and keyword arguments.
@@ -120,9 +122,9 @@ async def acreate_container(
 @overload
 def create_container(
     name: str,
-    expires_after: dict[str, Any] | None = None,
+    expires_after: Mapping[str, object] | None = None,
     file_ids: list[str] | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -130,16 +132,16 @@ def create_container(
     *,
     acreate_container: Literal[True],
     **kwargs,
-) -> Coroutine[Any, Any, ContainerObject]:
+) -> Coroutine[object, object, ContainerObject]:
     ...
 
 
 @overload
 def create_container(
     name: str,
-    expires_after: dict[str, Any] | None = None,
+    expires_after: Mapping[str, object] | None = None,
     file_ids: list[str] | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -156,20 +158,20 @@ def create_container(
 @client
 def create_container(
     name: str,
-    expires_after: dict[str, Any] | None = None,
+    expires_after: Mapping[str, object] | None = None,
     file_ids: list[str] | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
     # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
     # The extra values given here take precedence over values defined on the client or passed to this method.
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
-) -> ContainerObject | Coroutine[Any, Any, ContainerObject]:
+) -> ContainerObject | Coroutine[object, object, ContainerObject]:
     """Create a container using the OpenAI Container API.
 
     Currently supports OpenAI
@@ -281,13 +283,13 @@ async def alist_containers(
     after: str | None = None,
     limit: int | None = None,
     order: str | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
     # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
     # The extra values given here take precedence over values defined on the client or passed to this method.
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
 ) -> ContainerListResponse:
     """Asynchronously list containers.
@@ -351,7 +353,7 @@ def list_containers(
     after: str | None = None,
     limit: int | None = None,
     order: str | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -359,7 +361,7 @@ def list_containers(
     *,
     alist_containers: Literal[True],
     **kwargs,
-) -> Coroutine[Any, Any, ContainerListResponse]:
+) -> Coroutine[object, object, ContainerListResponse]:
     ...
 
 
@@ -368,7 +370,7 @@ def list_containers(
     after: str | None = None,
     limit: int | None = None,
     order: str | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -387,18 +389,18 @@ def list_containers(
     after: str | None = None,
     limit: int | None = None,
     order: str | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
     # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
     # The extra values given here take precedence over values defined on the client or passed to this method.
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
-) -> ContainerListResponse | Coroutine[Any, Any, ContainerListResponse]:
+) -> ContainerListResponse | Coroutine[object, object, ContainerListResponse]:
     """List containers using the OpenAI Container API.
 
     Currently supports OpenAI
@@ -481,13 +483,13 @@ def list_containers(
 @client
 async def aretrieve_container(
     container_id: str,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
     # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
     # The extra values given here take precedence over values defined on the client or passed to this method.
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
 ) -> ContainerObject:
     """Asynchronously retrieve a container.
@@ -545,7 +547,7 @@ async def aretrieve_container(
 @overload
 def retrieve_container(
     container_id: str,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -553,14 +555,14 @@ def retrieve_container(
     *,
     aretrieve_container: Literal[True],
     **kwargs,
-) -> Coroutine[Any, Any, ContainerObject]:
+) -> Coroutine[object, object, ContainerObject]:
     ...
 
 
 @overload
 def retrieve_container(
     container_id: str,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -577,18 +579,18 @@ def retrieve_container(
 @client
 def retrieve_container(
     container_id: str,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
     # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
     # The extra values given here take precedence over values defined on the client or passed to this method.
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
-) -> ContainerObject | Coroutine[Any, Any, ContainerObject]:
+) -> ContainerObject | Coroutine[object, object, ContainerObject]:
     """Retrieve a container using the OpenAI Container API.
 
     Currently supports OpenAI
@@ -696,13 +698,13 @@ def retrieve_container(
 @client
 async def adelete_container(
     container_id: str,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
     # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
     # The extra values given here take precedence over values defined on the client or passed to this method.
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
 ) -> DeleteContainerResult:
     """Asynchronously delete a container.
@@ -760,7 +762,7 @@ async def adelete_container(
 @overload
 def delete_container(
     container_id: str,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -768,14 +770,14 @@ def delete_container(
     *,
     adelete_container: Literal[True],
     **kwargs,
-) -> Coroutine[Any, Any, DeleteContainerResult]:
+) -> Coroutine[object, object, DeleteContainerResult]:
     ...
 
 
 @overload
 def delete_container(
     container_id: str,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -792,18 +794,18 @@ def delete_container(
 @client
 def delete_container(
     container_id: str,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
     # Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
     # The extra values given here take precedence over values defined on the client or passed to this method.
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
-) -> DeleteContainerResult | Coroutine[Any, Any, DeleteContainerResult]:
+) -> DeleteContainerResult | Coroutine[object, object, DeleteContainerResult]:
     """Delete a container using the OpenAI Container API.
 
     Currently supports OpenAI
@@ -914,11 +916,11 @@ async def alist_container_files(
     after: str | None = None,
     limit: int | None = None,
     order: str | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
 ) -> ContainerFileListResponse:
     """Asynchronously list files in a container.
@@ -985,7 +987,7 @@ def list_container_files(
     after: str | None = None,
     limit: int | None = None,
     order: str | None = None,
-    timeout=600,
+    timeout: float | httpx.Timeout = 600,
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -993,7 +995,7 @@ def list_container_files(
     *,
     alist_container_files: Literal[True],
     **kwargs,
-) -> Coroutine[Any, Any, ContainerFileListResponse]:
+) -> Coroutine[object, object, ContainerFileListResponse]:
     ...
 
 
@@ -1003,7 +1005,7 @@ def list_container_files(
     after: str | None = None,
     limit: int | None = None,
     order: str | None = None,
-    timeout=600,
+    timeout: float | httpx.Timeout = 600,
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -1023,16 +1025,16 @@ def list_container_files(
     after: str | None = None,
     limit: int | None = None,
     order: str | None = None,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
-) -> ContainerFileListResponse | Coroutine[Any, Any, ContainerFileListResponse]:
+) -> ContainerFileListResponse | Coroutine[object, object, ContainerFileListResponse]:
     """List files in a container using the OpenAI Container API.
 
     Currently supports OpenAI
@@ -1125,11 +1127,11 @@ def list_container_files(
 async def aupload_container_file(
     container_id: str,
     file: FileTypes,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
 ) -> ContainerFileObject:
     """Asynchronously upload a file to a container.
@@ -1211,7 +1213,7 @@ async def aupload_container_file(
 def upload_container_file(
     container_id: str,
     file: FileTypes,
-    timeout=600,
+    timeout: float | httpx.Timeout = 600,
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -1219,7 +1221,7 @@ def upload_container_file(
     *,
     aupload_container_file: Literal[True],
     **kwargs,
-) -> Coroutine[Any, Any, ContainerFileObject]:
+) -> Coroutine[object, object, ContainerFileObject]:
     ...
 
 
@@ -1227,7 +1229,7 @@ def upload_container_file(
 def upload_container_file(
     container_id: str,
     file: FileTypes,
-    timeout=600,
+    timeout: float | httpx.Timeout = 600,
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
@@ -1245,16 +1247,16 @@ def upload_container_file(
 def upload_container_file(
     container_id: str,
     file: FileTypes,
-    timeout=600,  # default to 10 minutes
+    timeout: float | httpx.Timeout = 600,  # default to 10 minutes
     api_key: str | None = None,
     api_base: str | None = None,
     api_version: str | None = None,
     custom_llm_provider: Literal["openai", "azure", "azure_text"] = "openai",
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     **kwargs,
-) -> ContainerFileObject | Coroutine[Any, Any, ContainerFileObject]:
+) -> ContainerFileObject | Coroutine[object, object, ContainerFileObject]:
     """Upload a file to a container using the OpenAI Container API.
 
     This endpoint allows uploading files directly to a container session,

litellm/cost_calculator.py

@@ -26,11 +26,11 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
     _generic_cost_per_character,
     _get_regional_uplift_multiplier,
     _get_service_tier_cost_key,
-    _parse_prompt_tokens_details,
     calculate_cost_component,
     generic_cost_per_token,
     get_billable_input_tokens,
     get_token_type_cost_breakdown,
+    parse_prompt_tokens_details,
     select_cost_metric_for_model,
 )
 from litellm.llms.anthropic.cost_calculation import (
@@ -645,7 +645,11 @@ def cost_per_token(
     else:
         model_info: Final = _cached_get_model_info_helper(model=model, custom_llm_provider=custom_llm_provider)
 
-        if (model_info.get("input_cost_per_token") or 0.0) > 0 or (model_info.get("output_cost_per_token") or 0.0) > 0:
+        if (
+            (model_info.get("input_cost_per_token") or 0.0) > 0
+            or (model_info.get("output_cost_per_token") or 0.0) > 0
+            or model_info.get("tiered_pricing") is not None
+        ):
             return generic_cost_per_token(
                 model=model,
                 usage=usage_block,
@@ -2159,7 +2163,7 @@ def batch_cost_calculator(
     if input_cost_per_token_batches:
         total_prompt_cost = usage.prompt_tokens * input_cost_per_token_batches
     elif input_cost_per_token:
-        details: Final = _parse_prompt_tokens_details(usage)
+        details: Final = parse_prompt_tokens_details(usage)
         cache_read_tokens: Final = details["cache_hit_tokens"]
         cache_creation_tokens: Final = details["cache_creation_tokens"]
 

litellm/evals/main.py

@@ -7,7 +7,7 @@ import asyncio
 import contextvars
 from collections.abc import Coroutine
 from functools import partial
-from typing import Any, Final
+from typing import Final
 
 import httpx
 
@@ -21,8 +21,10 @@ from litellm.types.llms.openai_evals import (
     CancelRunResponse,
     CreateEvalRequest,
     CreateRunRequest,
+    DataSourceConfig,
     DeleteEvalResponse,
     Eval,
+    GraderConfig,
     ListEvalsParams,
     ListEvalsResponse,
     ListRunsParams,
@@ -41,13 +43,13 @@ DEFAULT_OPENAI_API_BASE: Final = "https://api.openai.com"
 
 @client
 async def acreate_eval(
-    data_source_config: dict[str, Any],
-    testing_criteria: list[dict[str, Any]],
+    data_source_config: DataSourceConfig,
+    testing_criteria: list[GraderConfig],
     name: str | None = None,
-    metadata: dict[str, Any] | None = None,
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    metadata: dict[str, object] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     timeout: float | httpx.Timeout | None = None,
     custom_llm_provider: str | None = None,
     **kwargs,
@@ -110,17 +112,17 @@ async def acreate_eval(
 
 @client
 def create_eval(
-    data_source_config: dict[str, Any],
-    testing_criteria: list[dict[str, Any]],
+    data_source_config: DataSourceConfig,
+    testing_criteria: list[GraderConfig],
     name: str | None = None,
-    metadata: dict[str, Any] | None = None,
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    metadata: dict[str, object] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     timeout: float | httpx.Timeout | None = None,
     custom_llm_provider: str | None = None,
     **kwargs,
-) -> Eval | Coroutine[Any, Any, Eval]:
+) -> Eval | Coroutine[object, object, Eval]:
     """
     Create a new evaluation
 
@@ -231,8 +233,8 @@ async def alist_evals(
     before: str | None = None,
     order: str | None = None,
     order_by: str | None = None,
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
     timeout: float | httpx.Timeout | None = None,
     custom_llm_provider: str | None = None,
     **kwargs,
@@ -300,12 +302,12 @@ def list_evals(
     before: str | None = None,
     order: str | None = None,
     order_by: str | None = None,
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
     timeout: float | httpx.Timeout | None = None,
     custom_llm_provider: str | None = None,
     **kwargs,
-) -> ListEvalsResponse | Coroutine[Any, Any, ListEvalsResponse]:
+) -> ListEvalsResponse | Coroutine[object, object, ListEvalsResponse]:
     """
     List all evaluations
 
@@ -413,8 +415,8 @@ def list_evals(
 @client
 async def aget_eval(
     eval_id: str,
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
     timeout: float | httpx.Timeout | None = None,
     custom_llm_provider: str | None = None,
     **kwargs,
@@ -470,12 +472,12 @@ async def aget_eval(
 @client
 def get_eval(
     eval_id: str,
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
     timeout: float | httpx.Timeout | None = None,
     custom_llm_provider: str | None = None,
     **kwargs,
-) -> Eval | Coroutine[Any, Any, Eval]:
+) -> Eval | Coroutine[object, object, Eval]:
     """
     Get an evaluation by ID
 
@@ -564,10 +566,10 @@ def get_eval(
 async def aupdate_eval(
     eval_id: str,
     name: str | None = None,
-    metadata: dict[str, Any] | None = None,
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    metadata: dict[str, object] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     timeout: float | httpx.Timeout | None = None,
     custom_llm_provider: str | None = None,
     **kwargs,
@@ -630,14 +632,14 @@ async def aupdate_eval(
 def update_eval(
     eval_id: str,
     name: str | None = None,
-    metadata: dict[str, Any] | None = None,
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    metadata: dict[str, object] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     timeout: float | httpx.Timeout | None = None,
     custom_llm_provider: str | None = None,
     **kwargs,
-) -> Eval | Coroutine[Any, Any, Eval]:
+) -> Eval | Coroutine[object, object, Eval]:
     """
     Update an evaluation
 
@@ -783,8 +785,8 @@ def update_eval(
 @client
 async def adelete_eval(
     eval_id: str,
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
     timeout: float | httpx.Timeout | None = None,
     custom_llm_provider: str | None = None,
     **kwargs,
@@ -840,12 +842,12 @@ async def adelete_eval(
 @client
 def delete_eval(
     eval_id: str,
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
     timeout: float | httpx.Timeout | None = None,
     custom_llm_provider: str | None = None,
     **kwargs,
-) -> DeleteEvalResponse | Coroutine[Any, Any, DeleteEvalResponse]:
+) -> DeleteEvalResponse | Coroutine[object, object, DeleteEvalResponse]:
     """
     Delete an evaluation
 
@@ -933,8 +935,8 @@ def delete_eval(
 @client
 async def acancel_eval(
     eval_id: str,
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
     timeout: float | httpx.Timeout | None = None,
     custom_llm_provider: str | None = None,
     **kwargs,
@@ -990,12 +992,12 @@ async def acancel_eval(
 @client
 def cancel_eval(
     eval_id: str,
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
     timeout: float | httpx.Timeout | None = None,
     custom_llm_provider: str | None = None,
     **kwargs,
-) -> CancelEvalResponse | Coroutine[Any, Any, CancelEvalResponse]:
+) -> CancelEvalResponse | Coroutine[object, object, CancelEvalResponse]:
     """
     Cancel a running evaluation
 
@@ -1092,12 +1094,12 @@ def cancel_eval(
 @client
 async def acreate_run(
     eval_id: str,
-    data_source: dict[str, Any],
+    data_source: dict[str, object],
     name: str | None = None,
-    metadata: dict[str, Any] | None = None,
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    metadata: dict[str, object] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     timeout: float | httpx.Timeout | None = None,
     custom_llm_provider: str | None = None,
     **kwargs,
@@ -1161,16 +1163,16 @@ async def acreate_run(
 @client
 def create_run(
     eval_id: str,
-    data_source: dict[str, Any],
+    data_source: dict[str, object],
     name: str | None = None,
-    metadata: dict[str, Any] | None = None,
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
-    extra_body: dict[str, Any] | None = None,
+    metadata: dict[str, object] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
+    extra_body: dict[str, object] | None = None,
     timeout: float | httpx.Timeout | None = None,
     custom_llm_provider: str | None = None,
     **kwargs,
-) -> Run | Coroutine[Any, Any, Run]:
+) -> Run | Coroutine[object, object, Run]:
     """
     Create a new run for an evaluation
 
@@ -1280,8 +1282,8 @@ async def alist_runs(
     after: str | None = None,
     before: str | None = None,
     order: str | None = None,
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
     timeout: float | httpx.Timeout | None = None,
     custom_llm_provider: str | None = None,
     **kwargs,
@@ -1349,12 +1351,12 @@ def list_runs(
     after: str | None = None,
     before: str | None = None,
     order: str | None = None,
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
     timeout: float | httpx.Timeout | None = None,
     custom_llm_provider: str | None = None,
     **kwargs,
-) -> ListRunsResponse | Coroutine[Any, Any, ListRunsResponse]:
+) -> ListRunsResponse | Coroutine[object, object, ListRunsResponse]:
     """
     List all runs for an evaluation
 
@@ -1462,8 +1464,8 @@ def list_runs(
 async def aget_run(
     eval_id: str,
     run_id: str,
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
     timeout: float | httpx.Timeout | None = None,
     custom_llm_provider: str | None = None,
     **kwargs,
@@ -1522,12 +1524,12 @@ async def aget_run(
 def get_run(
     eval_id: str,
     run_id: str,
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
     timeout: float | httpx.Timeout | None = None,
     custom_llm_provider: str | None = None,
     **kwargs,
-) -> Run | Coroutine[Any, Any, Run]:
+) -> Run | Coroutine[object, object, Run]:
     """
     Get a specific run
 
@@ -1618,8 +1620,8 @@ def get_run(
 async def acancel_run(
     eval_id: str,
     run_id: str,
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
     timeout: float | httpx.Timeout | None = None,
     custom_llm_provider: str | None = None,
     **kwargs,
@@ -1678,12 +1680,12 @@ async def acancel_run(
 def cancel_run(
     eval_id: str,
     run_id: str,
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
     timeout: float | httpx.Timeout | None = None,
     custom_llm_provider: str | None = None,
     **kwargs,
-) -> CancelRunResponse | Coroutine[Any, Any, CancelRunResponse]:
+) -> CancelRunResponse | Coroutine[object, object, CancelRunResponse]:
     """
     Cancel a running run
 
@@ -1783,8 +1785,8 @@ def cancel_run(
 async def adelete_run(
     eval_id: str,
     run_id: str,
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
     timeout: float | httpx.Timeout | None = None,
     custom_llm_provider: str | None = None,
     **kwargs,
@@ -1843,12 +1845,12 @@ async def adelete_run(
 def delete_run(
     eval_id: str,
     run_id: str,
-    extra_headers: dict[str, Any] | None = None,
-    extra_query: dict[str, Any] | None = None,
+    extra_headers: dict[str, object] | None = None,
+    extra_query: dict[str, object] | None = None,
     timeout: float | httpx.Timeout | None = None,
     custom_llm_provider: str | None = None,
     **kwargs,
-) -> RunDeleteResponse | Coroutine[Any, Any, RunDeleteResponse]:
+) -> RunDeleteResponse | Coroutine[object, object, RunDeleteResponse]:
     """
     Delete a run
 

litellm/experimental_mcp_client/client.py

@@ -6,10 +6,11 @@ import asyncio
 import base64
 import os
 from collections.abc import Awaitable, Callable, Generator
+from datetime import timedelta
 from typing import Any, Final, TypeVar
 
 import httpx
-from mcp import ClientSession, ReadResourceResult, Resource, StdioServerParameters
+from mcp import ClientSession, McpError, ReadResourceResult, Resource, StdioServerParameters
 from mcp.client.sse import sse_client
 from mcp.client.stdio import stdio_client
 
@@ -69,6 +70,29 @@ def _first_non_cancelled_cause(exc: BaseException) -> BaseException | None:
     return None
 
 
+_SDK_READ_TIMEOUT_CODE: Final = int(httpx.codes.REQUEST_TIMEOUT)
+"""The code the MCP SDK puts on its own elapsed read timeout, an HTTP status in a field that
+otherwise carries JSON-RPC error codes."""
+
+
+def _as_read_timeout(exc: BaseException) -> TimeoutError | None:
+    """The session read timeout elapsing, re-expressed as a ``TimeoutError``, or ``None``.
+
+    The SDK reports its own elapsed read timeout as ``McpError`` carrying an HTTP status code in a
+    field that otherwise holds JSON-RPC error codes, and it relays an upstream's JSON-RPC error
+    through that same class and field. The numeric code alone therefore cannot separate the two, and
+    an upstream answering with application code 408 would be reported as a gateway timeout it never
+    caused. The SDK raises its own from inside an ``except TimeoutError``, so the elapsed timeout is
+    on the context chain, while a relayed error is built from a received message and has no such
+    chain; that is the discriminator.
+    """
+    if not isinstance(exc, McpError) or exc.error.code != _SDK_READ_TIMEOUT_CODE:
+        return None
+    if not isinstance(exc.__context__, TimeoutError):
+        return None
+    return TimeoutError(exc.error.message)
+
+
 TSessionResult = TypeVar("TSessionResult")
 
 
@@ -347,7 +371,14 @@ class MCPClient:
                 session_kwargs["elicitation_callback"] = self._elicitation_callback
             if self._logging_callback is not None:
                 session_kwargs["logging_callback"] = self._logging_callback
-            session_ctx: Final = ClientSession(read_stream, write_stream, **session_kwargs)
+            # The SDK drops a response stream that ends without a JSON-RPC reply, so nothing else
+            # ever fails the request.
+            session_ctx: Final = ClientSession(
+                read_stream,
+                write_stream,
+                read_timeout_seconds=timedelta(seconds=self.timeout),
+                **session_kwargs,
+            )
             session: Final = await session_ctx.__aenter__()
             try:
                 init_result: Final = await session.initialize()
@@ -390,7 +421,16 @@ class MCPClient:
             self._last_initialize_instructions = None
             transport_ctx, http_client = self._create_transport_context()
             return await self._execute_session_operation(transport_ctx, operation)
-        except Exception:
+        except Exception as e:
+            read_timeout: Final = _as_read_timeout(e)
+            if read_timeout is not None:
+                verbose_logger.warning(
+                    "MCP client timed out after %ss waiting for %s to answer; the server accepted the "
+                    "request and ended its response stream without a JSON-RPC reply",
+                    self.timeout,
+                    self.server_url or "stdio",
+                )
+                raise read_timeout from e
             _log: Final = verbose_logger.debug if quiet_on_error else verbose_logger.warning
             _log("MCP client run_with_session failed for %s", self.server_url or "stdio")
             raise

litellm/files/main.py

@@ -11,7 +11,6 @@ import time
 import uuid as uuid_module
 from collections.abc import Coroutine
 from functools import partial
-from types import MappingProxyType
 from typing import Any, Final, Literal, cast
 
 import httpx
@@ -34,6 +33,7 @@ import litellm
 from litellm import get_secret_str
 from litellm.files.streaming import FileContentStreamingResponse
 from litellm.files.types import FileContentProvider, FileContentStreamingResult
+from litellm.litellm_core_utils.get_litellm_params import add_trusted_model_credentials_to_litellm_params
 from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.azure.common_utils import get_azure_credentials
@@ -85,14 +85,6 @@ bedrock_files_instance: Final = BedrockFilesHandler()
 #################################################
 
 
-def _add_trusted_model_credentials_to_litellm_params(
-    litellm_params_dict: dict[str, Any], kwargs: dict[str, Any]
-) -> None:
-    trusted_model_credentials: Final = kwargs.get("_litellm_internal_model_credentials")
-    if isinstance(trusted_model_credentials, type(MappingProxyType({}))):
-        litellm_params_dict["_litellm_internal_model_credentials"] = trusted_model_credentials
-
-
 @client
 async def acreate_file(
     file: FileTypes,
@@ -372,7 +364,7 @@ def file_retrieve(
             )
             if provider_config is not None:
                 litellm_params_dict: Final = get_litellm_params(**kwargs)
-                _add_trusted_model_credentials_to_litellm_params(
+                add_trusted_model_credentials_to_litellm_params(
                     litellm_params_dict=litellm_params_dict,
                     kwargs=kwargs,
                 )
@@ -494,7 +486,7 @@ def file_delete(
             pass
         optional_params: Final = GenericLiteLLMParams(**kwargs)
         litellm_params_dict: Final = get_litellm_params(**kwargs)
-        _add_trusted_model_credentials_to_litellm_params(
+        add_trusted_model_credentials_to_litellm_params(
             litellm_params_dict=litellm_params_dict,
             kwargs=kwargs,
         )
@@ -834,7 +826,7 @@ def file_content(
     try:
         optional_params: Final = GenericLiteLLMParams(**kwargs)
         litellm_params_dict: Final = get_litellm_params(**kwargs)
-        _add_trusted_model_credentials_to_litellm_params(
+        add_trusted_model_credentials_to_litellm_params(
             litellm_params_dict=litellm_params_dict,
             kwargs=kwargs,
         )

litellm/google_genai/adapters/transformation.py

@@ -1,6 +1,8 @@
 import json
 from collections.abc import AsyncIterator, Iterator
-from typing import Any, Final, cast
+from typing import Any, Final, TypedDict, cast
+
+from typing_extensions import ReadOnly
 
 from litellm import verbose_logger
 from litellm.litellm_core_utils.json_validation_rule import normalize_tool_schema
@@ -28,6 +30,19 @@ from litellm.types.utils import (
 )
 
 
+class _GenAITextPart(TypedDict, total=False):
+    text: ReadOnly[str]
+
+
+class _GenAISystemInstruction(TypedDict, total=False):
+    parts: ReadOnly[list[_GenAITextPart]]
+
+
+class _GenAIPart(TypedDict, total=False):
+    text: ReadOnly[str]
+    functionCall: ReadOnly[dict[str, object]]
+
+
 class GoogleGenAIStreamWrapper(AdapterCompletionStreamWrapper):
     """
     Wrapper for streaming Google GenAI generate_content responses.
@@ -36,9 +51,9 @@ class GoogleGenAIStreamWrapper(AdapterCompletionStreamWrapper):
 
     sent_first_chunk: bool = False
     # State tracking for accumulating partial tool calls
-    accumulated_tool_calls: dict[str, dict[str, Any]]
+    accumulated_tool_calls: dict[str, dict[str, str]]
 
-    def __init__(self, completion_stream: Any):
+    def __init__(self, completion_stream: object):
         self.sent_first_chunk = False
         self.accumulated_tool_calls = {}
         self._returned_response = False
@@ -85,7 +100,7 @@ class GoogleGenAIStreamWrapper(AdapterCompletionStreamWrapper):
             # After the stream is exhausted, check for any remaining accumulated tool calls
             if self.accumulated_tool_calls:
                 try:
-                    parts: Final = []
+                    parts: Final[list[_GenAIPart]] = []
                     for (
                         tool_call_index,
                         tool_call_data,
@@ -94,7 +109,7 @@ class GoogleGenAIStreamWrapper(AdapterCompletionStreamWrapper):
                             # For tool calls with no arguments, accumulated_args will be "", which is not valid JSON.
                             # We default to an empty JSON object in this case.
                             parsed_args = json.loads(tool_call_data["arguments"] or "{}")
-                            function_call_part = {
+                            function_call_part: _GenAIPart = {
                                 "functionCall": {
                                     "name": tool_call_data["name"] or "undefined_tool_name",
                                     "args": parsed_args,
@@ -110,7 +125,7 @@ class GoogleGenAIStreamWrapper(AdapterCompletionStreamWrapper):
                                 tool_call_data["arguments"],
                             )
                     if parts:
-                        final_chunk: Final = {
+                        final_chunk: Final[dict[str, object]] = {
                             "candidates": [
                                 {
                                     "content": {"parts": parts, "role": "model"},
@@ -273,9 +288,9 @@ class GoogleGenAIAdapter:
 
     def _add_generic_litellm_params_to_request(
         self,
-        completion_request_dict: dict[str, Any],
+        completion_request_dict: dict[str, object],
         litellm_params: GenericLiteLLMParams | None = None,
-    ) -> dict:
+    ) -> dict[str, object]:
         """Add generic litellm params to request. e.g add api_base, api_key, api_version, etc.
 
         Args:
@@ -295,7 +310,7 @@ class GoogleGenAIAdapter:
 
     def translate_completion_output_params_streaming(
         self,
-        completion_stream: Any,
+        completion_stream: object,
     ) -> AsyncIterator[bytes] | None:
         """Transform streaming completion output to Google GenAI format"""
         google_genai_wrapper: Final = GoogleGenAIStreamWrapper(completion_stream=completion_stream)
@@ -307,12 +322,12 @@ class GoogleGenAIAdapter:
         tools: list[dict[str, Any]],
     ) -> list[ChatCompletionToolParam]:
         """Transform Google GenAI tools to OpenAI tools format"""
-        openai_tools: Final[list[dict[str, Any]]] = []
+        openai_tools: Final[list[dict[str, object]]] = []
 
         for tool in tools:
             if "functionDeclarations" in tool:
                 for func_decl in tool["functionDeclarations"]:
-                    function_chunk: dict[str, Any] = {
+                    function_chunk: dict[str, object] = {
                         "name": func_decl.get("name", ""),
                     }
 
@@ -321,7 +336,7 @@ class GoogleGenAIAdapter:
                     if "parametersJsonSchema" in func_decl:
                         function_chunk["parameters"] = func_decl["parametersJsonSchema"]
 
-                    openai_tool = {"type": "function", "function": function_chunk}
+                    openai_tool: dict[str, object] = {"type": "function", "function": function_chunk}
                     openai_tools.append(openai_tool)
 
         # normalize the tool schemas
@@ -345,7 +360,7 @@ class GoogleGenAIAdapter:
     def _transform_contents_to_messages(
         self,
         contents: list[dict[str, Any]],
-        system_instruction: dict[str, Any] | None = None,
+        system_instruction: _GenAISystemInstruction | None = None,
     ) -> list[AllMessageValues]:
         """Transform Google GenAI contents to OpenAI messages format"""
         messages: Final[list[AllMessageValues]] = []
@@ -461,7 +476,7 @@ class GoogleGenAIAdapter:
     def translate_completion_to_generate_content(
         self,
         response: ModelResponse,
-    ) -> dict[str, Any]:
+    ) -> dict[str, object]:
         """
         Transform litellm completion response to Google GenAI generate_content format
 
@@ -490,7 +505,7 @@ class GoogleGenAIAdapter:
             parts = [{"text": message_content}] if message_content else []
 
         # Create Google GenAI format response
-        generate_content_response: Final[dict[str, Any]] = {
+        generate_content_response: Final[dict[str, object]] = {
             "candidates": [
                 {
                     "content": {"parts": parts, "role": "model"},
@@ -524,7 +539,7 @@ class GoogleGenAIAdapter:
         self,
         response: ModelResponse | ModelResponseStream,
         wrapper: GoogleGenAIStreamWrapper,
-    ) -> dict[str, Any] | None:
+    ) -> dict[str, object] | None:
         """
         Transform streaming litellm completion chunk to Google GenAI generate_content format
 
@@ -560,7 +575,7 @@ class GoogleGenAIAdapter:
             return None
 
         # Create Google GenAI streaming format response
-        streaming_chunk: Final[dict[str, Any]] = {
+        streaming_chunk: Final[dict[str, object]] = {
             "candidates": [
                 {
                     "content": {"parts": parts, "role": "model"},
@@ -597,9 +612,9 @@ class GoogleGenAIAdapter:
     def _transform_openai_message_to_google_genai_parts(
         self,
         message: Any,
-    ) -> list[dict[str, Any]]:
+    ) -> list[_GenAIPart]:
         """Transform OpenAI message to Google GenAI parts format"""
-        parts: Final[list[dict[str, Any]]] = []
+        parts: Final[list[_GenAIPart]] = []
 
         # Add text content if present
         if hasattr(message, "content") and message.content:
@@ -614,7 +629,7 @@ class GoogleGenAIAdapter:
                     except json.JSONDecodeError:
                         args = {}
 
-                    function_call_part = {
+                    function_call_part: _GenAIPart = {
                         "functionCall": {
                             "name": tool_call.function.name or "undefined_tool_name",
                             "args": args,
@@ -626,14 +641,14 @@ class GoogleGenAIAdapter:
 
     def _transform_openai_delta_to_google_genai_parts_with_accumulation(
         self, delta: Any, wrapper: GoogleGenAIStreamWrapper
-    ) -> list[dict[str, Any]]:
+    ) -> list[_GenAIPart]:
         """Transforms OpenAI delta to Google GenAI parts, accumulating streaming tool calls."""
 
         # 1. Initialize wrapper state if it doesn't exist
         if not hasattr(wrapper, "accumulated_tool_calls"):
             wrapper.accumulated_tool_calls = {}
 
-        parts: Final[list[dict[str, Any]]] = []
+        parts: Final[list[_GenAIPart]] = []
 
         if hasattr(delta, "content") and delta.content:
             parts.append({"text": delta.content})
@@ -686,7 +701,7 @@ class GoogleGenAIAdapter:
                 # The part will be created by a later chunk that brings the name.
                 if accumulated_name:
                     # If successful, create the part and clean up
-                    function_call_part = {"functionCall": {"name": accumulated_name, "args": parsed_args}}
+                    function_call_part: _GenAIPart = {"functionCall": {"name": accumulated_name, "args": parsed_args}}
                     parts.append(function_call_part)
 
                     # Remove the completed tool call from the accumulator

litellm/images/main.py

@@ -315,7 +315,12 @@ def image_generation(
                 or get_secret_str("AZURE_API_KEY")
             )
 
-            azure_ad_token: Final = optional_params.pop("azure_ad_token", None) or get_secret_str("AZURE_AD_TOKEN")
+            azure_ad_token_param: Final = optional_params.pop("azure_ad_token", None)
+            azure_ad_token: Final = (
+                azure_ad_token_param
+                if isinstance(azure_ad_token_param, str) and azure_ad_token_param
+                else get_secret_str("AZURE_AD_TOKEN")
+            )
 
             # Create azure_ad_token_provider from tenant_id, client_id, client_secret if not already provided
             if azure_ad_token_provider is None:

litellm/integrations/SlackAlerting/slack_alerting.py

@@ -9,6 +9,7 @@ from datetime import timedelta
 from typing import TYPE_CHECKING, Any, Final, Literal
 
 from openai import APIError
+from pydantic import TypeAdapter
 
 import litellm
 import litellm.litellm_core_utils
@@ -16,7 +17,7 @@ import litellm.litellm_core_utils.litellm_logging
 import litellm.types
 from litellm._logging import verbose_logger, verbose_proxy_logger
 from litellm.caching.caching import DualCache
-from litellm.constants import HOURS_IN_A_DAY
+from litellm.constants import HOURS_IN_A_DAY, SLACK_DAILY_REPORT_LOCK_ID
 from litellm.integrations.custom_batch_logger import CustomBatchLogger
 from litellm.integrations.SlackAlerting.budget_alert_types import get_budget_alert_type
 from litellm.integrations.SlackAlerting.hanging_request_check import (
@@ -33,10 +34,14 @@ from litellm.llms.custom_httpx.http_handler import (
 from litellm.proxy._types import (
     AlertType,
     CallInfo,
+    InvitationModel,
+    InvitationNew,
     Litellm_EntityType,
+    UserAPIKeyAuth,
     VirtualKeyEvent,
     WebhookEvent,
 )
+from litellm.repositories.table_repositories import InvitationLinkRepository
 from litellm.repositories.team_repository import TeamRepository
 from litellm.repositories.user_repository import UserRepository
 from litellm.types.integrations.slack_alerting import *
@@ -46,6 +51,7 @@ from .batching_handler import send_to_webhook, squash_payloads
 from .utils import process_slack_alerting_variables
 
 if TYPE_CHECKING:
+    from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager
     from litellm.router import Router as _Router
 
     Router = _Router
@@ -1081,6 +1087,44 @@ Model Info:
             if email_logo_url is not None or email_support_contact is not None:
                 raise ValueError(f"Trying to Customize Email Alerting\n {CommonProxyErrors.not_premium_user.value}")
 
+    async def _construct_user_invitation_link(self, recipient_user_id: str | None, base_url: str) -> str:
+        from litellm.proxy.management_helpers.user_invitation import (
+            create_invitation_for_user,
+        )
+        from litellm.proxy.proxy_server import prisma_client
+
+        if recipient_user_id is None or prisma_client is None:
+            return base_url
+
+        try:
+            existing_invitations: Final = TypeAdapter(list[InvitationModel]).validate_python(
+                await InvitationLinkRepository(prisma_client).table.find_many(  # pyright: ignore[reportAny]  # untyped prisma boundary (any-ok), result validated by TypeAdapter
+                    where={"user_id": recipient_user_id},  # mutable-ok: prisma find_many requires a dict where filter
+                    order={"created_at": "desc"},  # mutable-ok: prisma find_many requires a dict order arg
+                ),
+                from_attributes=True,
+            )
+            invitation: Final = (
+                existing_invitations[0]
+                if existing_invitations
+                else TypeAdapter(InvitationModel).validate_python(
+                    await create_invitation_for_user(
+                        data=InvitationNew(user_id=recipient_user_id),
+                        user_api_key_dict=UserAPIKeyAuth(user_id=recipient_user_id),
+                    ),
+                    from_attributes=True,
+                )
+            )
+        except Exception as e:  # noqa: BLE001  # best-effort link build; any DB/creation failure falls back to base_url
+            verbose_proxy_logger.error(
+                "Error creating invitation link for user_id %s: %s",
+                recipient_user_id,
+                str(e),
+            )
+            return base_url
+
+        return f"{base_url.rstrip('/')}/ui/onboarding?invitation_id={invitation.id}"
+
     async def send_key_created_or_user_invited_email(self, webhook_event: WebhookEvent) -> bool:
         try:
             from litellm.proxy.utils import send_email
@@ -1139,11 +1183,14 @@ Model Info:
                     team_row: Final = await TeamRepository(prisma_client).table.find_unique(where={"team_id": team_id})
                     if team_row is not None:
                         team_name = team_row.team_alias or "-"
+                invitation_link: Final = await self._construct_user_invitation_link(
+                    recipient_user_id=recipient_user_id, base_url=base_url
+                )
                 email_html_content = USER_INVITED_EMAIL_TEMPLATE.format(
                     email_logo_url=email_logo_url,
                     recipient_email=recipient_email,
                     team_name=team_name,
-                    base_url=base_url,
+                    base_url=invitation_link,
                     email_support_contact=email_support_contact,
                 )
             else:
@@ -1530,7 +1577,11 @@ Model Info:
         except Exception:
             pass
 
-    async def _run_scheduler_helper(self, llm_router) -> bool:
+    async def _run_scheduler_helper(
+        self,
+        llm_router,
+        pod_lock_manager: "PodLockManager | None" = None,
+    ) -> bool:
         """
         Returns:
         - True -> report sent
@@ -1555,6 +1606,16 @@ Model Info:
             interval_seconds: Final = self.alerting_args.daily_report_frequency
 
             if current_time - report_sent >= interval_seconds:
+                if (
+                    pod_lock_manager is not None
+                    and (
+                        await pod_lock_manager.acquire_lock(
+                            cronjob_id=SLACK_DAILY_REPORT_LOCK_ID, ttl=interval_seconds, allow_reentrant=False
+                        )
+                    )
+                    is False
+                ):
+                    return False
                 # Sneak in the reporting logic here
                 await self.send_daily_reports(router=llm_router)
                 # Also, don't forget to update the report_sent time after sending the report!
@@ -1566,7 +1627,11 @@ Model Info:
 
         return report_sent_bool
 
-    async def _run_scheduled_daily_report(self, llm_router: Any | None = None):
+    async def _run_scheduled_daily_report(
+        self,
+        llm_router: Any | None = None,
+        pod_lock_manager: "PodLockManager | None" = None,
+    ):
         """
         If 'daily_reports' enabled
 
@@ -1579,7 +1644,7 @@ Model Info:
 
         if "daily_reports" in self.alert_types:
             while True:
-                await self._run_scheduler_helper(llm_router=llm_router)
+                await self._run_scheduler_helper(llm_router=llm_router, pod_lock_manager=pod_lock_manager)
                 interval = random.randint(
                     self.alerting_args.report_check_interval - 3,
                     self.alerting_args.report_check_interval + 3,

litellm/integrations/anthropic_cache_control_hook.py

@@ -382,19 +382,23 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         model: str,
         custom_llm_provider: str | None,
         tools: list | None = None,
+        enable_prompt_caching: bool | None = None,
     ) -> list[CacheControlInjectionPoint]:
         """Default breakpoints when ``litellm.enable_anthropic_prompt_caching`` is on.
 
-        Caches the system prompt and the trailing turn, so the stable prefix
-        (system + tools + history) is reused while the breakpoint advances with
-        the conversation. Returns [] (stand down) when the flag is off, the
-        provider does not consume cache_control breakpoints (only anthropic /
-        bedrock do), the model lacks prompt-caching support, or the request
-        already carries client-supplied cache_control.
+        ``enable_prompt_caching`` is the per-request override (stamped from key
+        metadata by the proxy); True turns auto-injection on for this request
+        even when the global flag is off. Caches the system prompt and the
+        trailing turn, so the stable prefix (system + tools + history) is
+        reused while the breakpoint advances with the conversation. Returns []
+        (stand down) when neither flag is on, the provider does not consume
+        cache_control breakpoints (only anthropic / bedrock do), the model
+        lacks prompt-caching support, or the request already carries
+        client-supplied cache_control.
         """
         import litellm
 
-        if litellm.enable_anthropic_prompt_caching is not True:
+        if litellm.enable_anthropic_prompt_caching is not True and enable_prompt_caching is not True:
             return []
 
         provider = custom_llm_provider
@@ -433,6 +437,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         model: str,
         custom_llm_provider: str | None,
         tools: list | None = None,
+        enable_prompt_caching: bool | None = None,
     ) -> None:
         """For /chat/completions: resolve the injection points the request should carry.
 
@@ -458,6 +463,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
             model=model,
             custom_llm_provider=custom_llm_provider,
             tools=tools,
+            enable_prompt_caching=enable_prompt_caching,
         )
         if points:
             non_default_params["cache_control_injection_points"] = points
@@ -478,12 +484,17 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         judgment happens once per request; points a prior pass wrote back
         carry the judged stamp and are never re-judged (see
         ``_should_stand_down``). When none are configured but
-        ``litellm.enable_anthropic_prompt_caching`` is on, synthesize default
-        breakpoints for the native /v1/messages path. Pops the key from kwargs;
+        ``litellm.enable_anthropic_prompt_caching`` or the per-request
+        ``enable_prompt_caching`` kwarg (stamped from key metadata) is on,
+        synthesize default breakpoints for the native /v1/messages path. Pops
+        both keys from kwargs;
         if remaining (non-message) points exist they are written back so
         downstream transforms can handle them.
         """
         typed_messages = cast(list[AllMessageValues], messages)  # cast-ok: Anthropic-shaped dicts from v1/messages
+        enable_prompt_caching: Final = cast(  # cast-ok: kwargs is untyped; key stamped as bool by the proxy
+            bool | None, kwargs.pop("enable_prompt_caching", None)
+        )
         configured: Final = cast(  # cast-ok: kwargs is untyped; this key only holds the documented injection-point list
             list[CacheControlInjectionPoint] | None, kwargs.pop("cache_control_injection_points", None)
         )
@@ -497,6 +508,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
                 tools=tools,
                 model=model,
                 custom_llm_provider=custom_llm_provider,
+                enable_prompt_caching=enable_prompt_caching,
             )
         if not injection_points:
             return messages, system

litellm/integrations/arize/_utils.py

@@ -1,4 +1,5 @@
 import json
+from collections.abc import Mapping
 from typing import TYPE_CHECKING, Any, Final
 
 from typing_extensions import override
@@ -12,7 +13,7 @@ from litellm.litellm_core_utils.redact_messages import (
     should_redact_message_logging,
 )
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
-from litellm.types.utils import StandardLoggingPayload
+from litellm.types.utils import CallTypes, StandardLoggingMCPToolCall, StandardLoggingPayload
 
 if TYPE_CHECKING:
     from opentelemetry.trace import Span
@@ -22,6 +23,7 @@ from litellm.integrations._types.open_inference import (
     ImageAttributes,
     MessageAttributes,
     MessageContentAttributes,
+    OpenInferenceMimeTypeValues,
     OpenInferenceSpanKindValues,
     SpanAttributes,
     ToolCallAttributes,
@@ -480,6 +482,7 @@ def set_attributes(span: "Span", kwargs, response_obj, attributes: type[BaseLLMO
         response_obj_for_attrs,
         slp,
     )
+    _safe_emit("mcp tool attrs", _maybe_set_mcp_tool_attrs, span, kwargs, slp, response_obj_for_attrs)
 
 
 def _sanitize_optional_params(optional_params: dict | None) -> dict:
@@ -538,9 +541,12 @@ def _set_request_attributes(
     if optional_params.get("user"):
         safe_set_attribute(span, "llm.user", optional_params.get("user"))
 
-    if response_obj and response_obj.get("id"):
+    if not hasattr(response_obj, "get"):
+        return
+
+    if response_obj.get("id"):
         safe_set_attribute(span, "llm.response.id", response_obj.get("id"))
-    if response_obj and response_obj.get("model"):
+    if response_obj.get("model"):
         safe_set_attribute(span, "llm.response.model", response_obj.get("model"))
 
 
@@ -588,6 +594,8 @@ def _coerce_response_obj_for_attrs(response_obj):
     - dicts and Pydantic models that already expose `.get` are returned
       unchanged (preserves all current behavior, including the Responses API
       flow which relies on Pydantic attribute access).
+    - Pydantic models without `.get` (e.g. the MCP SDK's `CallToolResult`,
+      logged for `call_mcp_tool` spans) are dumped to a dict.
     - `httpx.Response` and other text-only responses (passthrough routes)
       are JSON-decoded so the standard extraction paths can read fields like
       `id`, `model`, and `usage`. On failure the original object is returned
@@ -595,6 +603,9 @@ def _coerce_response_obj_for_attrs(response_obj):
     """
     if response_obj is None or hasattr(response_obj, "get"):
         return response_obj
+    dumped: Final = _to_plain_dict(response_obj)
+    if isinstance(dumped, dict):
+        return dumped
     text: Final = getattr(response_obj, "text", None)
     if isinstance(text, str) and text:
         try:
@@ -1058,3 +1069,65 @@ def _parse_passthrough_response(raw_response_obj, coerced_response_obj, kwargs):
         except Exception:
             return None
     return None
+
+
+def _maybe_set_mcp_tool_attrs(
+    span: "Span",
+    kwargs: Mapping[str, object],
+    standard_logging_payload: StandardLoggingPayload | None,
+    coerced_response_obj: object,
+) -> None:
+    """Render `call_mcp_tool` spans as OpenInference TOOL spans.
+
+    MCP tool calls carry neither `messages` nor `choices`, so the generic
+    extraction paths leave Input/Output blank. The tool name and arguments live
+    in `metadata.mcp_tool_call_metadata`; the result is an MCP `CallToolResult`
+    whose `content` is a list of typed parts.
+    """
+    if standard_logging_payload is None:
+        return
+    if standard_logging_payload.get("call_type") != CallTypes.call_mcp_tool.value:
+        return
+
+    metadata: Final = standard_logging_payload.get("metadata")
+    mcp_meta: Final[StandardLoggingMCPToolCall | None] = metadata.get("mcp_tool_call_metadata") if metadata else None
+    if mcp_meta is None:
+        return
+
+    tool_name: Final = mcp_meta.get("name") or mcp_meta.get("namespaced_tool_name")
+    if tool_name:
+        safe_set_attribute(span, SpanAttributes.TOOL_NAME, tool_name)
+
+    if should_redact_message_logging(kwargs):  # pyright: ignore[reportArgumentType]  # reads, never mutates
+        return
+
+    arguments: Final[object] = mcp_meta.get("arguments")
+    if arguments is not None:
+        safe_set_attribute(span, SpanAttributes.INPUT_VALUE, safe_dumps(arguments))
+        safe_set_attribute(span, SpanAttributes.INPUT_MIME_TYPE, OpenInferenceMimeTypeValues.JSON.value)
+
+    _set_mcp_tool_output(span, coerced_response_obj)
+
+
+def _has_only_text_parts(content: object) -> bool:
+    return not isinstance(content, list) or all(_coerce_text([part]) is not None for part in content)
+
+
+def _set_mcp_tool_output(span: "Span", coerced_response_obj: object) -> None:
+    if not isinstance(coerced_response_obj, Mapping):
+        return
+
+    content: Final[object] = coerced_response_obj.get("content")
+    text: Final[str | None] = _coerce_text(content)
+    if text and _has_only_text_parts(content):
+        safe_set_attribute(span, SpanAttributes.OUTPUT_VALUE, text)
+        safe_set_attribute(span, SpanAttributes.OUTPUT_MIME_TYPE, OpenInferenceMimeTypeValues.TEXT.value)
+        return
+
+    structured: Final[object] = coerced_response_obj.get("structuredContent")
+    payload: Final[object] = content if content else structured if structured is not None else content
+    if payload is None:
+        return
+
+    safe_set_attribute(span, SpanAttributes.OUTPUT_VALUE, safe_dumps(payload))
+    safe_set_attribute(span, SpanAttributes.OUTPUT_MIME_TYPE, OpenInferenceMimeTypeValues.JSON.value)

litellm/integrations/arize/arize_phoenix.py

@@ -430,7 +430,8 @@ class ArizePhoenixLogger(OpenTelemetry):
 
         otlp_auth_headers = None
         if api_key is not None:
-            otlp_auth_headers = f"Authorization=Bearer {api_key}"
+            auth_header_key = "authorization" if protocol == "otlp_grpc" else "Authorization"
+            otlp_auth_headers = f"{auth_header_key}=Bearer {api_key}"
         elif "app.phoenix.arize.com" in endpoint:
             raise ValueError("PHOENIX_API_KEY must be set when using Phoenix Cloud (app.phoenix.arize.com).")
 

litellm/integrations/azure_sentinel/azure_sentinel.py

@@ -16,7 +16,10 @@ import asyncio
 import os
 import time
 import traceback
+from collections.abc import Mapping
+from types import MappingProxyType
 from typing import Final
+from urllib.parse import urlparse
 
 from litellm._logging import verbose_logger
 from litellm.integrations.custom_batch_logger import CustomBatchLogger
@@ -27,6 +30,16 @@ from litellm.llms.custom_httpx.http_handler import (
 )
 from litellm.types.utils import StandardAuditLogPayload, StandardLoggingPayload
 
+DEFAULT_AZURE_AUTHORITY_HOST: Final = "https://login.microsoftonline.com"
+DEFAULT_AZURE_MONITOR_SCOPE: Final = "https://monitor.azure.com/.default"
+
+MONITOR_SCOPE_BY_AUTHORITY_HOST: Final[Mapping[str, str]] = MappingProxyType(
+    {
+        "login.microsoftonline.com": DEFAULT_AZURE_MONITOR_SCOPE,
+        "login.microsoftonline.us": "https://monitor.azure.us/.default",
+    }
+)
+
 
 class AzureSentinelLogger(CustomBatchLogger):
     """
@@ -42,6 +55,7 @@ class AzureSentinelLogger(CustomBatchLogger):
         client_id: str | None = None,
         client_secret: str | None = None,
         audit_stream_name: str | None = None,
+        authority_host: str | None = None,
         **kwargs,
     ):
         """
@@ -62,6 +76,10 @@ class AzureSentinelLogger(CustomBatchLogger):
                 If not provided, will use AZURE_SENTINEL_CLIENT_SECRET or AZURE_CLIENT_SECRET env var.
             audit_stream_name (str, optional): Stream name from DCR for audit logs.
                 If not provided, will use AZURE_SENTINEL_AUDIT_STREAM_NAME env var or the standard stream name.
+            authority_host (str, optional): Microsoft Entra authority host that issues the OAuth2 token,
+                e.g. "https://login.microsoftonline.us" for Azure Government. If not provided, will use
+                AZURE_SENTINEL_AUTHORITY_HOST or AZURE_AUTHORITY_HOST env vars, or default to the Azure
+                Public Cloud authority. The Azure Monitor audience is derived from it.
         """
         self.async_httpx_client = get_async_httpx_client(llm_provider=httpxSpecialProvider.LoggingCallback)
 
@@ -76,6 +94,12 @@ class AzureSentinelLogger(CustomBatchLogger):
         resolved_client_secret: Final = (
             client_secret or os.getenv("AZURE_SENTINEL_CLIENT_SECRET") or os.getenv("AZURE_CLIENT_SECRET")
         )
+        resolved_authority_host: Final = self._normalize_authority_host(
+            authority_host
+            or os.getenv("AZURE_SENTINEL_AUTHORITY_HOST")
+            or os.getenv("AZURE_AUTHORITY_HOST")
+            or DEFAULT_AZURE_AUTHORITY_HOST
+        )
 
         if not resolved_dcr_immutable_id:
             raise ValueError(
@@ -119,7 +143,8 @@ class AzureSentinelLogger(CustomBatchLogger):
         )
 
         # OAuth2 scope for Azure Monitor
-        self.oauth_scope = "https://monitor.azure.com/.default"
+        self.authority_host = resolved_authority_host
+        self.oauth_scope = self._resolve_oauth_scope(authority_host=resolved_authority_host)
         self.oauth_token: str | None = None
         self.oauth_token_expires_at: float | None = None
 
@@ -129,6 +154,26 @@ class AzureSentinelLogger(CustomBatchLogger):
         self.log_queue: list[StandardLoggingPayload] = []
         self.audit_log_queue: list[StandardAuditLogPayload] = []
 
+    @staticmethod
+    def _normalize_authority_host(authority_host: str) -> str:
+        """
+        Normalize an authority host into an absolute URL with no trailing slash.
+
+        Accepts the scheme-qualified form litellm documents ("https://login.microsoftonline.us")
+        and the bare-host form the azure-identity AzureAuthorityHosts constants use.
+        """
+        stripped: Final = authority_host.strip().rstrip("/")
+        return stripped if "://" in stripped else f"https://{stripped}"
+
+    @staticmethod
+    def _resolve_oauth_scope(authority_host: str) -> str:
+        """
+        Map an authority host to the Azure Monitor Logs Ingestion audience for the same cloud,
+        falling back to the Azure Public Cloud audience for an unrecognized host.
+        """
+        host: Final = urlparse(authority_host).hostname or ""
+        return MONITOR_SCOPE_BY_AUTHORITY_HOST.get(host, DEFAULT_AZURE_MONITOR_SCOPE)
+
     @staticmethod
     def _build_api_endpoint(endpoint: str, dcr_immutable_id: str, stream_name: str) -> str:
         return f"{endpoint.rstrip('/')}/dataCollectionRules/{dcr_immutable_id}/streams/{stream_name}?api-version=2023-01-01"
@@ -150,7 +195,7 @@ class AzureSentinelLogger(CustomBatchLogger):
         assert self.client_id is not None, "client_id is required"
         assert self.client_secret is not None, "client_secret is required"
 
-        token_url: Final = f"https://login.microsoftonline.com/{self.tenant_id}/oauth2/v2.0/token"
+        token_url: Final = f"{self.authority_host}/{self.tenant_id}/oauth2/v2.0/token"
 
         token_data: Final = {
             "client_id": self.client_id,

litellm/integrations/braintrust_mock_client.py

@@ -10,6 +10,7 @@ Usage:
 
 import os
 import time
+from collections.abc import AsyncIterable, Iterable
 from typing import Final
 from urllib.parse import urlparse
 
@@ -84,7 +85,7 @@ def _mock_http_handler_post(
     timeout=None,
     stream=False,
     files=None,
-    content=None,
+    content: str | bytes | Iterable[bytes] | AsyncIterable[bytes] | None = None,
     logging_obj=None,
 ):
     """Monkey-patched HTTPHandler.post that intercepts Braintrust calls with endpoint-specific responses."""

litellm/integrations/custom_guardrail.py

@@ -102,6 +102,9 @@ class CustomGuardrail(CustomLogger):
     # If True, during_call runs async_moderation_hook instead of the unified apply_guardrail path.
     use_native_during_call_hook: ClassVar[bool] = False
 
+    # If True, every proxy lifecycle event runs this guardrail's own hooks, not apply_guardrail.
+    use_native_lifecycle_hooks: ClassVar[bool] = False
+
     records_own_guardrail_information: ClassVar[bool] = False
 
     def __init__(
@@ -198,6 +201,7 @@ class CustomGuardrail(CustomLogger):
         violation_message: str,
         request_data: dict[str, Any],
         detection_info: dict[str, Any] | None = None,
+        original_response: object = None,
     ) -> None:
         """
         Raise a passthrough exception for guardrail violations.
@@ -213,6 +217,10 @@ class CustomGuardrail(CustomLogger):
             violation_message: The formatted violation message to return to the user
             request_data: The original request data dictionary
             detection_info: Optional dictionary with detection metadata (scores, rules, etc.)
+            original_response: The blocked LLM response when raising from a post-call
+                hook. It carries the real token usage the upstream call consumed, so
+                the synthetic block response reports it instead of zeros. Leave None
+                for pre-call/during-call blocks (the LLM was never invoked).
 
         Raises:
             ModifyResponseException: Always raises this exception to short-circuit
@@ -235,6 +243,7 @@ class CustomGuardrail(CustomLogger):
             request_data=request_data,
             guardrail_name=self.guardrail_name,
             detection_info=detection_info,
+            original_response=original_response,
         )
 
     def raise_sensitive_data_route_exception(
@@ -626,7 +635,7 @@ class CustomGuardrail(CustomLogger):
         return type(self).apply_guardrail is not CustomGuardrail.apply_guardrail
 
     def _deployment_pre_call_target(self) -> "CustomLogger":
-        if not self.uses_apply_guardrail_interface():
+        if not self.uses_apply_guardrail_interface() or self.use_native_lifecycle_hooks:
             return self
         try:
             from litellm.proxy.utils import unified_guardrail
@@ -714,6 +723,29 @@ class CustomGuardrail(CustomLogger):
 
         return result
 
+    def supports_scan_only_tool_results(self) -> bool:
+        """Whether this guardrail can scan tool-result content.
+
+        Guardrails whose own role filtering only ever scans human-authored
+        messages override this to return False, so configuring them with
+        ``scan_only_tool_results`` is rejected at initialization instead of
+        silently scanning nothing on every request.
+        """
+        return True
+
+    def structured_messages_cover_full_request(self) -> bool:
+        """Whether returned ``structured_messages`` span the whole request.
+
+        Translation handlers hand guardrails only the in-scope subset of the
+        conversation and merge a returned ``structured_messages`` list back
+        into the full request. A guardrail that already rebuilds the complete
+        conversation itself (like CrowdStrike AIDR with its skip filters
+        active) overrides this to return True so the handler installs the
+        returned list as-is instead of merging it a second time, which would
+        duplicate the out-of-scope messages.
+        """
+        return False
+
     def should_run_guardrail(
         self,
         data,

litellm/integrations/email_templates/templates.py

@@ -54,7 +54,7 @@ USER_INVITED_EMAIL_TEMPLATE: Final = """
 
                     You were invited to use OpenAI Proxy API for team {team_name}  <br /> <br />
 
-                    <a href="{base_url}" style="display: inline-block; padding: 10px 20px; background-color: #87ceeb; color: #fff; text-decoration: none; border-radius: 20px;">Get Started here</a> <br /> <br />
+                    <a href="{base_url}" style="display: inline-block; padding: 10px 20px; background-color: #87ceeb; color: #fff; text-decoration: none; border-radius: 20px;">Accept Invitation</a> <br /> <br />
 
                     
                     If you have any questions, please send an email to {email_support_contact} <br /> <br />

litellm/integrations/email_templates/user_invitation_email.py

@@ -131,7 +131,7 @@ USER_INVITATION_EMAIL_TEMPLATE: Final = """
             </div>
 
             <div class="btn-container">
-                <a href="{base_url}" class="btn">Accept Invitation</a>
+                <a href="{invitation_link}" class="btn">Accept Invitation</a>
             </div>
             
             <div class="quickstart">

litellm/integrations/galileo.py

@@ -4,8 +4,9 @@ import json
 import os
 import re
 import uuid
-from datetime import datetime, timezone
-from typing import Any, Final, cast
+from collections.abc import Mapping, Sequence
+from datetime import datetime, timezone, tzinfo
+from typing import Any, Final, TypedDict, cast
 
 import httpx
 from pydantic import BaseModel, Field
@@ -34,6 +35,17 @@ GALILEO_CLOUD_API_BASE_URL: Final = "https://api.galileo.ai"
 GALILEO_MAX_IN_MEMORY_RECORDS: Final = 1000
 
 
+class GalileoStandardLoggingFields(TypedDict, total=False):
+    call_type: str
+    model: str
+    prompt_tokens: int
+    completion_tokens: int
+    total_tokens: int
+    response_cost: float
+    startTime: float
+    endTime: float
+
+
 class LLMResponse(BaseModel):
     latency_ms: int
     status_code: int
@@ -59,7 +71,7 @@ class LLMResponse(BaseModel):
 
 class GalileoObserve(CustomLogger):
     def __init__(self) -> None:
-        self.in_memory_records: list[dict] = []
+        self.in_memory_records: list[Mapping[str, object]] = []
         self.batch_size = 1
         self.api_key = os.getenv("GALILEO_API_KEY")
         self.project_id = os.getenv("GALILEO_PROJECT_ID")
@@ -176,7 +188,7 @@ class GalileoObserve(CustomLogger):
             return False
 
     @staticmethod
-    def _galileo_input_messages(messages: Any | None, input_text: str) -> list[dict[str, str]]:
+    def _galileo_input_messages(messages: object, input_text: str) -> list[dict[str, str]]:
         if isinstance(messages, dict):
             messages = messages.get("messages")
         if not messages:
@@ -203,11 +215,11 @@ class GalileoObserve(CustomLogger):
         return [{"role": "user", "content": input_text}]
 
     @staticmethod
-    def _local_timezone():
+    def _local_timezone() -> tzinfo:
         return datetime.now().astimezone().tzinfo or timezone.utc
 
     @staticmethod
-    def _format_created_at(dt: datetime | Any) -> str:
+    def _format_created_at(dt: object) -> str:
         """Serialize timestamps as UTC ISO-8601 for Galileo."""
         if not isinstance(dt, datetime):
             return str(dt)
@@ -226,7 +238,7 @@ class GalileoObserve(CustomLogger):
         return created_at
 
     @staticmethod
-    def _token_metrics_from_record(record: dict[str, Any]) -> dict[str, Any]:
+    def _token_metrics_from_record(record: Mapping[str, Any]) -> dict[str, Any]:
         num_input_tokens: Final = int(record.get("num_input_tokens") or 0)
         num_output_tokens: Final = int(record.get("num_output_tokens") or 0)
         num_total_tokens = int(record.get("num_total_tokens") or 0)
@@ -244,7 +256,7 @@ class GalileoObserve(CustomLogger):
 
     @staticmethod
     def _record_to_v2_span(
-        record: dict[str, Any],
+        record: Mapping[str, Any],
         *,
         trace_id: str,
         span_id: str,
@@ -275,7 +287,7 @@ class GalileoObserve(CustomLogger):
         return span
 
     @staticmethod
-    def _record_to_v2_trace(record: dict[str, Any]) -> dict[str, Any]:
+    def _record_to_v2_trace(record: Mapping[str, Any]) -> dict[str, Any]:
         trace_id: Final = str(uuid.uuid4())
         span_id: Final = str(uuid.uuid4())
         created_at: Final = GalileoObserve._normalize_created_at(record.get("created_at", ""))
@@ -295,7 +307,7 @@ class GalileoObserve(CustomLogger):
             "spans": [GalileoObserve._record_to_v2_span(record, trace_id=trace_id, span_id=span_id)],
         }
 
-    def _build_traces_payload(self, records: list[dict]) -> dict[str, Any]:
+    def _build_traces_payload(self, records: Sequence[Mapping[str, Any]]) -> dict[str, Any]:
         payload: Final[dict[str, Any]] = {
             "traces": [self._record_to_v2_trace(record) for record in records],
             "logging_method": "api_direct",
@@ -357,7 +369,7 @@ class GalileoObserve(CustomLogger):
     @staticmethod
     def _log_v2_payload_validation(payload: dict[str, Any]) -> None:
         missing_fields: Final[list[str]] = []
-        traces: Final = payload.get("traces", [])
+        traces: Final[Sequence[object]] = payload.get("traces", [])
         if not traces:
             missing_fields.append("traces")
 
@@ -385,7 +397,7 @@ class GalileoObserve(CustomLogger):
             )
 
     def _log_flush_payload(self, url: str, payload: dict[str, Any]) -> None:
-        traces: Final = payload.get("traces", [])
+        traces: Final[Sequence[object]] = payload.get("traces", [])
         verbose_logger.debug(
             "Galileo Logger flush URL: %s trace_count=%s",
             url,
@@ -415,8 +427,8 @@ class GalileoObserve(CustomLogger):
             pass
 
     @staticmethod
-    def _build_prompt(kwargs: dict[str, Any]) -> dict[str, Any]:
-        optional_params: Final = kwargs.get("optional_params", {}) or {}
+    def _build_prompt(kwargs: Mapping[str, Any]) -> dict[str, Any]:
+        optional_params: Final[Mapping[str, object]] = kwargs.get("optional_params", {}) or {}
         prompt: Final[dict[str, Any]] = {"messages": kwargs.get("messages")}
         if optional_params.get("functions") is not None:
             prompt["functions"] = optional_params["functions"]
@@ -425,13 +437,13 @@ class GalileoObserve(CustomLogger):
         return prompt
 
     @staticmethod
-    def _serialize_galileo_output(value: Any) -> str:
+    def _serialize_galileo_output(value: object) -> str:
         if value is None:
             return ""
         if isinstance(value, str):
             return value
 
-        def _json_default(obj: Any) -> Any:
+        def _json_default(obj: Any) -> object:
             if hasattr(obj, "model_dump"):
                 return obj.model_dump()
             return str(obj)
@@ -439,8 +451,8 @@ class GalileoObserve(CustomLogger):
         return json.dumps(value, default=_json_default)
 
     @staticmethod
-    def _prompt_to_input_text(prompt: dict[str, Any]) -> str:
-        messages: Final = prompt.get("messages")
+    def _prompt_to_input_text(prompt: Mapping[str, Any]) -> str:
+        messages: Final[object] = prompt.get("messages")
         if messages is not None:
             text: Final = GalileoObserve._input_text_from_messages(messages)
             if text:
@@ -448,7 +460,7 @@ class GalileoObserve(CustomLogger):
         return json.dumps(prompt, default=str)
 
     @staticmethod
-    def _get_chat_content_for_galileo(response_obj: litellm.ModelResponse) -> Any:
+    def _get_chat_content_for_galileo(response_obj: litellm.ModelResponse) -> object:
         if response_obj.choices and len(response_obj.choices) > 0:
             message: Final = response_obj["choices"][0]["message"]
             if hasattr(message, "json"):
@@ -470,23 +482,23 @@ class GalileoObserve(CustomLogger):
     @staticmethod
     def _get_responses_api_content_for_galileo(
         response_obj: ResponsesAPIResponse,
-    ) -> Any:
+    ) -> object:
         if hasattr(response_obj, "output") and response_obj.output:
             return response_obj.output
         return None
 
     @staticmethod
-    def _langfuse_style_rerank_prompt(kwargs: dict[str, Any]) -> dict[str, Any]:
+    def _langfuse_style_rerank_prompt(kwargs: Mapping[str, object]) -> dict[str, Any]:
         """Match Langfuse rerank input: prompt = {"messages": kwargs.get("messages")}."""
         return {"messages": kwargs.get("messages")}
 
     def _get_galileo_input_output_content(
         self,
-        kwargs: dict[str, Any],
-        response_obj: Any,
+        kwargs: Mapping[str, object],
+        response_obj: object,
         level: str = "DEFAULT",
         status_message: str | None = None,
-    ) -> tuple[str, str, Any]:
+    ) -> tuple[str, str, object]:
         """
         Mirror Langfuse _get_langfuse_input_output_content for Galileo ingest.
 
@@ -582,12 +594,12 @@ class GalileoObserve(CustomLogger):
 
         return self._prompt_to_input_text(prompt), "", kwargs.get("messages") or []
 
-    def get_output_str_from_response(self, response_obj: Any, kwargs: dict[str, Any]) -> str:
+    def get_output_str_from_response(self, response_obj: object, kwargs: Mapping[str, object]) -> str:
         _, output_text, _ = self._get_galileo_input_output_content(kwargs=kwargs, response_obj=response_obj)
         return output_text
 
     @staticmethod
-    def _input_text_from_messages(messages: Any) -> str:
+    def _input_text_from_messages(messages: object) -> str:
         """Return a plain-string summary of the input suitable for the trace-level input field."""
         if isinstance(messages, str):
             return messages
@@ -613,7 +625,13 @@ class GalileoObserve(CustomLogger):
                     return str(content)
         return ""
 
-    async def async_log_success_event(self, kwargs: Any, response_obj: Any, start_time: Any, end_time: Any):
+    async def async_log_success_event(
+        self,
+        kwargs: Mapping[str, object],
+        response_obj: object,
+        start_time: object,
+        end_time: object,
+    ) -> None:
         verbose_logger.debug("On Async Success")
         try:
             await self._async_log_success_event_impl(
@@ -625,7 +643,13 @@ class GalileoObserve(CustomLogger):
         except Exception:
             verbose_logger.exception("Galileo Logger: unexpected error in async_log_success_event")
 
-    async def _async_log_success_event_impl(self, kwargs: Any, response_obj: Any, start_time: Any, end_time: Any):
+    async def _async_log_success_event_impl(
+        self,
+        kwargs: Mapping[str, Any],
+        response_obj: object,
+        start_time: object,
+        end_time: object,
+    ) -> None:
         if not self._is_configured():
             verbose_logger.debug(
                 "Galileo Logger: skipping — GALILEO_PROJECT_ID=%s GALILEO_API_KEY=%s GALILEO_BASE_URL=%s",
@@ -635,7 +659,7 @@ class GalileoObserve(CustomLogger):
             )
             return
 
-        slo: Final[dict[str, Any] | None] = kwargs.get("standard_logging_object")
+        slo: Final[GalileoStandardLoggingFields | None] = kwargs.get("standard_logging_object")
         if slo is None:
             verbose_logger.debug("Galileo Logger: no standard_logging_object in kwargs, skipping")
             return
@@ -646,8 +670,8 @@ class GalileoObserve(CustomLogger):
             kwargs=kwargs, response_obj=response_obj
         )
 
-        raw_start: Final = slo.get("startTime")
-        raw_end: Final = slo.get("endTime")
+        raw_start: Final[float | None] = slo.get("startTime")
+        raw_end: Final[float | None] = slo.get("endTime")
         if raw_start is None or raw_end is None:
             verbose_logger.debug(
                 "Galileo Logger: standard_logging_object missing startTime/endTime, "
@@ -710,7 +734,7 @@ class GalileoObserve(CustomLogger):
         if len(self.in_memory_records) >= self.batch_size:
             await self.flush_in_memory_records()
 
-    async def flush_in_memory_records(self):
+    async def flush_in_memory_records(self) -> None:
         if not self.in_memory_records:
             return
 
@@ -774,5 +798,11 @@ class GalileoObserve(CustomLogger):
             if not self.use_v2_api and response.status_code in (401, 403):
                 self.headers = None
 
-    async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
+    async def async_log_failure_event(
+        self,
+        kwargs: Mapping[str, object],
+        response_obj: object,
+        start_time: object,
+        end_time: object,
+    ) -> None:
         verbose_logger.debug("On Async Failure")

litellm/integrations/gcs_bucket/gcs_bucket_mock_client.py

@@ -9,6 +9,7 @@ Usage:
 """
 
 import asyncio
+from collections.abc import AsyncIterable, Iterable
 from typing import Final
 
 from litellm._logging import verbose_logger
@@ -113,7 +114,7 @@ async def _mock_async_handler_delete(
     headers=None,
     timeout=None,
     stream=False,
-    content=None,
+    content: str | bytes | Iterable[bytes] | AsyncIterable[bytes] | None = None,
 ):
     """Monkey-patched AsyncHTTPHandler.delete that intercepts GCS calls."""
     # Only mock GCS API calls

litellm/integrations/generic_api/generic_api_callback.py

@@ -11,7 +11,7 @@ import json
 import os
 import re
 import traceback
-from typing import Any, Final, Literal
+from typing import Final, Literal
 
 import httpx
 
@@ -158,7 +158,7 @@ class GenericAPILogger(CustomBatchLogger):
                 "endpoint not set for GenericAPILogger, GENERIC_LOGGER_ENDPOINT not found in environment variables"
             )
 
-        self.headers: dict = self._get_headers(headers)
+        self.headers: dict[str, str] = self._get_headers(headers)
         self.endpoint: str = endpoint
         self.event_types: list[API_EVENT_TYPES] | None = event_types
         self.callback_name: str | None = callback_name
@@ -248,18 +248,15 @@ class GenericAPILogger(CustomBatchLogger):
         await asyncio.sleep(delay)
 
     async def _post_with_retries(self, data: str) -> httpx.Response:
-        post_kwargs: Final[dict[str, Any]] = {
-            "url": self.endpoint,
-            "headers": self.headers,
-            "data": data,
-        }
-        if self.timeout is not None:
-            post_kwargs["timeout"] = self.timeout
-
         total_attempts: Final = self.max_retries + 1
         for attempt in range(total_attempts):
             try:
-                return await self.async_httpx_client.post(**post_kwargs)
+                return await self.async_httpx_client.post(
+                    url=self.endpoint,
+                    headers=self.headers,
+                    data=data,
+                    timeout=self.timeout,
+                )
             except Exception as e:
                 is_last_attempt = attempt == self.max_retries
                 should_retry = self._should_retry_exception(e)

litellm/integrations/langfuse/langfuse.py

@@ -2,8 +2,9 @@
 #    On success, logs events to Langfuse
 import os
 import traceback
-from collections.abc import Callable
+from collections.abc import Callable, Iterable, Mapping
 from datetime import datetime
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, cast
 
 from packaging.version import Version
@@ -30,6 +31,7 @@ from litellm.types.utils import (
     ImageResponse,
     ModelResponse,
     RerankResponse,
+    StandardLoggingMetadata,
     StandardLoggingPayload,
     StandardLoggingPromptManagementMetadata,
     TextCompletionResponse,
@@ -46,6 +48,11 @@ else:
     Langfuse = Any
 
 
+_DENIED_STEERING_KEYS: Final = frozenset({"headers", "endpoint", "caching_groups", "previous_models"})
+_NO_METADATA: Final[Mapping[str, Any]] = MappingProxyType({})
+_REDACTED_PROXY_HEADERS: Final[frozenset[str]] = frozenset({"authorization", "cookie", "referer"})
+
+
 def _extract_cache_read_input_tokens(usage_obj) -> int:
     """
     Extract cache_read_input_tokens from usage object.
@@ -75,6 +82,22 @@ def _extract_cache_read_input_tokens(usage_obj) -> int:
     return cache_read_input_tokens
 
 
+def _as_steering_flag(value: object) -> bool:
+    """A string ``str_to_bool`` does not recognise falls back to its truthiness."""
+    if isinstance(value, str):
+        parsed: Final = str_to_bool(value)
+        return bool(value) if parsed is None else parsed
+    return bool(value)
+
+
+def _as_steering_key_sequence(value: object) -> tuple[str, ...]:
+    if isinstance(value, str):
+        return tuple(key.strip() for key in value.split(",") if key.strip())
+    if isinstance(value, Iterable):
+        return tuple(str(key) for key in value)
+    return ()
+
+
 def resolve_langfuse_credentials(
     langfuse_public_key=None,
     langfuse_secret=None,
@@ -496,16 +519,14 @@ class LangFuseLogger:
                 else []
             )
 
-            if standard_logging_object is None:
-                end_user_id = None
-                prompt_management_metadata: StandardLoggingPromptManagementMetadata | None = None
-            else:
-                end_user_id = standard_logging_object["metadata"].get("user_api_key_end_user_id", None)
-
-                prompt_management_metadata = cast(
-                    StandardLoggingPromptManagementMetadata | None,
-                    standard_logging_object["metadata"].get("prompt_management_metadata", None),
-                )
+            allowlisted_metadata: Final[StandardLoggingMetadata | dict[str, Any]] = (
+                standard_logging_object["metadata"] if standard_logging_object is not None else _NO_METADATA
+            )
+            end_user_id: Final = allowlisted_metadata.get("user_api_key_end_user_id", None)
+            prompt_management_metadata: Final[StandardLoggingPromptManagementMetadata | None] = cast(
+                StandardLoggingPromptManagementMetadata | None,
+                allowlisted_metadata.get("prompt_management_metadata", None),
+            )
 
             # Clean Metadata before logging - never log raw metadata
             # the raw metadata can contain circular references which leads to infinite recursion
@@ -524,12 +545,7 @@ class LangFuseLogger:
                         tags.append(f"{key}:{value}")
 
                     # clean litellm metadata before logging
-                    if key in [
-                        "headers",
-                        "endpoint",
-                        "caching_groups",
-                        "previous_models",
-                    ]:
+                    if key in _DENIED_STEERING_KEYS:
                         continue
                     else:
                         clean_metadata[key] = value
@@ -552,10 +568,13 @@ class LangFuseLogger:
             # This allows continuing an existing trace while still returning the correct trace_id
             if existing_trace_id is not None:
                 trace_id = existing_trace_id
-            update_trace_keys: Final = cast(list, clean_metadata.pop("update_trace_keys", []))
+            requested_trace_keys: Final = _as_steering_key_sequence(clean_metadata.pop("update_trace_keys", ()))
+            update_trace_keys: Final = (
+                requested_trace_keys if _as_steering_flag(litellm.langfuse_enable_update_trace_keys) else ()
+            )
             debug: Final = clean_metadata.pop("debug_langfuse", None)
-            mask_input: Final = clean_metadata.pop("mask_input", False)
-            mask_output: Final = clean_metadata.pop("mask_output", False)
+            mask_input: Final = _as_steering_flag(clean_metadata.pop("mask_input", False))
+            mask_output: Final = _as_steering_flag(clean_metadata.pop("mask_output", False))
             # Look for masking function in the dedicated location first (set by scrub_sensitive_keys_in_metadata)
             # Fall back to metadata for backwards compatibility
             masking_function: Final = litellm_params.get("_langfuse_masking_function") or clean_metadata.pop(
@@ -614,19 +633,18 @@ class LangFuseLogger:
                     trace_params["output"] = output if not mask_output else "redacted-by-litellm"
 
             if debug is True or (isinstance(debug, str) and debug.lower() == "true"):
-                if "metadata" in trace_params:
-                    # log the raw_metadata in the trace
-                    trace_params["metadata"]["metadata_passed_to_litellm"] = metadata
-                else:
-                    trace_params["metadata"] = {"metadata_passed_to_litellm": metadata}
+                debug_metadata: Final = {
+                    key: value for key, value in metadata.items() if isinstance(value, (str, int, float, bool))
+                }
+                trace_params["metadata"] = {
+                    **(trace_params.get("metadata") or _NO_METADATA),
+                    "metadata_passed_to_litellm": debug_metadata,
+                }
 
             cost: Final = kwargs.get("response_cost", None)
             verbose_logger.debug("trace: %s", cost)
 
-            clean_metadata["litellm_response_cost"] = cost
-            if standard_logging_object is not None:
-                hidden_params: Final = standard_logging_object.get("hidden_params", {})
-                clean_metadata["hidden_params"] = filter_exceptions_from_params(hidden_params)
+            hidden_params: Final = standard_logging_object.get("hidden_params") if standard_logging_object else None
 
             if (
                 litellm.langfuse_default_tags is not None
@@ -638,22 +656,24 @@ class LangFuseLogger:
                     tags.append(f"proxy_base_url:{proxy_base_url}")
 
             api_base: Final = litellm_params.get("api_base", None)
-            if api_base:
-                clean_metadata["api_base"] = api_base
-
             vertex_location: Final = kwargs.get("vertex_location", None)
-            if vertex_location:
-                clean_metadata["vertex_location"] = vertex_location
-
             aws_region_name: Final = kwargs.get("aws_region_name", None)
-            if aws_region_name:
-                clean_metadata["aws_region_name"] = aws_region_name
+
+            candidate_enrichments: Final = (
+                ("litellm_response_cost", cost, True),
+                ("hidden_params", filter_exceptions_from_params(hidden_params), hidden_params is not None),
+                ("api_base", api_base, bool(api_base)),
+                ("vertex_location", vertex_location, bool(vertex_location)),
+                ("aws_region_name", aws_region_name, bool(aws_region_name)),
+                ("cache_hit", kwargs.get("cache_hit") or False, self._supports_tags() and "cache_hit" in kwargs),
+            )
+            enrichments: Final[Mapping[str, Any]] = {
+                key: value for key, value, include in candidate_enrichments if include
+            }
 
             if self._supports_tags():
-                if "cache_hit" in kwargs:
-                    if kwargs["cache_hit"] is None:
-                        kwargs["cache_hit"] = False
-                    clean_metadata["cache_hit"] = kwargs["cache_hit"]
+                if "cache_hit" in kwargs and kwargs["cache_hit"] is None:
+                    kwargs["cache_hit"] = False  # rebind-ok: pre-existing normalization other integrations rely on
                 if existing_trace_id is None:
                     trace_params.update({"tags": tags})
 
@@ -666,13 +686,13 @@ class LangFuseLogger:
                 if headers:
                     for key, value in headers.items():
                         # these headers can leak our API keys and/or JWT tokens
-                        if key.lower() not in ["authorization", "cookie", "referer"]:
+                        if key.lower() not in _REDACTED_PROXY_HEADERS:
                             clean_headers[key] = value
 
             trace: Final[StatefulTraceClient] = self.Langfuse.trace(**trace_params)
 
             # Log provider specific information as a span
-            log_provider_specific_information_as_span(trace, clean_metadata)
+            log_provider_specific_information_as_span(trace, enrichments)
 
             # Log guardrail information as a span
             self._log_guardrail_information_as_span(
@@ -745,7 +765,10 @@ class LangFuseLogger:
                 "output": output if not mask_output else "redacted-by-litellm",
                 "usage": usage,
                 "usage_details": usage_details,
-                "metadata": log_requester_metadata(clean_metadata),
+                "metadata": {
+                    **log_requester_metadata(redact_user_api_key_info(metadata=allowlisted_metadata)),
+                    **enrichments,
+                },
                 "level": level,
                 "version": clean_metadata.pop("version", None),
             }
@@ -1042,7 +1065,7 @@ def _add_prompt_to_generation_params(
 
 def log_provider_specific_information_as_span(
     trace,
-    clean_metadata,
+    clean_metadata: Mapping[str, Any],
 ):
     """
     Logs provider-specific information as spans.
@@ -1082,7 +1105,7 @@ def log_provider_specific_information_as_span(
             )
 
 
-def log_requester_metadata(clean_metadata: dict):
+def log_requester_metadata(clean_metadata: Mapping[str, Any]):
     returned_metadata: Final = {}
     requester_metadata: Final = clean_metadata.get("requester_metadata") or {}
     for k, v in clean_metadata.items():

litellm/integrations/langfuse/langfuse_otel.py

@@ -90,7 +90,6 @@ class LangfuseOtelLogger(OpenTelemetry):
             "generation_name": LangfuseSpanAttributes.GENERATION_NAME,
             "generation_id": LangfuseSpanAttributes.GENERATION_ID,
             "parent_observation_id": LangfuseSpanAttributes.PARENT_OBSERVATION_ID,
-            "version": LangfuseSpanAttributes.GENERATION_VERSION,
             "mask_input": LangfuseSpanAttributes.MASK_INPUT,
             "mask_output": LangfuseSpanAttributes.MASK_OUTPUT,
             "trace_user_id": LangfuseSpanAttributes.TRACE_USER_ID,
@@ -99,13 +98,18 @@ class LangfuseOtelLogger(OpenTelemetry):
             "trace_name": LangfuseSpanAttributes.TRACE_NAME,
             "trace_id": LangfuseSpanAttributes.TRACE_ID,
             "trace_metadata": LangfuseSpanAttributes.TRACE_METADATA,
-            "trace_version": LangfuseSpanAttributes.TRACE_VERSION,
-            "trace_release": LangfuseSpanAttributes.TRACE_RELEASE,
+            "trace_release": LangfuseSpanAttributes.RELEASE,
             "existing_trace_id": LangfuseSpanAttributes.EXISTING_TRACE_ID,
             "update_trace_keys": LangfuseSpanAttributes.UPDATE_TRACE_KEYS,
             "debug_langfuse": LangfuseSpanAttributes.DEBUG_LANGFUSE,
         }
 
+        version: Final = (
+            metadata.get("trace_version") if metadata.get("trace_version") is not None else metadata.get("version")
+        )
+        if version is not None:
+            safe_set_attribute(span, LangfuseSpanAttributes.VERSION.value, version)
+
         for key, enum_attr in mapping.items():
             if key in metadata and metadata[key] is not None:
                 value = metadata[key]

litellm/integrations/mock_client_factory.py

@@ -8,6 +8,7 @@ making actual network calls.
 
 import asyncio
 import json
+from collections.abc import AsyncIterable, Iterable
 from dataclasses import dataclass
 from datetime import timedelta
 from typing import Final, cast
@@ -140,7 +141,7 @@ def create_mock_client_factory(config: MockClientConfig):
         stream=False,
         logging_obj=None,
         files=None,
-        content=None,
+        content: str | bytes | Iterable[bytes] | AsyncIterable[bytes] | None = None,
     ):
         """Monkey-patched AsyncHTTPHandler.post that intercepts API calls."""
         if isinstance(url, str) and _is_mock_url(url):
@@ -193,7 +194,7 @@ def create_mock_client_factory(config: MockClientConfig):
         timeout=None,
         stream=False,
         files=None,
-        content=None,
+        content: str | bytes | Iterable[bytes] | AsyncIterable[bytes] | None = None,
         logging_obj=None,
     ):
         """Monkey-patched HTTPHandler.post that intercepts API calls."""