Merge branch 'main' of https://github.com/BerriAI/litellm into responses-tool-search-lowering

# Conflicts:
#	litellm/llms/base_llm/responses/transformation.py

.github/CODEOWNERS

@@ -1,5 +1,6 @@
 /model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
 /litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
+/ci_cd/cost_map_pins.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
 /helm/ @yassin-berriai
 /terraform/ @yassin-berriai
 /.github/workflows/zizmor.yml @yuneng-berri

.github/PULL_REQUEST_TEMPLATE/general.md

@@ -84,9 +84,10 @@ If you're seeing a delay in your PR being merged, ping the LiteLLM Team on [Slac
 ## Screenshots / Proof of Fix
 
 <!-- Include screenshots, screen recordings, or command (e.g., curl) + output demonstrating that your changes work as expected
-     The proof must be completely e2e with no mocks, using actual LLM calls costing real $$$ if applicable. `pytest` commands are not enough
-     Show ONLY the latest run: capture Before at the merge base and After at the PR's current tip, and when new commits change behavior, replace this whole section with the fresh run instead of stacking it on top of older ones. The run must be up to date. As soon as a new commit is made and it makes this PR description's after sha stale (it's no longer tip of PR), you must re-run the QA
-     Structure the section exactly as below: Before and After one heading level below this section, each naming the commit hash it was captured at, one lower-level heading per case inside each, the same case names in the same order on both sides, and numbered steps (command, observed output) under every case, never loose prose; shared setup (config, payloads) goes above Before, and with a single case, drop the case headings and number the steps directly
+     Simple PR exception (2026-10-28, #49030). A mechanical change fully proven by focused checks may skip before/after e2e. Follow AGENTS.md's eligibility rule, trace callers, and verify bug-fix regression sensitivity. Record E2E exception at <tip SHA>, scope, rationale, commands and results here. Tests/static checks suffice only for this exception. Keep CI and bot gates, reassess changed diffs, and honor explicit e2e requests. Omit the Before/After scaffold for an exception
+     Otherwise proof must be fully e2e, no mocks, with real provider calls where applicable. Pytest alone is insufficient. The following format applies to live proof
+     Show only the latest proof. Before uses the merge base and After the tested tip. Rerun affected QA after behavior changes. Verified neutral diffs may carry proof with the tested SHA, current tip, equivalence reason and focused checks, following AGENTS.md and /qa
+     Use matching Before/After case headings with numbered commands and observed output. Put shared setup above them. For one case, omit case headings
 
 ### Before (<hash>)
 

.github/workflows/cost-map-guard.yml

@@ -21,16 +21,18 @@ jobs:
         with:
           ref: ${{ github.event.pull_request.base.sha }}
           persist-credentials: false
-      - name: Fetch the pull request head and its merge base
+      - name: Fetch the pull request head, its merge base, and the base branch tip
         id: revisions
         env:
           GH_TOKEN: ${{ github.token }}
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          BASE_REF: ${{ github.event.pull_request.base.ref }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
         run: |
-          merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
-          git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
+          target="$(gh api "repos/${GITHUB_REPOSITORY}/branches/${BASE_REF}" --jq '.commit.sha')"
+          merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${target}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
+          git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA" "$target"
           echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"
+          echo "target=$target" >> "$GITHUB_OUTPUT"
       - name: Set up uv
         uses: ./.github/actions/setup-uv-with-retries
         with:
@@ -38,6 +40,8 @@ jobs:
       - name: Run the guard
         env:
           MERGE_BASE: ${{ steps.revisions.outputs.merge_base }}
+          COST_MAP_GUARD_TARGET: ${{ steps.revisions.outputs.target }}
+          COST_MAP_GUARD_AUTHOR_TYPE: ${{ github.event.pull_request.user.type }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
           HEAD_REF: ${{ github.event.pull_request.head.ref }}
         run: |

.github/workflows/cost-map-pins-recheck.yml

@@ -0,0 +1,91 @@
+name: Cost map pins recheck
+
+on:
+  push:
+    branches:
+      - main
+      - "litellm_**"
+    paths:
+      - model_prices_and_context_window.json
+      - litellm/model_prices_and_context_window_backup.json
+      - ci_cd/cost_map_pins.json
+
+permissions: {}
+
+jobs:
+  rerun-cost-map-guard:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    permissions:
+      actions: write
+      contents: read
+      pull-requests: read
+    steps:
+      - name: Re-run the cost map guard on open pull requests the push could break
+        env:
+          GH_TOKEN: ${{ github.token }}
+          BASE_REF: ${{ github.ref_name }}
+          BEFORE_SHA: ${{ github.event.before }}
+          AFTER_SHA: ${{ github.event.after }}
+          FORCED: ${{ github.event.forced }}
+          COST_MAP_PATHS: '["model_prices_and_context_window.json", "litellm/model_prices_and_context_window_backup.json", "ci_cd/cost_map_pins.json"]'
+          PINS_PATHS: '["ci_cd/cost_map_pins.json"]'
+        run: |
+          if [[ "$BEFORE_SHA" =~ ^0+$ || "$FORCED" == true ]] \
+            || ! gh api "repos/${GITHUB_REPOSITORY}/compare/${BEFORE_SHA}...${AFTER_SHA}?per_page=1" \
+              --jq '(env.PINS_PATHS | fromjson) as $pins | (.files | length) < 300 and all(.files[]; [.filename, .previous_filename] | any(IN($pins[])) | not)' \
+              | grep -qx true; then
+            export RECHECK_PATHS="$COST_MAP_PATHS"
+          else
+            export RECHECK_PATHS="$PINS_PATHS"
+          fi
+          rerun_cutoff="$(date -u -d '30 days ago' +%Y-%m-%dT%H:%M:%SZ)"
+          candidates="$(gh api graphql --paginate -F owner="${GITHUB_REPOSITORY_OWNER}" -F name="${GITHUB_REPOSITORY#*/}" -F base="$BASE_REF" \
+            -f query='query($owner: String!, $name: String!, $base: String!, $endCursor: String) {
+              repository(owner: $owner, name: $name) {
+                pullRequests(states: OPEN, baseRefName: $base, first: 25, after: $endCursor) {
+                  nodes { number headRefOid changedFiles files(first: 100) { nodes { path } } }
+                  pageInfo { hasNextPage endCursor }
+                }
+              }
+            }' \
+            --jq '(env.RECHECK_PATHS | fromjson) as $paths | .data.repository.pullRequests.nodes[]
+              | {number, headRefOid, listed: any(.files.nodes[].path; IN($paths[])), unlisted: (.changedFiles > 100)}
+              | select(.listed or .unlisted) | "\(.number) \(.headRefOid) \(.listed)"')"
+          failed=0
+          declare -A rerun_heads
+          while read -r number head_sha listed; do
+            [ -n "$number" ] || continue
+            if [ "$listed" = false ]; then
+              if ! changed="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/pulls/${number}/files?per_page=100" \
+                --jq '(env.RECHECK_PATHS | fromjson) as $paths | .[].filename | select(IN($paths[]))')"; then
+                echo "::error::could not list the files of #$number"
+                failed=1
+                continue
+              fi
+              [ -n "$changed" ] || continue
+            fi
+            [ -z "${rerun_heads[$head_sha]:-}" ] || continue
+            rerun_heads[$head_sha]=1
+            if ! runs="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow cost-map-guard.yml --commit "$head_sha" --limit 100 --json databaseId,createdAt --jq '.[] | "\(.databaseId) \(.createdAt)"')"; then
+              echo "::error::could not list the cost map guard runs of #$number"
+              failed=1
+              continue
+            fi
+            if [ -z "$runs" ]; then
+              echo "::warning::no cost map guard run found for #$number"
+              continue
+            fi
+            while read -r run_id created_at; do
+              if [[ "$created_at" < "$rerun_cutoff" ]]; then
+                echo "::warning::cost map guard run $run_id for #$number is over 30 days old, so GitHub cannot re-run it; #$number gets the pin checks on its next push"
+                continue
+              fi
+              gh run watch "$run_id" --repo "$GITHUB_REPOSITORY" > /dev/null || true
+              if ! gh run rerun "$run_id" --repo "$GITHUB_REPOSITORY"; then
+                echo "::error::could not re-run cost map guard run $run_id for #$number"
+                failed=1
+              fi
+            done <<< "$runs"
+          done <<< "$candidates"
+          exit "$failed"

.github/workflows/test-unit.yml

@@ -236,6 +236,7 @@ jobs:
               tests/unit/llms/azure_ai
               tests/unit/llms/base_llm
               tests/unit/llms/baseten
+              tests/unit/llms/bigmodel
               tests/unit/llms/black_forest_labs
               tests/unit/llms/brave
               tests/unit/llms/bytez

.grype.yaml

@@ -11,3 +11,12 @@ ignore:
     package:
       name: zlib
       type: apk
+  - vulnerability: GHSA-v4x8-3wr5-3wq8
+    reason: >-
+      filelock ReadWriteLock advisory, and no package in the image takes ReadWriteLock;
+      the fix (4.2.1) is blocked by litellm's own filelock<4.0 cap until #48635 or #48802 lifts it,
+      and the image then moves off 3.32.7, which retires this version-scoped rule
+    package:
+      name: filelock
+      version: 3.32.7
+      type: python

AGENTS.md

@@ -31,6 +31,8 @@ A test must only fail when litellm code changes. Never pin facts we don't own (a
 
 End-to-end tests belong in `tests/e2e/`. A rule about one directory's files, like its harness or which CI runs it, goes in that directory's `AGENTS.md`, never only in a README or here. Read a directory's `AGENTS.md` before editing files in it
 
+Before you build a new way to move data between the endpoints, the provider transformations, and cost tracking or logging, read the architecture decision records in `docs/adrs/`, which say why those layers are shaped the way they are
+
 When creating PRs, target the repository's current default branch for both internal and external / OSS contributions. Check it with `python3 scripts/default_branch.py --branch` instead of assuming a branch name or relying on cached `origin/HEAD`
 
 When writing a PR body, treat the comments and imperative instructions inside .github/PULL_REQUEST_TEMPLATE/general.md as rules to follow, not just layout. Agent harnesses may strip HTML comments from copies of that file injected into context, so read .github/PULL_REQUEST_TEMPLATE/general.md from disk before writing a PR body to make sure you see every comment rule. A section you have nothing to put in (Relevant issues, Affected release, Linear ticket, Caveats, QA runbook, and so on) is removed entirely, heading included, never left as an empty title
@@ -39,7 +41,9 @@ Same applies for filing bug reports and feature requests, with .github/ISSUE_TEM
 
 If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just drop the section
 
-Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`. The Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into. I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, or the main use case runs through a headful agentic coding tool like Claude Code or Codex, drive that surface yourself and embed your own before and after screenshots of it in the PR (the Admin UI page, or what the coding tool shows), next to an ordered list of the URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, and what fields to fill out so a reviewer can reproduce it
+Simple PRs may skip before/after e2e when the full diff is mechanical and focused checks completely prove it, such as #49030 restoring an existing helper name. Trace affected callers and run relevant tests/static checks at the tip. For a bug fix, establish that the regression catches the original defect. Size alone never qualifies. New behavior, API/provider semantics, auth, billing, persistence, concurrency or uncertain integration risk need live e2e. In Screenshots / Proof of Fix, record `E2E exception at <tip SHA>`, scope, rationale, commands and results. Tests count as proof only for this exception. Reassess changed diffs, retain CI/bot gates, and honor explicit e2e requests (2026-10-28, unnecessary live QA for #49030)
+
+For other PRs, proof must show the end user's real flow before and after, with commands and output against a live proxy and real provider calls where applicable. Pytest alone is insufficient. For UI or interactive coding-tool flows, drive that surface and embed before/after screenshots, with URLs, clicks and input fields a reviewer can reproduce
 
 If you ever write any human-facing text (pull requests, issues, commit messages, discussion posts, github comments, release notes, docs, etc.), always follow these guidelines to sound less AI-y:
 - don't use emojis
@@ -105,6 +109,7 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega
 - No file sprawl: deliberate file and folder structure
 - Standard over hand-rolled: use the official SDK or a library where one exists. Where none does, follow industry standards instead of inventing local conventions
 - API-fragmentation-aware: when logic must branch on which API surface produced or consumes data (e.g. chat completions vs Anthropic Messages vs Responses API shapes), proactively look for an existing shared helper (e.g. `litellm_core_utils/prompt_templates/factory.py`) before writing per-surface parsing in the new module. If none exists, add one there instead of duplicating the same format-detection logic in every new guardrail/integration
+- Current default for new model names: logic that picks behavior from a model-name table (tokenizer, message framing, supported params) gives a provider's model the table does not list yet that provider's current behavior, never its oldest, since providers ship names faster than any table is updated
 
 Follow conventional commits for commit names and PR titles
 

ARCHITECTURE.md

@@ -2,6 +2,8 @@
 
 This document helps contributors understand where to make changes in LiteLLM.
 
+For why a layer is shaped the way it is, read the architecture decision records in [`docs/adrs/`](docs/adrs/AGENTS.md) (Python SDK and proxy) and [`litellm-rust/docs/adrs/`](litellm-rust/docs/adrs/AGENTS.md) (Rust core)
+
 ---
 
 ## How It Works

CONTRIBUTING.md

@@ -8,6 +8,7 @@ Here are the core requirements for any PR submitted to LiteLLM:
 
 - [ ] **Sign the Contributor License Agreement (CLA)** - [see details](#contributor-license-agreement-cla)
 - [ ] **Keep scope isolated** - Your changes should address 1 specific problem at a time
+- [ ] **Prove it works end to end** - If your change affects what the proxy or SDK does, paste the commands you ran against a live proxy and their output in the PR - [see details](#proving-your-change-works-end-to-end)
 
 #### Proxy (Backend) PRs
 
@@ -248,6 +249,41 @@ If `make test-unit` fails:
 - **Test edge cases**: Don't just test the happy path
 - **Update documentation**: If you change APIs, update docs
 
+## Proving Your Change Works End to End
+
+Tests are required, but they do not prove that your change works for a user. A PR that changes what the proxy or SDK does shows a real run in its "Screenshots / Proof of Fix" section, meaning the exact commands you sent to a live proxy and the output that came back, against real provider APIs and not mocks. `pytest` output does not count as proof, because a passing test only shows that the code does what its mocks expect. A PR that changes only docs or CI shows the rendered page or the CI run instead
+
+A good proof looks like this:
+
+1. Install the proxy dependencies with `make install-proxy-dev`, then start the proxy from your branch with a config that has the model you changed, `uv run litellm --config your_config.yaml --detailed_debug`
+2. Send the request a user would send with `curl`, and paste both the command and the response. When cost or routing is involved, include the response headers, since `x-litellm-response-cost` is what the user sees
+3. For a bug fix, do this twice, once at the merge base with BerriAI/litellm's main (`git merge-base origin/main HEAD`, with your remote for BerriAI/litellm in place of `origin` in a fork clone) to show the bug and once at your branch's latest commit to show the fix. Name both commit hashes
+4. If your change is reachable from more than one endpoint (`/v1/chat/completions`, `/v1/responses`, `/v1/messages`), show each one
+5. If the request can stream, show a `"stream": true` run too, because streaming builds usage and cost on a different path. A stream sends its headers before its usage exists, so show the `usage.cost` on the last chunk, which `/v1/chat/completions` sends when the request has `"stream_options": {"include_usage": true}` and other endpoints send with `include_cost_in_streaming_usage: true` under `litellm_settings`, or, when the proxy has a database, the request's row on the Admin UI's Logs page or `GET /spend/logs`
+6. For UI changes, add before and after screenshots and name the page
+7. If users reach your change through a coding tool such as Claude Code or Codex, drive that tool the way a user does and add before and after screenshots of it, because a `curl` or a headless run alone does not show what the user sees
+
+```bash
+curl -sD - http://localhost:4000/v1/chat/completions \
+  -H "Authorization: Bearer $LITELLM_MASTER_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{"model": "your-model", "messages": [{"role": "user", "content": "hi"}]}'
+```
+
+When the provider reports its own usage or spend numbers, compare the proxy's numbers against them in the proof
+
+## Contributing an LLM Provider Integration
+
+Provider work (a new provider, a new parameter, a new usage or cost field) is the most common kind of contribution, and it is where PRs get reworked most often. Follow these rules
+
+**Find the existing machinery before you add plumbing.** LiteLLM already has generic paths that move data between the endpoints, the provider transformations, and cost tracking. A second, hand-rolled path for the same data is a common reason a provider PR gets rewritten before merge. [`ARCHITECTURE.md`](ARCHITECTURE.md) shows where the layers live, and the decision records in [`docs/adrs/`](docs/adrs/AGENTS.md) explain why they are shaped that way. For example, [the ADR on provider usage fields](docs/adrs/adr_001_provider_usage_fields_ride_as_usage_extras.md) covers how provider-specific usage fields reach cost tracking. If you cannot find a mechanism for what you need, say so in the PR or ask in [#pr-review on Slack](https://join.slack.com/t/litellmossslack/shared_invite/zt-3o7nkuyfr-p_kbNJj8taRfXGgQI1~YyA) before you build one
+
+**Bill a new provider usage field from the usage object.** Leave the field on the usage object as an extra and read it in that provider's `cost_calculator.py`. A provider without one gets a new `cost_calculator.py` and an arm for it in `cost_per_token` (`litellm/cost_calculator.py`), which picks the calculator by provider. If the provider's transform builds `Usage` from named fields, as the Anthropic, Gemini, and Bedrock Converse ones do, pass the new field there too. Streamed requests rebuild usage in `ChunkProcessor.calculate_usage` (`litellm/litellm_core_utils/streaming_chunk_builder_utils.py`), which does not copy extras yet the way the Responses-to-chat bridge does (minus a short exclusion list), so until it does, bill the field on streams by naming it there, as Anthropic's `inference_geo` is. A provider that reports its own charge for the request, as OpenRouter does in `usage.cost`, is billed at that charge from `_hidden_params` before any calculator runs, unless the deployment sets its own prices. When the field counts a built-in tool such as web search, add the provider to `get_cost_for_web_search_request` in `litellm/llms/__init__.py`, and add a gate to `litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py` only when no existing gate fires. Then show the field's cost in your proof on every endpoint it reaches, streaming included
+
+**Do not change what a caller sees to make internals easier.** Each endpoint keeps the schema of the API it emulates. A `/v1/responses` caller gets `input_tokens` and a `/v1/chat/completions` caller gets `prompt_tokens`, whatever the provider sent and whatever cost tracking needs. A provider usage field that shows up as an extra key is fine, since it only adds a key, but renaming, removing, or reshaping a key the endpoint defines is not. Reshaping a public response to feed an internal consumer breaks every user of that endpoint
+
+If your change makes an architecture decision, such as a new cross-cutting mechanism or a deliberate difference from a provider's API shape, add an ADR to [`docs/adrs/`](docs/adrs/AGENTS.md) in the same PR
+
 ## Building and Running Locally
 
 ### LiteLLM Proxy Server

ci_cd/cost_map_guard.py

@@ -1,37 +1,67 @@
 """Guard the cost map on pull requests.
 
-Every pull request whose diff against its merge base touches one of the two cost map files gets the file
-checks: the files parse and the backup copy matches the root file. A pull request that leaves both untouched
-skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the
-branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always
-get the file checks and additionally may only touch those two files and may only add or update models.
+Every pull request whose diff against its merge base touches one of the two cost map files or the pins file gets the
+file checks: the files parse, the backup copy matches the root file, and every capability pinned in
+ci_cd/cost_map_pins.json (a value verified against a live provider call where the provider's model listing says
+otherwise) still holds in the tree that merging the pull request into its base branch produces, so a branch cut before
+a pin was added cannot lower it, and a pull request that deletes the pins file fails too. A pull request that leaves
+all three untouched skips them, since merging it keeps the base branch's copies and its head tree only carries
+whatever state the branch was cut from. Pull requests from the cost map sync bot (branches named
+litellm_cost_map_sync_*) always get the file checks and additionally may only touch the two cost map files and may
+only add or update models, so a sync can never lower a pinned capability or edit the pins. Price sync pull requests
+(branches named litellm-providers/price-sync*) and pull requests a bot account opened (COST_MAP_GUARD_AUTHOR_TYPE is
+Bot) get the human checks and may not touch the pins file either.
 """
 
 from __future__ import annotations
 
 import argparse
 import json
+import os
 import subprocess
 import sys
-from collections.abc import Mapping, Sequence
+from collections.abc import Iterator, Mapping, Sequence
 from dataclasses import dataclass
 from types import MappingProxyType
 from typing import Final
 
 COST_MAP_PATH: Final = "model_prices_and_context_window.json"
 BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json"
+PINS_PATH: Final = "ci_cd/cost_map_pins.json"
 GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH)
+CHECKED_PATHS: Final = (*GUARDED_PATHS, PINS_PATH)
 BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_"
+PRICE_SYNC_BRANCH_PREFIX: Final = "litellm-providers/price-sync"
+TARGET_ENV: Final = "COST_MAP_GUARD_TARGET"
+AUTHOR_TYPE_ENV: Final = "COST_MAP_GUARD_AUTHOR_TYPE"
+BOT_AUTHOR_TYPE: Final = "Bot"
+MERGE_CONFLICT_EXIT_CODE: Final = 1
 SPECIAL_ROOT_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"})
 EVALUATION_OUTPUT_PRICE_SOURCES: Final[Mapping[str, str]] = MappingProxyType({})
 
 CostMap = dict[str, object]
+_MISSING: Final = object()
 
 
 @dataclass(frozen=True, slots=True)
 class Snapshot:
     cost_map: str
     backup: str
+    pins: str
+
+
+@dataclass(frozen=True, slots=True)
+class MergeFailed:
+    stderr: str
+
+
+@dataclass(frozen=True, slots=True)
+class Pin:
+    key: str
+    field: str
+    value: object
+    checked: str
+    source: str
 
 
 def _parse_object(text: str, path: str) -> CostMap | str:
@@ -91,6 +121,48 @@ def _entries(cost_map: CostMap) -> dict[str, dict[str, object]]:
     return {key: entry for key, entry in cost_map.items() if isinstance(entry, dict)}
 
 
+def _pin(key: str, field: str, spec: object) -> Pin | str:
+    malformed: Final = f"{PINS_PATH}: {key}.{field} needs value, checked, and source"
+    if not isinstance(spec, dict) or "value" not in spec:
+        return malformed
+    checked: Final = spec.get("checked")
+    source: Final = spec.get("source")
+    if not isinstance(checked, str) or not isinstance(source, str) or not checked or not source:
+        return malformed
+    return Pin(key, field, spec["value"], checked, source)
+
+
+def _pins(parsed: CostMap) -> Iterator[Pin | str]:
+    for key, fields in parsed.items():
+        if not isinstance(fields, dict):
+            yield f"{PINS_PATH}: {key} must map field names to pins"
+            continue
+        for field, spec in fields.items():
+            yield _pin(key, field, spec)
+
+
+def _pin_violations(parsed: CostMap, entries: Mapping[str, Mapping[str, object]]) -> Iterator[str]:
+    for pin in _pins(parsed):
+        if isinstance(pin, str):
+            yield pin
+        elif pin.key not in entries:
+            yield f"{PINS_PATH} pins {pin.key}.{pin.field} but {COST_MAP_PATH} has no {pin.key} entry"
+        elif entries[pin.key].get(pin.field, _MISSING) != pin.value:
+            yield (
+                f"{COST_MAP_PATH}: {pin.key}.{pin.field} must stay {json.dumps(pin.value)} "
+                f"(verified {pin.checked}: {pin.source}); change {PINS_PATH} with new evidence first"
+            )
+
+
+def pin_failures(pins_text: str, head_map: CostMap) -> tuple[str, ...]:
+    if not pins_text:
+        return (f"{PINS_PATH} is missing; restore it, its pins were verified against live provider calls",)
+    parsed: Final = _parse_object(pins_text, PINS_PATH)
+    if isinstance(parsed, str):
+        return (parsed,)
+    return tuple(_pin_violations(parsed, _entries(head_map)))
+
+
 def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str]) -> tuple[str, ...]:
     base_map: Final = _parse_object(base.cost_map, COST_MAP_PATH)
     if isinstance(base_map, str):
@@ -120,8 +192,18 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str
     )
 
 
+def pin_author_failures(head_ref: str, author_type: str, changed_files: Sequence[str]) -> tuple[str, ...]:
+    if PINS_PATH not in changed_files:
+        return ()
+    if head_ref.startswith(PRICE_SYNC_BRANCH_PREFIX):
+        return (f"price sync PRs may not change {PINS_PATH}; a pin moves only in a human PR with new evidence",)
+    if author_type == BOT_AUTHOR_TYPE:
+        return (f"PRs opened by a bot may not change {PINS_PATH}; a pin moves only in a human PR with new evidence",)
+    return ()
+
+
 def touches_cost_map(changed_files: Sequence[str]) -> bool:
-    return any(path in GUARDED_PATHS for path in changed_files)
+    return any(path in CHECKED_PATHS for path in changed_files)
 
 
 def contract_for(bot: bool, changed_files: Sequence[str]) -> str:
@@ -130,9 +212,21 @@ def contract_for(bot: bool, changed_files: Sequence[str]) -> str:
     return "human PR, file checks only" if touches_cost_map(changed_files) else "human PR, cost map untouched"
 
 
+def _merged_pin_failures(base: Snapshot, merged: Snapshot | None) -> tuple[str, ...]:
+    if merged is None:
+        return ("the pull request does not merge cleanly into its base branch; merge the base branch in first",)
+    if not merged.pins and not base.pins:
+        return ()
+    merged_map: Final = _parse_object(merged.cost_map, COST_MAP_PATH)
+    if isinstance(merged_map, str):
+        return (f"after merging into the base branch, {merged_map}",)
+    return pin_failures(merged.pins, merged_map)
+
+
 def guard_failures(
     base: Snapshot,
     head: Snapshot,
+    merged: Snapshot | None,
     changed_files: Sequence[str],
     bot: bool,
     *,
@@ -147,6 +241,7 @@ def guard_failures(
         *_file_failures(
             head, head_map, evaluation_output_price_sources=evaluation_output_price_sources
         ),
+        *_merged_pin_failures(base, merged),
         *(_bot_failures(base, head_map, changed_files) if bot else ()),
     )
 
@@ -157,7 +252,22 @@ def _git(*args: str) -> str | None:
 
 
 def snapshot(revision: str) -> Snapshot:
-    return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in GUARDED_PATHS))
+    return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in CHECKED_PATHS))
+
+
+def merged_tree(merge_base: str, target: str, head: str) -> str | MergeFailed | None:
+    result: Final = subprocess.run(
+        ("git", "merge-tree", "--write-tree", f"--merge-base={merge_base}", target, head),
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+    tree: Final = result.stdout.split("\n", 1)[0]
+    if result.returncode == 0:
+        return tree
+    if result.returncode == MERGE_CONFLICT_EXIT_CODE and tree:
+        return None
+    return MergeFailed(result.stderr.strip())
 
 
 def changed_files(base: str, head: str) -> tuple[str, ...] | None:
@@ -169,6 +279,13 @@ def main(argv: Sequence[str]) -> int:
     parser: Final = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--base", required=True, help="merge base of the pull request")
     parser.add_argument("--head", required=True, help="head commit of the pull request")
+    env_target: Final = os.environ.get(TARGET_ENV)
+    parser.add_argument(
+        "--target",
+        default=env_target,
+        required=env_target is None,
+        help=f"tip of the base branch the pull request merges into, defaulting to ${TARGET_ENV}",
+    )
     parser.add_argument("--head-ref", required=True, help="head branch name of the pull request")
     args: Final = parser.parse_args(argv)
     bot: Final = args.head_ref.startswith(BOT_BRANCH_PREFIX)
@@ -176,7 +293,19 @@ def main(argv: Sequence[str]) -> int:
     if changed is None:
         print(f"cost map guard failed: git diff {args.base} {args.head} failed, so the changed files are unknown")
         return 1
-    failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed, bot)
+    if not bot and not touches_cost_map(changed):
+        print(f"cost map guard passed ({contract_for(bot, changed)})")
+        return 0
+    merged: Final = merged_tree(args.base, args.target, args.head)
+    if isinstance(merged, MergeFailed):
+        print(f"cost map guard failed: git merge-tree into {args.target} failed: {merged.stderr}")
+        return 1
+    failures: Final = (
+        *guard_failures(
+            snapshot(args.base), snapshot(args.head), None if merged is None else snapshot(merged), changed, bot
+        ),
+        *pin_author_failures(args.head_ref, os.environ.get(AUTHOR_TYPE_ENV, ""), changed),
+    )
     contract: Final = contract_for(bot, changed)
     if failures:
         print(f"cost map guard failed ({contract}):")

ci_cd/cost_map_pins.json

@@ -0,0 +1,16 @@
+{
+    "fireworks_ai/accounts/fireworks/models/minimax-m3": {
+        "supports_vision": {
+            "value": true,
+            "checked": "2026-09-26",
+            "source": "POST https://api.fireworks.ai/inference/v1/chat/completions read a 512x512 PNG back as digit=7, shape=circle, color=red while GET https://api.fireworks.ai/inference/v1/models still said supports_image_input false and GET https://api.fireworks.ai/v1/serverless/models?format=nested carried no supportsImageInput on the row"
+        }
+    },
+    "fireworks_ai/minimax-m3": {
+        "supports_vision": {
+            "value": true,
+            "checked": "2026-09-26",
+            "source": "POST https://api.fireworks.ai/inference/v1/chat/completions read a 512x512 PNG back as digit=7, shape=circle, color=red while GET https://api.fireworks.ai/inference/v1/models still said supports_image_input false and GET https://api.fireworks.ai/v1/serverless/models?format=nested carried no supportsImageInput on the row"
+        }
+    }
+}

docs/adrs/AGENTS.md

@@ -0,0 +1,5 @@
+# Architecture decision records for the Python SDK and proxy
+
+This folder records decisions about the Python SDK and proxy the way [`litellm-rust/docs/adrs/`](../../litellm-rust/docs/adrs/AGENTS.md) records them for the Rust core, and that file's process applies here unchanged: one short `adr_NNN_short_title.md` per decision that is hard to reverse or that a new contributor would reasonably question, with Context, Decision, Alternatives Considered, and Consequences, superseded by a new ADR instead of edited once accepted. Each folder numbers its own ADRs, so a reference to an ADR in the other folder names its path
+
+Read the ADRs here before you build a new way to move data between the endpoints, the provider transformations, and cost tracking or logging. A PR that makes a decision of that kind adds its ADR in the same PR

docs/adrs/adr_001_provider_usage_fields_ride_as_usage_extras.md

@@ -0,0 +1,39 @@
+# ADR 001: Provider usage fields ride as extras on the normalized usage object
+
+Status: Accepted
+
+## Context
+
+Providers report usage fields that no other provider has, such as xAI's `usage.server_side_tool_usage_details.web_search_calls` or Anthropic's `usage.inference_geo`, and cost tracking has to read them or that spend never reaches `LiteLLM_SpendLogs`. One provider response can be served on `/v1/chat/completions`, `/v1/responses`, and `/v1/messages`, streaming or not, so a field that only one of those paths carries is billed wrong on the others
+
+[#30817](https://github.com/BerriAI/litellm/pull/30817) fixed unbilled xAI web searches. One draft did the copy in the xAI Responses transform to keep provider code out of shared files, and that swapped `/v1/responses` usage to the chat shape, so callers of that endpoint lost the Responses schema. The merged version keeps the schema, copies unknown usage fields in the shared Responses-to-chat bridge for every provider, and gates on the reported field without naming the provider
+
+In October 2026, `Usage` and `ResponseAPIUsage` kept unknown usage keys as pydantic extras, and that bridge copied them. Paths that build usage from named fields kept only the fields they name: the Anthropic, Gemini, and Bedrock Converse transforms, the stream builder, and the bridges from chat usage to `/v1/responses` and `/v1/messages`
+
+## Decision
+
+Provider-only usage fields stay on the usage object as pydantic extras, and cost tracking reads them there, in that provider's cost calculator or in a shared gate that checks the field instead of the provider. An extra reaches callers as an added usage key. No transform renames, removes, or reshapes a usage field the endpoint's schema defines to feed cost tracking, though it may fill one LiteLLM's usage object already defines, such as `server_tool_use`
+
+A shared path that rebuilds usage, such as the stream builder or a bridge between endpoint shapes, copies the extras instead of naming provider fields. A provider transform that builds usage from named fields names the new field. Neither is bypassed with a side channel. Older code that differs, such as the Azure branches in the shared tracker and the Gemini pricing tier read from `_hidden_params`, predates this ADR
+
+## Alternatives Considered
+
+**Reshape usage in the provider transform**, as a draft of #30817 did. It changes the schema that callers of that endpoint rely on, and every provider and endpoint needs its own copy
+
+**A typed field on `ResponseAPIUsage` and `Usage` for each provider's quirk**. The types grow without bound and every bridge needs an update for each one. A field several providers report, like `server_tool_use`, does get one
+
+**A side channel such as `_hidden_params`**. Cost calculators receive `Usage`, so a side channel makes a second source of truth for the same number
+
+**Provider branches or provider field names in shared code**. They put one provider's logic in a file every provider shares, which is why #30817 replaced its xAI-only gate with one that checks the reported field
+
+## Consequences
+
+Benefits:
+
+- Billing a new provider usage field touches that provider's calculator (plus a `cost_per_token` arm when the calculator is new), its transform when that names fields, and for a built-in tool at most one gate and one `get_cost_for_web_search_request` arm
+
+Costs:
+
+- Extras are untyped, so a reader validates them before use, as `_usage_reports_server_side_web_search_calls` does
+- The stream builder and the bridges from chat usage did not copy extras when this was written, so until they do, a field billed on those paths is named in them, and tests and proof runs cover each endpoint and `"stream": true`
+- This covers usage fields LiteLLM prices, not request-side params or a provider's own charge for the request, which OpenRouter and opencode Go report and `response_cost_calculator` reads from `_hidden_params` first

enterprise/enterprise_hooks/google_text_moderation.py

@@ -15,7 +15,10 @@ import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.proxy._types import UserAPIKeyAuth
-from litellm.proxy.guardrails._content_utils import iter_message_text
+from litellm.proxy.guardrails._content_utils import (
+    is_non_conversational_request,
+    iter_message_text,
+)
 from litellm.types.utils import CallTypesLiteral
 
 _THRESHOLD_VALUE: Final = TypeAdapter(object)
@@ -94,7 +97,7 @@ class ENTERPRISE_GoogleTextModeration(CustomLogger):
 
     async def async_moderation_hook(
         self,
-        data: dict,
+        data: dict[str, object],
         user_api_key_dict: UserAPIKeyAuth,
         call_type: CallTypesLiteral,
     ):
@@ -102,6 +105,9 @@ class ENTERPRISE_GoogleTextModeration(CustomLogger):
         - Calls Google's Text Moderation API
         - Rejects request if it fails safety check
         """
+        if is_non_conversational_request(call_type, data):
+            verbose_proxy_logger.debug("Google text moderation: skipping non-conversational call type %s", call_type)
+            return
         # Covers multimodal list content + Responses-API input.
         text = "".join(iter_message_text(data))
         if text:

enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py

@@ -30,7 +30,7 @@ from collections.abc import Awaitable, Callable, Mapping
 from dataclasses import dataclass
 from datetime import datetime, timedelta, timezone
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Final, cast
+from typing import TYPE_CHECKING, Final
 
 from litellm_enterprise.proxy.common_utils.managed_object_attribution import (
     ManagedObjectCreatorAttribution,
@@ -290,12 +290,14 @@ class CheckResponsesCost:
         sees provider env vars, so it fails for every deployment whose credentials
         live in the config; the row then never leaves ``queued``.
         """
-        if not via_router:
-            return await litellm.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata)
-        router_response = await self.llm_router.aget_responses(
-            response_id=response_id, litellm_metadata=litellm_metadata
+        retrieved: Final[object] = (
+            await self.llm_router.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata)
+            if via_router
+            else await litellm.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata)
         )
-        return cast(ResponsesAPIResponse, router_response)
+        if not isinstance(retrieved, ResponsesAPIResponse):
+            raise TypeError(f"Retrieving {response_id} did not return a response object")
+        return retrieved
 
     async def _fetch_unbilled(self, target: _PollTarget) -> ResponsesAPIResponse:
         return await self._get_response(

examples/bigmodel.md

@@ -0,0 +1,63 @@
+# BigModel (Zhipu AI)
+
+Use `bigmodel/` for the domestic BigModel platform and `zai/` for international Z.AI. Each platform uses its own credentials and default URL
+
+```yaml
+model_list:
+  - model_name: glm
+    litellm_params:
+      model: bigmodel/glm-5.3
+      api_key: os.environ/BIGMODEL_API_KEY
+      api_base: https://open.bigmodel.cn
+```
+
+One model configuration supports all three native APIs. The provider selects the upstream endpoint from the requested API
+
+| Proxy endpoint | BigModel endpoint |
+| --- | --- |
+| `/v1/chat/completions` | `/api/paas/v4/chat/completions` |
+| `/v1/responses` | `/api/v1/responses` |
+| `/v1/messages` | `/api/anthropic/v1/messages` |
+
+`api_base` is optional and defaults to `https://open.bigmodel.cn`. Set `BIGMODEL_API_BASE` to override the default, or pass an explicit `api_base` to override the environment value. Existing API-specific base URLs and complete endpoint URLs are normalized, preserving a custom proxy's path prefix
+
+A GLM Coding Plan key uses `https://open.bigmodel.cn/api/coding/paas/v4` as its `api_base`, since Coding Plan and standard keys do not work on each other's endpoints. Chat Completions then calls the Coding Plan endpoint, and Responses and Messages keep the endpoints in the table above
+
+The Admin UI lists BigModel (Zhipu AI), displays its official logo, and provides a required password API Key field and an optional API Base field with the default URL. Authentication uses an explicit `api_key` or `BIGMODEL_API_KEY`
+
+```python
+import litellm
+
+chat = litellm.completion(
+    model="bigmodel/glm-5.3",
+    messages=[{"role": "user", "content": "Hello"}],
+    thinking={"type": "enabled"},
+    reasoning_effort="low",
+)
+
+response = litellm.responses(
+    model="bigmodel/glm-5.3",
+    input="Hello",
+    reasoning={"effort": "low"},
+    store=True,
+)
+
+async def messages():
+    return await litellm.anthropic.messages.acreate(
+        model="bigmodel/glm-5.3",
+        max_tokens=1024,
+        messages=[{"role": "user", "content": "Hello"}],
+        thinking={"type": "adaptive"},
+        output_config={"effort": "low"},
+    )
+```
+
+Each API retains its native response format and streaming events. The native Messages SDK entry point is asynchronous, as is the proxy's `/v1/messages` route
+
+GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX use always-on thinking. GLM-5.3 supports `low`, `high`, and `max` reasoning effort. Messages preserves native `thinking`, `output_config`, and `reasoning_effort` parameters
+
+Set `store=True` when creating a Responses request if you need continuation, retrieval, or deletion. Use the response ID returned by LiteLLM. BigModel documents a seven-day retention period and does not support cancel, compact, native WebSocket, or file_search
+
+The model catalog and packaged backup include GLM-5.3, GLM-5.3-Flash, and GLM-5.3-FlashX. Prices are expressed in USD/token using a static 7.0 CNY/USD conversion, with the source and verification date in each entry's metadata. Use existing `litellm_params` cost fields for contract-specific deployment prices
+
+API references: [Chat Completions](https://docs.bigmodel.cn/cn/guide/develop/openai/introduction), [Responses](https://docs.bigmodel.cn/cn/guide/develop/responses/introduction), [Messages](https://docs.bigmodel.cn/cn/guide/develop/claude/introduction), and [pricing](https://docs.bigmodel.cn/cn/guide/start/pricing), checked on 2026-10-27

litellm-rust/crates/inference-ocr/tests/ocr/azure_document_intelligence.rs

@@ -12,6 +12,7 @@ use rstest::rstest;
 use super::*;
 
 const MODEL: &str = "azure_ai/doc-intelligence/prebuilt-read";
+const LAYOUT_MODEL: &str = "azure_ai/doc-intelligence/prebuilt-layout";
 
 fn read_request(base: &str, options: Value) -> LiteLLMOcrRequest {
     ocr_request(MODEL, base, options)
@@ -70,6 +71,7 @@ async fn pages_features_and_extra_options_map_to_the_analyze_call() {
 #[case(json!({"pages": "1&&features=bad"}), Error::Pages(PagesFault::Range))]
 #[case(json!({"features": "languages&pages=1"}), Error::Features)]
 #[case(json!({"req_format": "azure"}), Error::RequestFormat)]
+#[case(json!({"outputContentFormat": "html"}), Error::OutputContentFormat)]
 #[tokio::test]
 async fn invalid_pages_features_and_format_are_rejected_before_sending(
     #[case] options: Value,
@@ -130,6 +132,137 @@ async fn an_inline_document_is_sent_as_base64_and_only_page_text_is_kept(#[case]
     assert_eq!(sent.json(), json!({"base64Source": "YWJj"}));
 }
 
+#[rstest]
+#[case::lower_case("markdown")]
+#[case::capitalized("Markdown")]
+#[tokio::test]
+async fn markdown_output_is_requested_on_the_analyze_call_and_split_into_pages(
+    #[case] output_content_format: &str,
+) {
+    let upstream = upstream([json_response(json!({
+        "status": "succeeded",
+        "analyzeResult": {
+            "content": "# Tïtle\n\n<!-- PageBreak -->\n\n| a |",
+            "contentFormat": "markdown",
+            "stringIndexType": "unicodeCodePoint",
+            "pages": [
+                {"pageNumber": 1, "spans": [{"offset": 0, "length": 29}], "lines": [{"content": "Tïtle"}]},
+                {"pageNumber": 2, "spans": [{"offset": 29, "length": 5}], "lines": [{"content": "a"}]}
+            ]
+        }
+    }))])
+    .await;
+
+    let response = perform(ocr_request(
+        LAYOUT_MODEL,
+        &upstream.uri(),
+        json!({"outputContentFormat": output_content_format, "pages": [0, 1]}),
+    ))
+    .await
+    .unwrap();
+
+    let sent = only_request(&upstream).await;
+    assert_eq!(
+        sent.query("outputContentFormat").as_deref(),
+        Some("markdown")
+    );
+    assert_eq!(
+        sent.query("stringIndexType").as_deref(),
+        Some("unicodeCodePoint")
+    );
+    assert_eq!(sent.query("pages").as_deref(), Some("1,2"));
+    assert_eq!(sent.json(), json!({"base64Source": "YWJj"}));
+    assert_eq!(
+        response.content.as_deref(),
+        Some("# Tïtle\n\n<!-- PageBreak -->\n\n| a |")
+    );
+    let pages = response
+        .pages
+        .iter()
+        .map(|page| page.markdown.as_str())
+        .collect::<Vec<_>>();
+    assert_eq!(pages, ["# Tïtle\n\n<!-- PageBreak -->\n\n", "| a |"]);
+    assert_eq!(
+        response.extra_fields.get("stringIndexType"),
+        Some(&json!("unicodeCodePoint"))
+    );
+    assert_eq!(
+        response.usage_info.as_ref().unwrap().pages_processed,
+        Some(2)
+    );
+}
+
+#[rstest]
+#[case::native_markdown(LAYOUT_MODEL, json!({"outputContentFormat": "markdown", "req_format": "native"}), Some("markdown"))]
+#[case::markdown_on_read(MODEL, json!({"outputContentFormat": "markdown"}), Some("markdown"))]
+#[case::text(LAYOUT_MODEL, json!({"outputContentFormat": "text"}), Some("text"))]
+#[case::omitted(LAYOUT_MODEL, json!({}), None)]
+#[tokio::test]
+async fn only_layout_markdown_in_litellm_format_asks_for_code_point_offsets(
+    #[case] model: &str,
+    #[case] options: Value,
+    #[case] output_content_format: Option<&str>,
+) {
+    let upstream = upstream([json_response(json!({
+        "status": "succeeded",
+        "analyzeResult": {
+            "contentFormat": "text",
+            "stringIndexType": "textElements",
+            "pages": [{"pageNumber": 1, "lines": [{"content": "hello"}]}]
+        }
+    }))])
+    .await;
+
+    let response = perform(ocr_request(model, &upstream.uri(), options))
+        .await
+        .unwrap();
+
+    let sent = only_request(&upstream).await;
+    assert_eq!(
+        sent.query("outputContentFormat").as_deref(),
+        output_content_format
+    );
+    assert_eq!(sent.query("stringIndexType"), None);
+    assert_eq!(sent.json(), json!({"base64Source": "YWJj"}));
+    assert_eq!(response.pages[0].markdown, "hello");
+    assert_eq!(response.extra_fields.get("stringIndexType"), None);
+}
+
+#[rstest]
+#[case::snake_case(json!({"output_content_format": "markdown"}), Some("markdown"))]
+#[case::padded(json!({"outputContentFormat": " markdown "}), Some("markdown"))]
+#[case::empty(json!({"outputContentFormat": ""}), None)]
+#[case::empty_camel_case_falls_back_to_snake_case(
+    json!({"outputContentFormat": "", "output_content_format": "markdown"}),
+    Some("markdown")
+)]
+#[case::dropped(json!({"outputContentFormat": "html", "drop_params": true}), None)]
+#[tokio::test]
+async fn the_requested_output_content_format_reaches_azure_in_its_query_spelling(
+    #[case] options: Value,
+    #[case] output_content_format: Option<&str>,
+) {
+    let upstream = upstream([json_response(json!({
+        "status": "succeeded",
+        "analyzeResult": {"pages": [{"pageNumber": 1, "lines": [{"content": "hello"}]}]}
+    }))])
+    .await;
+
+    let response = perform(ocr_request(MODEL, &upstream.uri(), options))
+        .await
+        .unwrap();
+
+    let sent = only_request(&upstream).await;
+    assert_eq!(
+        sent.query("outputContentFormat").as_deref(),
+        output_content_format
+    );
+    assert_eq!(sent.query("output_content_format"), None);
+    assert_eq!(sent.query("drop_params"), None);
+    assert_eq!(sent.json(), json!({"base64Source": "YWJj"}));
+    assert_eq!(response.pages[0].markdown, "hello");
+}
+
 #[tokio::test]
 async fn native_format_normalizes_pages_and_keeps_the_provider_response() {
     let operation = json!({

litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs

@@ -1,4 +1,7 @@
-use std::{collections::BTreeSet, time::Duration};
+use std::{
+    collections::{BTreeMap, BTreeSet},
+    time::Duration,
+};
 
 use base64::{Engine, engine::general_purpose::STANDARD};
 use litellm_auth::{InputSource, Sourced};
@@ -8,7 +11,7 @@ use litellm_llms_types::serde_compat::{FiniteF64, LaxI64};
 use reqwest::Url;
 use serde::{Deserialize, Serialize};
 use serde_json::{Map, Value};
-use serde_with::serde_as;
+use serde_with::{DefaultOnError, serde_as};
 use tokio::time::Instant;
 
 use crate::base_llm::ocr::{
@@ -17,9 +20,10 @@ use crate::base_llm::ocr::{
     handler::{CallHooks, OcrClient, read_json_response},
     settings::OcrSettings,
     transformation::{
-        BaseOcrConfig, DecodedOcrResponse, OCR_INLINE_MAX_BYTES, OCR_POLL_RETRY_SECS,
-        OcrConnection, OcrCredentialInputs, OcrResponseContext, PreparedOcrRequest,
-        ResolvedOcrCredentials, decode_and_normalize_response, decode_response,
+        BaseOcrConfig, DROP_PARAMS_PARAM, DecodedOcrResponse, OCR_INLINE_MAX_BYTES,
+        OCR_POLL_RETRY_SECS, OcrConnection, OcrCredentialInputs, OcrResponseContext,
+        PreparedOcrRequest, ResolvedOcrCredentials, decode_and_normalize_response, decode_response,
+        drops_unsupported_params, response_format,
     },
 };
 use litellm_llms_types::formats::ocr::{
@@ -32,6 +36,30 @@ const AZURE_DI_DEFAULT_HEIGHT: f64 = 11.0;
 
 const AZURE_DI_API_KEY_ENV: &str = "AZURE_DOCUMENT_INTELLIGENCE_API_KEY";
 const AZURE_DI_ENDPOINT_ENV: &str = "AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT";
+const OUTPUT_CONTENT_FORMAT_PARAM: &str = "outputContentFormat";
+const OUTPUT_CONTENT_FORMAT_ALIAS: &str = "output_content_format";
+const CODE_POINT_STRING_INDEX: &str = "unicodeCodePoint";
+const STRING_INDEX_TYPE_FIELD: &str = "stringIndexType";
+const MARKDOWN_MODEL_ID: &str = "prebuilt-layout";
+
+#[derive(
+    Clone,
+    Copy,
+    Debug,
+    PartialEq,
+    strum::EnumString,
+    strum::IntoStaticStr,
+    strum::Display,
+    strum::VariantArray,
+    serde_with::SerializeDisplay,
+)]
+#[strum(ascii_case_insensitive)]
+pub enum OutputContentFormat {
+    #[strum(serialize = "text")]
+    Text,
+    #[strum(serialize = "markdown")]
+    Markdown,
+}
 
 #[derive(Clone, Debug, PartialEq, Serialize)]
 pub struct DocumentIntelligenceParams {
@@ -39,6 +67,13 @@ pub struct DocumentIntelligenceParams {
     pub pages: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub features: Option<String>,
+    #[serde(
+        rename = "outputContentFormat",
+        skip_serializing_if = "Option::is_none"
+    )]
+    pub output_content_format: Option<OutputContentFormat>,
+    #[serde(rename = "stringIndexType", skip_serializing_if = "Option::is_none")]
+    pub string_index_type: Option<&'static str>,
 }
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
@@ -80,6 +115,10 @@ pub struct AzureDocumentIntelligenceOperation {
 #[derive(Clone, Debug, Default, Deserialize)]
 struct AzureDocumentIntelligenceAnalyzeResult {
     pub content: Option<String>,
+    #[serde(rename = "contentFormat")]
+    pub content_format: Option<String>,
+    #[serde(rename = "stringIndexType")]
+    pub string_index_type: Option<String>,
     #[serde(default)]
     pub pages: Vec<AzureDocumentIntelligencePage>,
     pub tables: Option<Vec<Map<String, Value>>>,
@@ -100,6 +139,9 @@ struct AzureDocumentIntelligencePage {
     pub unit: Option<String>,
     #[serde(default)]
     pub lines: Vec<AzureDocumentIntelligenceLine>,
+    #[serde(default)]
+    #[serde_as(deserialize_as = "DefaultOnError")]
+    pub spans: Option<Vec<AzureDocumentIntelligenceSpan>>,
 }
 
 #[derive(Clone, Debug, Deserialize)]
@@ -107,6 +149,18 @@ struct AzureDocumentIntelligenceLine {
     pub content: Option<String>,
 }
 
+#[derive(Clone, Debug, Deserialize)]
+struct AzureDocumentIntelligenceSpan {
+    pub offset: usize,
+    pub length: usize,
+}
+
+impl AzureDocumentIntelligenceSpan {
+    fn end(&self) -> usize {
+        self.offset.saturating_add(self.length)
+    }
+}
+
 #[derive(Clone, Debug)]
 pub struct AzureDocumentIntelligenceOcrConfig;
 
@@ -116,7 +170,14 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig {
     type Environment = Vec<(String, String)>;
 
     fn get_supported_ocr_params(&self, _model: &str) -> &'static [&'static str] {
-        &["pages", "features", "req_format"]
+        &[
+            "pages",
+            "features",
+            OUTPUT_CONTENT_FORMAT_PARAM,
+            OUTPUT_CONTENT_FORMAT_ALIAS,
+            "req_format",
+            DROP_PARAMS_PARAM,
+        ]
     }
 
     fn get_api_key_env_var(&self) -> Option<&'static str> {
@@ -154,11 +215,17 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig {
     fn map_ocr_params(
         &self,
         non_default_params: &CallArguments,
-        _model: &str,
+        model: &str,
     ) -> Result<DocumentIntelligenceParams, Error> {
+        let output_content_format = requested_output_content_format(non_default_params)?;
+        let slices_markdown_pages = output_content_format == Some(OutputContentFormat::Markdown)
+            && response_format(non_default_params)? != OcrResponseFormat::Native
+            && model_id(model).is_ok_and(|id| id == MARKDOWN_MODEL_ID);
         Ok(DocumentIntelligenceParams {
             pages: normalize_pages_param(non_default_params.get("pages"))?,
             features: normalize_features_param(non_default_params.get("features"))?,
+            output_content_format,
+            string_index_type: slices_markdown_pages.then_some(CODE_POINT_STRING_INDEX),
         })
     }
 
@@ -337,6 +404,36 @@ fn normalize_features_param(features: Option<&Value>) -> Result<Option<String>,
     Ok(Some(normalized.join(",")))
 }
 
+fn requested_output_content_format(
+    params: &CallArguments,
+) -> Result<Option<OutputContentFormat>, Error> {
+    let requested = [OUTPUT_CONTENT_FORMAT_PARAM, OUTPUT_CONTENT_FORMAT_ALIAS]
+        .into_iter()
+        .filter_map(|name| params.get(name))
+        .find(|format| !is_unset(format));
+    match requested.map(parse_output_content_format).transpose() {
+        Err(Error::OutputContentFormat) if drops_unsupported_params(params) => Ok(None),
+        parsed => parsed,
+    }
+}
+
+fn is_unset(format: &Value) -> bool {
+    match format {
+        Value::Null => true,
+        Value::String(format) => format.trim().is_empty(),
+        _ => false,
+    }
+}
+
+fn parse_output_content_format(format: &Value) -> Result<OutputContentFormat, Error> {
+    format
+        .as_str()
+        .ok_or(Error::OutputContentFormat)?
+        .trim()
+        .parse()
+        .map_err(|_| Error::OutputContentFormat)
+}
+
 fn build_request(document: OcrDocument) -> Result<DocumentIntelligenceRequest, Error> {
     let source = document.source();
     if source.is_empty() {
@@ -367,10 +464,27 @@ fn transform_completed_response(
         ));
     }
     let result = response.analyze_result.unwrap_or_default();
+    let markdown_content = result
+        .content
+        .as_deref()
+        .filter(|_| {
+            result.content_format.as_deref() == Some(<&str>::from(OutputContentFormat::Markdown))
+                && result.string_index_type.as_deref() == Some(CODE_POINT_STRING_INDEX)
+                && result.pages.iter().all(|page| page.spans.is_some())
+        })
+        .map(|content| {
+            CodePointText::new(
+                content,
+                result
+                    .pages
+                    .iter()
+                    .flat_map(|page| page.spans.iter().flatten()),
+            )
+        });
     let pages = result
         .pages
         .into_iter()
-        .map(|page| transform_azure_page(page, dpi))
+        .map(|page| transform_azure_page(page, markdown_content.as_ref(), dpi))
         .collect::<Result<Vec<_>, _>>()?;
     let pages_processed = i64::try_from(pages.len()).map_err(|_| Error::NumericRange("pages"))?;
     Ok(LiteLLMOcrResponse {
@@ -381,11 +495,20 @@ fn transform_completed_response(
             pages_processed: Some(pages_processed),
             ..Default::default()
         }),
+        extra_fields: result
+            .string_index_type
+            .filter(|unit| unit == CODE_POINT_STRING_INDEX)
+            .map(|unit| Map::from_iter([(STRING_INDEX_TYPE_FIELD.to_owned(), Value::String(unit))]))
+            .unwrap_or_default(),
         ..LiteLLMOcrResponse::new(model, pages)
     })
 }
 
-fn transform_azure_page(page: AzureDocumentIntelligencePage, dpi: i64) -> Result<OcrPage, Error> {
+fn transform_azure_page(
+    page: AzureDocumentIntelligencePage,
+    markdown_content: Option<&CodePointText<'_>>,
+    dpi: i64,
+) -> Result<OcrPage, Error> {
     let index = page
         .page_number
         .unwrap_or(1)
@@ -397,12 +520,20 @@ fn transform_azure_page(page: AzureDocumentIntelligencePage, dpi: i64) -> Result
         page.unit.as_deref().unwrap_or("inch"),
         dpi,
     )?;
-    let markdown = page
-        .lines
-        .iter()
-        .map(|line| line.content.as_deref().unwrap_or_default())
-        .collect::<Vec<_>>()
-        .join("\n");
+    let markdown = match markdown_content {
+        Some(content) => page
+            .spans
+            .iter()
+            .flatten()
+            .map(|span| content.slice(span))
+            .collect::<String>(),
+        None => page
+            .lines
+            .iter()
+            .map(|line| line.content.as_deref().unwrap_or_default())
+            .collect::<Vec<_>>()
+            .join("\n"),
+    };
     Ok(OcrPage {
         index,
         markdown,
@@ -411,6 +542,43 @@ fn transform_azure_page(page: AzureDocumentIntelligencePage, dpi: i64) -> Result
     })
 }
 
+struct CodePointText<'a> {
+    text: &'a str,
+    byte_offsets: BTreeMap<usize, usize>,
+}
+
+impl<'a> CodePointText<'a> {
+    fn new<'s>(
+        text: &'a str,
+        spans: impl Iterator<Item = &'s AzureDocumentIntelligenceSpan>,
+    ) -> Self {
+        let span_bounds = spans
+            .flat_map(|span| [span.offset, span.end()])
+            .collect::<BTreeSet<_>>();
+        Self {
+            text,
+            byte_offsets: text
+                .char_indices()
+                .map(|(byte_offset, _)| byte_offset)
+                .enumerate()
+                .filter(|(code_point, _)| span_bounds.contains(code_point))
+                .collect(),
+        }
+    }
+
+    fn slice(&self, span: &AzureDocumentIntelligenceSpan) -> &'a str {
+        let byte_offset = |code_point: usize| {
+            self.byte_offsets
+                .get(&code_point)
+                .copied()
+                .unwrap_or(self.text.len())
+        };
+        self.text
+            .get(byte_offset(span.offset)..byte_offset(span.end()))
+            .unwrap_or_default()
+    }
+}
+
 fn convert_dimensions(
     width: f64,
     height: f64,
@@ -578,6 +746,16 @@ impl AzureDocumentIntelligenceOcrConfig {
                                 .features
                                 .iter()
                                 .map(|features| ("features", features.as_str())),
+                        )
+                        .chain(
+                            params
+                                .output_content_format
+                                .map(|format| (OUTPUT_CONTENT_FORMAT_PARAM, format.into())),
+                        )
+                        .chain(
+                            params
+                                .string_index_type
+                                .map(|index_type| ("stringIndexType", index_type)),
                         ),
                 )
                 .into_string()
@@ -697,8 +875,12 @@ mod tests {
     }
 
     fn map(value: Value) -> Result<DocumentIntelligenceParams, Error> {
+        map_for_model(value, "azure_ai/doc-intelligence/prebuilt-layout")
+    }
+
+    fn map_for_model(value: Value, model: &str) -> Result<DocumentIntelligenceParams, Error> {
         let arguments = serde_json::from_value(value).unwrap();
-        AzureDocumentIntelligenceOcrConfig.map_ocr_params(&arguments, "model")
+        AzureDocumentIntelligenceOcrConfig.map_ocr_params(&arguments, model)
     }
 
     #[test]
@@ -828,6 +1010,218 @@ mod tests {
         assert_eq!(map(json!({"features": []})).unwrap().features, None);
     }
 
+    #[rstest]
+    #[case::markdown(
+        json!({"outputContentFormat": "markdown"}),
+        Some(OutputContentFormat::Markdown),
+        Some("unicodeCodePoint")
+    )]
+    #[case::markdown_litellm_format(
+        json!({"outputContentFormat": "markdown", "req_format": "litellm"}),
+        Some(OutputContentFormat::Markdown),
+        Some("unicodeCodePoint")
+    )]
+    #[case::markdown_native_format(
+        json!({"outputContentFormat": "markdown", "req_format": "native"}),
+        Some(OutputContentFormat::Markdown),
+        None
+    )]
+    #[case::capitalized_markdown(
+        json!({"outputContentFormat": "Markdown"}),
+        Some(OutputContentFormat::Markdown),
+        Some("unicodeCodePoint")
+    )]
+    #[case::text(json!({"outputContentFormat": "text"}), Some(OutputContentFormat::Text), None)]
+    #[case::upper_case_text(json!({"outputContentFormat": "TEXT"}), Some(OutputContentFormat::Text), None)]
+    #[case::padded_markdown(
+        json!({"outputContentFormat": " markdown\n"}),
+        Some(OutputContentFormat::Markdown),
+        Some("unicodeCodePoint")
+    )]
+    #[case::snake_case_markdown(
+        json!({"output_content_format": "markdown"}),
+        Some(OutputContentFormat::Markdown),
+        Some("unicodeCodePoint")
+    )]
+    #[case::camel_case_wins_over_snake_case(
+        json!({"outputContentFormat": "text", "output_content_format": "markdown"}),
+        Some(OutputContentFormat::Text),
+        None
+    )]
+    #[case::empty_camel_case_falls_back_to_snake_case(
+        json!({"outputContentFormat": "", "output_content_format": "markdown"}),
+        Some(OutputContentFormat::Markdown),
+        Some("unicodeCodePoint")
+    )]
+    #[case::null_camel_case_falls_back_to_snake_case(
+        json!({"outputContentFormat": null, "output_content_format": "markdown"}),
+        Some(OutputContentFormat::Markdown),
+        Some("unicodeCodePoint")
+    )]
+    #[case::empty(json!({"outputContentFormat": ""}), None, None)]
+    #[case::blank(json!({"outputContentFormat": "  "}), None, None)]
+    #[case::null(json!({"outputContentFormat": null}), None, None)]
+    #[case::omitted(json!({}), None, None)]
+    fn output_content_format_maps_to_query_fields(
+        #[case] options: Value,
+        #[case] format: Option<OutputContentFormat>,
+        #[case] string_index_type: Option<&str>,
+    ) {
+        let mapped = map(options).unwrap();
+
+        assert_eq!(mapped.output_content_format, format);
+        assert_eq!(mapped.string_index_type, string_index_type);
+    }
+
+    #[rstest]
+    #[case::layout("azure_ai/doc-intelligence/prebuilt-layout", Some("unicodeCodePoint"))]
+    #[case::bare_layout("prebuilt-layout", Some("unicodeCodePoint"))]
+    #[case::read("azure_ai/doc-intelligence/prebuilt-read", None)]
+    #[case::invoice("azure_ai/doc-intelligence/prebuilt-invoice", None)]
+    #[case::custom("azure_ai/doc-intelligence/my-extraction-model", None)]
+    fn only_the_layout_model_asks_for_code_point_offsets(
+        #[case] model: &str,
+        #[case] string_index_type: Option<&str>,
+    ) {
+        let mapped = map_for_model(json!({"outputContentFormat": "markdown"}), model).unwrap();
+
+        assert_eq!(
+            mapped.output_content_format,
+            Some(OutputContentFormat::Markdown)
+        );
+        assert_eq!(mapped.string_index_type, string_index_type);
+    }
+
+    #[rstest]
+    #[case::unknown(json!({"outputContentFormat": "html"}))]
+    #[case::number(json!({"outputContentFormat": 1}))]
+    #[case::boolean(json!({"outputContentFormat": true}))]
+    #[case::list(json!({"outputContentFormat": ["markdown"]}))]
+    #[case::snake_case_unknown(json!({"output_content_format": "html"}))]
+    #[case::drop_params_off(json!({"outputContentFormat": "html", "drop_params": false}))]
+    fn invalid_output_content_format_is_a_request_error(#[case] options: Value) {
+        let error = map(options).unwrap_err();
+
+        assert!(matches!(error, Error::OutputContentFormat), "{error:?}");
+        assert_eq!(error.http_status_code(), Some(400));
+    }
+
+    #[rstest]
+    #[case::unknown(json!({"outputContentFormat": "html", "drop_params": true, "pages": [0]}))]
+    #[case::number(json!({"outputContentFormat": 1, "drop_params": true, "pages": [0]}))]
+    #[case::snake_case_unknown(
+        json!({"output_content_format": "html", "drop_params": true, "pages": [0]})
+    )]
+    fn drop_params_drops_an_invalid_output_content_format(#[case] options: Value) {
+        let mapped = map(options).unwrap();
+
+        assert_eq!(serde_json::to_value(mapped).unwrap(), json!({"pages": "1"}));
+    }
+
+    #[rstest]
+    #[case::camel_case(json!({"outputContentFormat": "markdown", "drop_params": true}))]
+    #[case::snake_case(json!({"output_content_format": "markdown", "drop_params": true}))]
+    fn drop_params_keeps_a_valid_output_content_format(#[case] options: Value) {
+        let mapped = map(options).unwrap();
+
+        assert_eq!(
+            mapped.output_content_format,
+            Some(OutputContentFormat::Markdown)
+        );
+    }
+
+    fn markdown_pages(analyze_result: Value) -> Vec<String> {
+        let raw =
+            serde_json::to_vec(&json!({"status": "succeeded", "analyzeResult": analyze_result}))
+                .unwrap();
+        AzureDocumentIntelligenceOcrConfig
+            .transform_ocr_response("model", &raw, OcrResponseFormat::Litellm)
+            .unwrap()
+            .pages
+            .into_iter()
+            .map(|page| page.markdown)
+            .collect()
+    }
+
+    #[rstest]
+    fn markdown_pages_are_code_point_slices_of_the_content() {
+        let content = "# Ünïcode क्ष\n\n<!-- PageBreak -->\n\n## Två 🙂\n\n|a|\n\n> tail";
+        let second_page_start = "# Ünïcode क्ष\n\n<!-- PageBreak -->\n\n".chars().count();
+        let pages = markdown_pages(json!({
+            "content": content,
+            "contentFormat": "markdown",
+            "stringIndexType": "unicodeCodePoint",
+            "pages": [
+                {"pageNumber": 1, "spans": [{"offset": 0, "length": second_page_start}],
+                 "lines": [{"content": "Ünïcode क्ष"}]},
+                {"pageNumber": 2, "spans": [
+                    {"offset": second_page_start, "length": 8},
+                    {"offset": second_page_start + 10, "length": 3}
+                ], "lines": [{"content": "Två"}]}
+            ]
+        }));
+
+        assert_eq!(
+            pages,
+            ["# Ünïcode क्ष\n\n<!-- PageBreak -->\n\n", "## Två 🙂|a|"]
+        );
+    }
+
+    #[rstest]
+    fn spans_past_the_content_end_are_clamped() {
+        let pages = markdown_pages(json!({
+            "content": "ab",
+            "contentFormat": "markdown",
+            "stringIndexType": "unicodeCodePoint",
+            "pages": [{"spans": [{"offset": 1, "length": 9}, {"offset": 7, "length": 1}]}]
+        }));
+
+        assert_eq!(pages, ["b"]);
+    }
+
+    #[rstest]
+    #[case::text_content(json!("text"), json!("unicodeCodePoint"))]
+    #[case::grapheme_offsets(json!("markdown"), json!("textElements"))]
+    #[case::no_index_type(json!("markdown"), Value::Null)]
+    #[case::no_content_format(Value::Null, json!("unicodeCodePoint"))]
+    fn other_responses_keep_line_joined_pages(
+        #[case] content_format: Value,
+        #[case] string_index_type: Value,
+    ) {
+        let pages = markdown_pages(json!({
+            "content": "# A\n\nB",
+            "contentFormat": content_format,
+            "stringIndexType": string_index_type,
+            "pages": [{"spans": [{"offset": 0, "length": 6}],
+                       "lines": [{"content": "A"}, {"content": "B"}]}]
+        }));
+
+        assert_eq!(pages, ["A\nB"]);
+    }
+
+    #[rstest]
+    #[case::null_spans(json!("markdown"), json!(null))]
+    #[case::fractional_offset(json!("markdown"), json!([{"offset": 0.5, "length": 1}]))]
+    #[case::missing_length(json!("markdown"), json!([{"offset": 0}]))]
+    #[case::negative_offset(json!("markdown"), json!([{"offset": -1, "length": 1}]))]
+    #[case::text_content(json!("text"), json!([{"offset": "0", "length": 1}]))]
+    fn unusable_page_spans_keep_line_joined_pages(
+        #[case] content_format: Value,
+        #[case] broken_spans: Value,
+    ) {
+        let pages = markdown_pages(json!({
+            "content": "# A\n\nB",
+            "contentFormat": content_format,
+            "stringIndexType": "unicodeCodePoint",
+            "pages": [
+                {"spans": [{"offset": 0, "length": 5}], "lines": [{"content": "A"}]},
+                {"spans": broken_spans, "lines": [{"content": "B"}]}
+            ]
+        }));
+
+        assert_eq!(pages, ["A", "B"]);
+    }
+
     #[tokio::test]
     async fn request_endpoint_cannot_receive_environment_key() {
         let connection = OcrConnection {

litellm-rust/crates/llms/src/base_llm/ocr/error.rs

@@ -59,6 +59,10 @@ pub enum Error {
     Pages(PagesFault),
     #[error("invalid OCR features")]
     Features,
+    #[error(
+        "invalid OCR outputContentFormat, expected text or markdown. To drop unsupported params, set `litellm.drop_params = True`."
+    )]
+    OutputContentFormat,
     #[error("OCR model cannot be a dot segment")]
     DotModel,
     #[error("OCR response exceeds the size limit of {limit} bytes")]
@@ -200,6 +204,7 @@ impl Error {
             | Self::InlineDocumentTooLarge
             | Self::Pages(_)
             | Self::Features
+            | Self::OutputContentFormat
             | Self::DotModel
             | Self::TooLarge { .. }
             | Self::ResponseField { .. }
@@ -242,6 +247,7 @@ impl Error {
             | Self::InlineDocumentTooLarge
             | Self::Pages(_)
             | Self::Features
+            | Self::OutputContentFormat
             | Self::DotModel
             | Self::TooLarge { .. }
             | Self::ResponseField { .. }
@@ -291,6 +297,7 @@ impl Error {
             | Self::InlineDocumentTooLarge
             | Self::Pages(_)
             | Self::Features
+            | Self::OutputContentFormat
             | Self::DotModel
             | Self::InvalidProvider(_)
             | Self::InvalidModel { .. }
@@ -345,6 +352,7 @@ impl Error {
             | Self::InlineDocumentTooLarge
             | Self::Pages(_)
             | Self::Features
+            | Self::OutputContentFormat
             | Self::DotModel
             | Self::InvalidProvider(_)
             | Self::InvalidModel { .. }

litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs

@@ -21,6 +21,7 @@ pub const OCR_RESPONSE_MAX_BYTES: usize = 64 * 1024 * 1024;
 pub const OCR_INLINE_MAX_BYTES: usize = 50 * 1024 * 1024;
 pub const OCR_MAX_FETCH_REDIRECTS: usize = 10;
 pub const OCR_POLL_RETRY_SECS: u64 = 2;
+pub const DROP_PARAMS_PARAM: &str = "drop_params";
 
 #[derive(Clone, Default)]
 pub struct OcrCredentialInputs {
@@ -185,6 +186,10 @@ pub fn response_format(optional_params: &CallArguments) -> Result<OcrResponseFor
         .map(|format| format.unwrap_or_default())
 }
 
+pub fn drops_unsupported_params(optional_params: &CallArguments) -> bool {
+    optional_params.get(DROP_PARAMS_PARAM) == Some(&Value::Bool(true))
+}
+
 #[derive(Debug)]
 pub struct DecodedOcrResponse<T> {
     pub data: T,

litellm-rust/crates/python-bridge/src/routes/ocr/host.rs

@@ -13,7 +13,7 @@ use pyo3::{
 
 use super::{
     errors::route_failure,
-    project::{OcrHostHandles, SETTINGS_ERROR_MARKER, project_request},
+    project::{OcrHostHandles, ROUTE_HOST_MODULE, SETTINGS_ERROR_MARKER, project_request},
 };
 use crate::{marshal::public_response, routes::RequestView};
 
@@ -77,7 +77,7 @@ impl OcrPythonHost {
             _ => "",
         };
         let mapped = py
-            .import("litellm.rust_bridge.host.routes.ocr")
+            .import(ROUTE_HOST_MODULE)
             .and_then(|module| module.getattr("map_failure"))
             .and_then(|map| map.call1((error.value(py), self.view.request(py)?, provider)))
             .and_then(|mapped| mapped.extract::<Py<PyBaseException>>().map_err(PyErr::from));
@@ -115,7 +115,7 @@ impl PythonBinding for OcrPythonHost {
         py: Python<'_>,
         response: LiteLLMOcrResponse,
     ) -> PyResult<Py<PyAny>> {
-        public_response(py, "litellm.rust_bridge.host.routes.ocr", &response)
+        public_response(py, ROUTE_HOST_MODULE, &response)
     }
 
     fn encode_stream_head(

litellm-rust/crates/python-bridge/src/routes/ocr/project.rs

@@ -5,7 +5,7 @@ use litellm_inference_ocr::{
     types::{LiteLLMOcrRequest, OcrDocumentInput},
     wire::{OcrWireRequest, consumed_optional_params, decode_document, decode_request_input},
 };
-use litellm_llms::base_llm::ocr::error::Error;
+use litellm_llms::base_llm::ocr::{error::Error, transformation::DROP_PARAMS_PARAM};
 use pyo3::{prelude::*, types::PyDict};
 use serde_json::{Map, Value};
 
@@ -19,6 +19,8 @@ use crate::{
     python_settings::{PythonSettings, Snapshot},
 };
 
+pub(super) const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.host.routes.ocr";
+
 /// What the host keeps after projection: the caller's token callable that answers the
 /// token operation, and the provider name the failure mapping reports.
 pub(super) struct OcrHostHandles {
@@ -139,6 +141,15 @@ pub(super) fn project_request(
         Some(defaults) => fold_module_globals(bound.py(), optional_params, &globals, &defaults)?,
         None => optional_params,
     };
+    let optional_params = if names.contains(&DROP_PARAMS_PARAM) {
+        fold_drop_params(
+            bound.py(),
+            optional_params,
+            present(kwargs, bound, DROP_PARAMS_PARAM)?,
+        )?
+    } else {
+        optional_params
+    };
     let input_sources = request_input_sources(
         kwargs,
         names
@@ -216,6 +227,23 @@ fn fold_module_globals(
         .collect())
 }
 
+fn fold_drop_params(
+    py: Python<'_>,
+    optional_params: Map<String, Value>,
+    requested: Option<Bound<'_, PyAny>>,
+) -> PyResult<Map<String, Value>> {
+    let drops: bool = py
+        .import(ROUTE_HOST_MODULE)?
+        .getattr("drops_unsupported_params")?
+        .call1((requested,))?
+        .extract()?;
+    Ok(optional_params
+        .into_iter()
+        .filter(|(name, _)| name != DROP_PARAMS_PARAM)
+        .chain(drops.then(|| (DROP_PARAMS_PARAM.to_string(), Value::Bool(true))))
+        .collect())
+}
+
 /// A misconfigured litellm setting is the operator's error, not the request's or the
 /// provider's, so the host raises it as is instead of mapping it onto a provider failure.
 pub(super) const SETTINGS_ERROR_MARKER: &str = "native_settings_error";

litellm/__init__.py

@@ -591,7 +591,7 @@ vertex_ai_safety_settings: Optional[dict] = None
 ####### COMPLETION MODELS ###################
 from typing import Set
 
-open_ai_chat_completion_models: Set = set()
+open_ai_chat_completion_models: Set[str] = set()
 open_ai_text_completion_models: Set = set()
 cohere_models: Set = set()
 cohere_chat_models: Set = set()
@@ -619,6 +619,8 @@ vertex_openai_models: Set = set()
 vertex_minimax_models: Set = set()
 vertex_moonshot_models: Set = set()
 vertex_zai_models: Set = set()
+vertex_qwen_models: Final[Set[str]] = set()
+vertex_ai_models: Final[Set[str]] = set()
 ai21_models: Set = set()
 ai21_chat_models: Set = set()
 nlp_cloud_models: Set = set()
@@ -635,6 +637,7 @@ watsonx_models: Set = set()
 gemini_models: Set = set()
 xai_models: Set = set()
 zai_models: Set = set()
+bigmodel_models: Final[set[str]] = set()  # mutable-ok: provider registry is populated and extended at runtime
 deepseek_models: Set = set()
 tencent_models: Set = set()
 runwayml_models: Set = set()
@@ -681,6 +684,7 @@ moonshot_models: Set = set()
 publicai_models: Set = set()
 darkbloom_models: Set = set()
 openinfer_models: Final[Set[str]] = set()
+streamlake_models: Final[Set[str]] = set()
 zerogpu_models: Final[Set[str]] = set()
 v0_models: Set = set()
 morph_models: Set = set()
@@ -832,6 +836,10 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
         elif value.get("litellm_provider") == "vertex_ai-zai_models":
             key = key.replace("vertex_ai/", "")
             vertex_zai_models.add(key)
+        elif value.get("litellm_provider") == "vertex_ai-qwen_models":
+            vertex_qwen_models.add(key.removeprefix("vertex_ai/"))
+        elif value.get("litellm_provider") == "vertex_ai":
+            vertex_ai_models.add(key.removeprefix("vertex_ai/"))
         elif value.get("litellm_provider") == "ai21":
             if value.get("mode") == "chat":
                 ai21_chat_models.add(key)
@@ -876,6 +884,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
             xai_models.add(key)
         elif value.get("litellm_provider") == "zai":
             zai_models.add(key)
+        elif value.get("litellm_provider") == "bigmodel":
+            bigmodel_models.add(key)
         elif value.get("litellm_provider") == "fal_ai":
             fal_ai_models.add(key)
         elif value.get("litellm_provider") == "deepseek":
@@ -970,6 +980,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
             darkbloom_models.add(key)
         elif value.get("litellm_provider") == "openinfer":
             openinfer_models.add(key)
+        elif value.get("litellm_provider") == "streamlake":
+            streamlake_models.add(key)
         elif value.get("litellm_provider") == "zerogpu":
             zerogpu_models.add(key)
         elif value.get("litellm_provider") == "v0":
@@ -1104,6 +1116,7 @@ model_list = list(
     | text_completion_inception_models
     | xai_models
     | zai_models
+    | bigmodel_models
     | fal_ai_models
     | deepseek_models
     | modelscope_models
@@ -1145,6 +1158,7 @@ model_list = list(
     | publicai_models
     | darkbloom_models
     | openinfer_models
+    | streamlake_models
     | zerogpu_models
     | v0_models
     | morph_models
@@ -1200,7 +1214,16 @@ def _build_models_by_provider() -> dict:
         | vertex_deepseek_models
         | vertex_minimax_models
         | vertex_moonshot_models
-        | vertex_zai_models,
+        | vertex_zai_models
+        | vertex_qwen_models
+        | vertex_mistral_models
+        | vertex_llama3_models
+        | vertex_ai_ai21_models
+        | vertex_openai_models
+        | vertex_ai_image_models
+        | vertex_ai_video_models
+        | vertex_embedding_models
+        | vertex_ai_models,
         "ai21": ai21_models | ai21_chat_models,
         "bedrock": bedrock_models | bedrock_converse_models,
         "petals": petals_models,
@@ -1217,6 +1240,7 @@ def _build_models_by_provider() -> dict:
         "text-completion-inception": text_completion_inception_models,
         "xai": xai_models,
         "zai": zai_models,
+        "bigmodel": bigmodel_models,
         "fal_ai": fal_ai_models,
         "deepseek": deepseek_models,
         "tencent": tencent_models,
@@ -1265,6 +1289,7 @@ def _build_models_by_provider() -> dict:
         "publicai": publicai_models,
         "darkbloom": darkbloom_models,
         "openinfer": openinfer_models,
+        "streamlake": streamlake_models,
         "zerogpu": zerogpu_models,
         "v0": v0_models,
         "morph": morph_models,

litellm/completion_extras/litellm_responses_transformation/handler.py

@@ -260,6 +260,7 @@ class ResponsesToCompletionBridgeHandler:
                 structured_output=is_structured_output_request(
                     cast("Mapping[str, object]", optional_params)  # cast-ok: optional_params is an untyped dict
                 ),
+                replays_openai_history=custom_llm_provider == "openai",
             )
             streamwrapper: Final = CustomStreamWrapper(
                 completion_stream=completion_stream,
@@ -363,6 +364,7 @@ class ResponsesToCompletionBridgeHandler:
                 structured_output=is_structured_output_request(
                     cast("Mapping[str, object]", optional_params)  # cast-ok: optional_params is an untyped dict
                 ),
+                replays_openai_history=custom_llm_provider == "openai",
             )
             streamwrapper: Final = CustomStreamWrapper(
                 completion_stream=completion_stream,

litellm/completion_extras/litellm_responses_transformation/history.py

@@ -0,0 +1,432 @@
+import hashlib
+import json
+from collections.abc import Iterator, Mapping, Sequence
+from dataclasses import dataclass
+from itertools import accumulate, chain, pairwise
+from types import MappingProxyType
+from typing import Annotated, Final, Literal, TypeAlias
+
+from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
+
+REPLAY_FIELD: Final = "openai_responses_replay"
+PHASE_FIELD: Final = "phase"
+
+Phase: TypeAlias = Literal["commentary", "final_answer"]
+
+
+class _Frozen(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+
+class _OutputText(_Frozen):
+    type: Literal["output_text"]
+    text: str
+
+
+class _OutputMessage(_Frozen):
+    type: Literal["message"]
+    role: Literal["assistant"]
+    content: tuple[_OutputText, ...]
+    phase: Phase | None = None
+
+
+class _OutputReasoning(_Frozen):
+    type: Literal["reasoning"]
+    id: str
+
+
+class _OutputToolCall(_Frozen):
+    type: Literal["function_call", "custom_tool_call"]
+    call_id: str
+    id: str | None = None
+
+
+class _OutputToolSearch(_Frozen):
+    type: Literal["tool_search_call", "tool_search_output"]
+
+
+_OutputItem: TypeAlias = Annotated[
+    _OutputMessage | _OutputReasoning | _OutputToolCall | _OutputToolSearch, Field(discriminator="type")
+]
+_OUTPUT_ITEM: Final[TypeAdapter[_OutputItem]] = TypeAdapter(_OutputItem)
+
+
+class _AnyMessage(_Frozen):
+    type: Literal["message"]
+
+
+class _MessageEntry(_Frozen):
+    type: Literal["message"]
+    parts: tuple[Annotated[int, Field(ge=0)], ...]
+    phase: Phase | None = None
+
+
+class _IndexedEntry(_Frozen):
+    type: Literal["reasoning", "function_call", "custom_tool_call", "tool_search_call", "tool_search_output"]
+    index: Annotated[int, Field(ge=0)]
+
+
+_Entry: TypeAlias = Annotated[_MessageEntry | _IndexedEntry, Field(discriminator="type")]
+
+
+class _Replay(_Frozen):
+    version: Literal[1]
+    items: tuple[_Entry, ...]
+    checksum: str
+
+
+class _Namespaced(_Frozen):
+    namespace: str | None = None
+
+
+class _ToolFunction(_Frozen):
+    name: str = ""
+    arguments: str = ""
+    provider_specific_fields: _Namespaced | None = None
+
+
+class _ToolCall(_Frozen):
+    id: str
+    function: _ToolFunction
+    provider_specific_fields: _Namespaced | None = None
+
+
+class _SummaryText(_Frozen):
+    type: str = "summary_text"
+    text: str = ""
+
+
+class _Reasoning(_Frozen):
+    id: str | None = None
+    encrypted_content: str | None = None
+    summary: tuple[_SummaryText, ...] | None = None
+
+
+class _ProviderFields(_Frozen):
+    phase: Phase | None = None
+    tool_search_items: tuple[Mapping[str, object], ...] | None = None
+    openai_responses_replay: object = None
+
+
+class _ChatMessage(_Frozen):
+    role: str | None = None
+    content: str | None = None
+    tool_calls: tuple[_ToolCall, ...] | None = None
+    reasoning_items: tuple[_Reasoning, ...] | None = None
+    phase: Phase | None = None
+    provider_specific_fields: _ProviderFields | None = None
+
+
+class _PhaseFields(_Frozen):
+    phase: Phase | None = None
+
+
+class _RequestedPhaseFields(_Frozen):
+    phase: object = None
+
+
+class _RequestedPhase(_Frozen):
+    phase: object = None
+    provider_specific_fields: _RequestedPhaseFields | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class UnknownPhase:
+    value: object
+
+
+_PHASE: Final[TypeAdapter[Phase]] = TypeAdapter(Phase)
+
+
+class _RetainedFields(_PhaseFields):
+    openai_responses_replay: object = None
+
+
+@dataclass(frozen=True, slots=True)
+class ReplayedTurn:
+    items: tuple[Mapping[str, object], ...]
+    custom_call_ids: frozenset[str]
+
+
+def _as_json(value: object) -> object:
+    return value.model_dump() if isinstance(value, BaseModel) else value
+
+
+def _parsed_output_item(item: object) -> _OutputItem | None:
+    try:
+        return _OUTPUT_ITEM.validate_python(_as_json(item))
+    except ValidationError:
+        return None
+
+
+def _is_message(item: object) -> bool:
+    try:
+        _AnyMessage.model_validate(_as_json(item))
+    except ValidationError:
+        return False
+    return True
+
+
+def _chat_message(message: object) -> _ChatMessage | None:
+    try:
+        return _ChatMessage.model_validate(_as_json(message))
+    except ValidationError:
+        return None
+
+
+def requested_phase(message: object) -> Phase | UnknownPhase | None:
+    try:
+        requested: Final = _RequestedPhase.model_validate(_as_json(message))
+    except ValidationError:
+        return None
+    fields: Final = requested.provider_specific_fields
+    value: Final = requested.phase or (fields.phase if fields else None)
+    if value is None:
+        return None
+    try:
+        return _PHASE.validate_python(value)
+    except ValidationError:
+        return UnknownPhase(value)
+
+
+def reasoning_input_item(
+    item_id: str | None, summary: Sequence[object] | None, encrypted_content: str | None
+) -> Mapping[str, object]:
+    return {
+        "type": "reasoning",
+        **({"id": item_id} if item_id else {}),
+        "summary": list(summary or ()),
+        **({"encrypted_content": encrypted_content} if encrypted_content else {}),
+    }
+
+
+def retained_replay_fields(provider_specific_fields: object) -> Mapping[str, object] | None:
+    try:
+        retained: Final = _RetainedFields.model_validate(provider_specific_fields)
+    except ValidationError:
+        return None
+    return retained.model_dump(exclude_none=True) or None
+
+
+def output_message_text(output_items: Sequence[object], *, first_message_only: bool) -> str:
+    parsed: Final = map(_parsed_output_item, output_items)
+    messages: Final = tuple(item for item in parsed if isinstance(item, _OutputMessage))
+    return "".join(_texts(messages[:1] if first_message_only else messages))
+
+
+def _tool_search_entries(message: _ChatMessage) -> tuple[Mapping[str, object], ...]:
+    fields: Final = message.provider_specific_fields
+    items: Final = (fields.tool_search_items if fields else None) or ()
+    return tuple(item for item in items if item.get("type") in ("tool_search_call", "tool_search_output"))
+
+
+def _checksum(message: _ChatMessage, entries: Sequence[_MessageEntry | _IndexedEntry]) -> str:
+    fields: Final = message.provider_specific_fields
+    bound: Final = {
+        "content": message.content or "",
+        "tool_calls": [[call.id, call.function.name, call.function.arguments] for call in message.tool_calls or ()],
+        "reasoning": [
+            [item.id, item.encrypted_content, [part.text for part in item.summary or ()]]
+            for item in message.reasoning_items or ()
+        ],
+        "tool_search_items": list((fields.tool_search_items if fields else None) or ()),
+        "phase": message.phase or (fields.phase if fields else None),
+        "items": [entry.model_dump(mode="json", exclude_none=True) for entry in entries],
+    }
+    encoded: Final = json.dumps(bound, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
+    return hashlib.sha256(encoded.encode("utf-8", "surrogatepass")).hexdigest()
+
+
+def _kind(item: _OutputItem) -> str:
+    match item:
+        case _OutputToolCall():
+            return "tool_call"
+        case _OutputToolSearch():
+            return "tool_search"
+        case _OutputMessage() | _OutputReasoning():
+            return item.type
+
+
+def _entry(items: Sequence[_OutputItem], position: int) -> _MessageEntry | _IndexedEntry:
+    item: Final = items[position]
+    if isinstance(item, _OutputMessage):
+        return _MessageEntry(type="message", parts=tuple(len(part.text) for part in item.content), phase=item.phase)
+    ordinal: Final = sum(1 for earlier in items[:position] if _kind(earlier) == _kind(item))
+    return _IndexedEntry(type=item.type, index=ordinal)
+
+
+def _texts(items: Sequence[_OutputItem]) -> Iterator[str]:
+    for item in items:
+        if isinstance(item, _OutputMessage):
+            yield from (part.text for part in item.content)
+
+
+def _matches(items: Sequence[_OutputItem], message: _ChatMessage) -> bool:
+    reasoning_ids: Final = tuple(item.id for item in items if isinstance(item, _OutputReasoning))
+    calls: Final = tuple(item for item in items if isinstance(item, _OutputToolCall))
+    tool_calls: Final = message.tool_calls or ()
+    searches: Final = sum(1 for item in items if isinstance(item, _OutputToolSearch))
+    return (
+        "".join(_texts(items)) == (message.content or "")
+        and reasoning_ids == tuple(item.id for item in message.reasoning_items or ())
+        and len(calls) == len(tool_calls)
+        and all(chat.id in (call.call_id, call.id) for call, chat in zip(calls, tool_calls))
+        and searches == len(_tool_search_entries(message))
+    )
+
+
+def _content_phase(
+    output_items: Sequence[object], parsed: Sequence[_OutputItem | None], *, first_message_only: bool
+) -> Phase | None:
+    messages: Final = tuple(item for raw, item in zip(output_items, parsed) if _is_message(raw))
+    if not messages:
+        return None
+    content_message: Final = messages[0] if first_message_only else messages[-1]
+    return content_message.phase if isinstance(content_message, _OutputMessage) else None
+
+
+def _with_phase(message: _ChatMessage, phase: Phase | None) -> _ChatMessage:
+    if phase is None:
+        return message
+    fields: Final = (message.provider_specific_fields or _ProviderFields()).model_copy(update={PHASE_FIELD: phase})
+    return message.model_copy(update={"provider_specific_fields": fields})
+
+
+def replay_fields(
+    output_items: Sequence[object], message: object, *, completed: bool, first_message_only: bool
+) -> Mapping[str, object]:
+    parsed: Final = tuple(map(_parsed_output_item, output_items))
+    phase: Final = _content_phase(output_items, parsed, first_message_only=first_message_only)
+    phase_fields: Final[Mapping[str, object]] = MappingProxyType({PHASE_FIELD: phase} if phase else {})
+    items: Final = tuple(item for item in parsed if item is not None)
+    parsed_message: Final = _chat_message(message)
+    chat: Final = _with_phase(parsed_message, phase) if parsed_message is not None else None
+    if not completed or not items or len(items) != len(parsed) or chat is None or not _matches(items, chat):
+        return phase_fields
+    entries: Final = tuple(_entry(items, position) for position in range(len(items)))
+    replay: Final = _Replay(version=1, items=entries, checksum=_checksum(chat, entries))
+    return MappingProxyType({**phase_fields, REPLAY_FIELD: replay.model_dump(mode="json", exclude_none=True)})
+
+
+def _stored_replay(message: _ChatMessage) -> _Replay | None:
+    fields: Final = message.provider_specific_fields
+    if fields is None or fields.openai_responses_replay is None:
+        return None
+    try:
+        return _Replay.model_validate(fields.openai_responses_replay)
+    except ValidationError:
+        return None
+
+
+def _message_texts(entries: Sequence[_MessageEntry], content: str) -> tuple[tuple[str, ...], ...]:
+    lengths: Final = tuple(chain.from_iterable(entry.parts for entry in entries))
+    bounds: Final = tuple(accumulate(lengths, initial=0))
+    parts: Final = tuple(content[start:end] for start, end in pairwise(bounds))
+    groups: Final = tuple(accumulate((len(entry.parts) for entry in entries), initial=0))
+    return tuple(parts[start:end] for start, end in pairwise(groups))
+
+
+def _message_item(texts: Sequence[str], phase: Phase | None) -> Mapping[str, object]:
+    return {
+        "type": "message",
+        "role": "assistant",
+        "content": [{"type": "output_text", "text": text} for text in texts],
+        **({PHASE_FIELD: phase} if phase else {}),
+    }
+
+
+def _function_call_item(call: _ToolCall) -> Mapping[str, object]:
+    namespaced: Final = call.provider_specific_fields or call.function.provider_specific_fields
+    namespace: Final = namespaced.namespace if namespaced else None
+    return {
+        "type": "function_call",
+        "call_id": call.id,
+        "name": call.function.name,
+        "arguments": call.function.arguments,
+        **({"namespace": namespace} if namespace else {}),
+    }
+
+
+def _custom_tool_call_item(call: _ToolCall) -> Mapping[str, object]:
+    return {
+        "type": "custom_tool_call",
+        "call_id": call.id,
+        "name": call.function.name,
+        "input": call.function.arguments,
+    }
+
+
+def _reasoning_item(item: _Reasoning) -> Mapping[str, object]:
+    summary: Final = tuple(part.model_dump() for part in item.summary or ())
+    return reasoning_input_item(item.id, summary, item.encrypted_content)
+
+
+@dataclass(frozen=True, slots=True)
+class _Resolved:
+    message: _ChatMessage
+    texts: Mapping[int, tuple[str, ...]]
+    tool_searches: tuple[Mapping[str, object], ...]
+
+
+def _input_item(resolved: _Resolved, position: int, entry: _MessageEntry | _IndexedEntry) -> Mapping[str, object]:
+    if isinstance(entry, _MessageEntry):
+        return _message_item(resolved.texts[position], entry.phase)
+    tool_calls: Final = resolved.message.tool_calls or ()
+    match entry.type:
+        case "reasoning":
+            return _reasoning_item((resolved.message.reasoning_items or ())[entry.index])
+        case "function_call":
+            return _function_call_item(tool_calls[entry.index])
+        case "custom_tool_call":
+            return _custom_tool_call_item(tool_calls[entry.index])
+        case "tool_search_call" | "tool_search_output":
+            return dict(resolved.tool_searches[entry.index])
+
+
+def _fits(message: _ChatMessage, entries: Sequence[_MessageEntry | _IndexedEntry]) -> bool:
+    tool_calls: Final = len(message.tool_calls or ())
+    tool_searches: Final = len(_tool_search_entries(message))
+    sizes: Final = MappingProxyType(
+        {
+            "reasoning": len(message.reasoning_items or ()),
+            "function_call": tool_calls,
+            "custom_tool_call": tool_calls,
+            "tool_search_call": tool_searches,
+            "tool_search_output": tool_searches,
+        }
+    )
+    parts: Final = chain.from_iterable(entry.parts for entry in entries if isinstance(entry, _MessageEntry))
+    indexed: Final = (entry for entry in entries if isinstance(entry, _IndexedEntry))
+    return sum(parts) == len(message.content or "") and all(entry.index < sizes[entry.type] for entry in indexed)
+
+
+def _resolve(
+    message: _ChatMessage, entries: Sequence[_MessageEntry | _IndexedEntry], replay_tool_search: bool
+) -> ReplayedTurn | None:
+    indexed: Final = tuple(entry for entry in entries if isinstance(entry, _IndexedEntry))
+    if not replay_tool_search and any(entry.type in ("tool_search_call", "tool_search_output") for entry in indexed):
+        return None
+    messages: Final = tuple(entry for entry in entries if isinstance(entry, _MessageEntry))
+    calls: Final = tuple(entry for entry in indexed if entry.type in ("function_call", "custom_tool_call"))
+    texts: Final = _message_texts(messages, message.content or "")
+    positions: Final = tuple(position for position, entry in enumerate(entries) if isinstance(entry, _MessageEntry))
+    resolved: Final = _Resolved(
+        message=message,
+        texts=MappingProxyType(dict(zip(positions, texts, strict=True))),
+        tool_searches=_tool_search_entries(message),
+    )
+    tool_calls: Final = message.tool_calls or ()
+    return ReplayedTurn(
+        items=tuple(_input_item(resolved, position, entry) for position, entry in enumerate(entries)),
+        custom_call_ids=frozenset(tool_calls[entry.index].id for entry in calls if entry.type == "custom_tool_call"),
+    )
+
+
+def replay_items(message: object, *, replay_tool_search: bool) -> ReplayedTurn | None:
+    chat: Final = _chat_message(message)
+    stored: Final = _stored_replay(chat) if chat is not None and chat.role == "assistant" else None
+    if chat is None or stored is None or stored.checksum != _checksum(chat, stored.items):
+        return None
+    if not _fits(chat, stored.items):
+        return None
+    return _resolve(chat, stored.items, replay_tool_search)

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -24,6 +24,15 @@ from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
 import litellm
 from litellm import ModelResponse
 from litellm._logging import verbose_logger
+from litellm.completion_extras.litellm_responses_transformation.history import (
+    PHASE_FIELD,
+    UnknownPhase,
+    output_message_text,
+    reasoning_input_item,
+    replay_fields,
+    replay_items,
+    requested_phase,
+)
 from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint
 from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
@@ -490,7 +499,7 @@ def _fold_choices_into_one(
     choices: Sequence["Choices"],
     tool_calls: Sequence[Mapping[str, object]],
     trailing_reasoning_content: str | None,
-    trailing_reasoning_item: _BuiltReasoningItem | None,
+    trailing_reasoning_items: Sequence[_BuiltReasoningItem],
     tool_search_items: Sequence[Mapping[str, object]],
     structured_output: bool = False,
 ) -> "Choices":
@@ -509,7 +518,7 @@ def _fold_choices_into_one(
     reasoning_contents: Final = (*(_message_reasoning_content(m) for m in messages), trailing_reasoning_content)
     reasoning_items: Final = (
         *chain.from_iterable(_message_reasoning_items(m) for m in messages),
-        *(() if trailing_reasoning_item is None else (trailing_reasoning_item,)),
+        *trailing_reasoning_items,
     )
     return Choices(
         message=Message(
@@ -526,6 +535,61 @@ def _fold_choices_into_one(
     )
 
 
+def _with_replay_fields(
+    choice: "Choices", output_items: Sequence[object], *, completed: bool, structured_output: bool
+) -> "Choices":
+    fields: Final = {
+        **(choice.message.provider_specific_fields or {}),
+        **replay_fields(output_items, choice.message, completed=completed, first_message_only=structured_output),
+    }
+    message: Final = choice.message.model_copy(update={"provider_specific_fields": fields or None})
+    return choice.model_copy(update={"message": message})
+
+
+_STREAMED_OUTPUT_ITEMS: Final[TypeAdapter[tuple[Mapping[str, object], ...]]] = TypeAdapter(
+    tuple[Mapping[str, object], ...]
+)
+
+
+def completed_stream_history(
+    output_payload: object,
+    tool_search_fields: Mapping[str, object] | None,
+    *,
+    completed: bool,
+    structured_output: bool,
+) -> dict[str, object] | None:  # mutable-ok: Delta.provider_specific_fields is typed as a dict
+    items: Final = _STREAMED_OUTPUT_ITEMS.validate_python(output_payload)
+    calls: Final = tuple(item for item in items if item.get("type") in ("function_call", "custom_tool_call"))
+    streamed_message: Final = {
+        "role": "assistant",
+        "content": output_message_text(items, first_message_only=structured_output),
+        "tool_calls": [tool_call_dict_from_output_item(call, index) for index, call in enumerate(calls)],
+        "reasoning_items": list(_reasoning_items_from_output_items(items)),
+        "provider_specific_fields": tool_search_fields,
+    }
+    fields: Final = {
+        **(tool_search_fields or {}),
+        **replay_fields(items, streamed_message, completed=completed, first_message_only=structured_output),
+    }
+    return fields or None
+
+
+def _assistant_phase_field(msg: object, replay_openai_history: bool, drop_params: bool) -> Mapping[str, object]:
+    match requested_phase(msg) if replay_openai_history else None:
+        case None:
+            return {}
+        case UnknownPhase(value=value) if drop_params:
+            verbose_logger.debug("Chat provider: dropping assistant phase=%r under drop_params", value)
+            return {}
+        case UnknownPhase(value=value):
+            raise litellm.UnsupportedParamsError(
+                message=f"phase must be 'commentary' or 'final_answer', got {value!r}. "
+                "To drop it, set `litellm.drop_params=True` or pass `drop_params=True`."
+            )
+        case phase:
+            return {PHASE_FIELD: phase}
+
+
 def _responses_event_type(chunk: Mapping[str, object]) -> object:
     event_type: Final = chunk.get("type")
     return event_type.value if isinstance(event_type, ResponsesAPIStreamEvents) else event_type
@@ -667,15 +731,7 @@ def _reasoning_item_to_response_input(
     An item without an id is sent without one: the Responses API accepts that and
     verifies the encrypted content on its own, while it rejects any id it did not mint.
     """
-    item_id: Final = r_item.get("id")
-    encrypted_content: Final = r_item.get("encrypted_content")
-    return {
-        "type": "reasoning",
-        **({"id": item_id} if item_id else {}),
-        # summary is always required by the Responses API, even when empty
-        "summary": r_item.get("summary") or [],
-        **({"encrypted_content": encrypted_content} if encrypted_content else {}),
-    }
+    return dict(reasoning_input_item(r_item.get("id"), r_item.get("summary"), r_item.get("encrypted_content")))
 
 
 def _requested_defer_loading(tool: Mapping[str, object], function_tool: Mapping[str, object]) -> object:
@@ -804,12 +860,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         keep_prompt_cache_breakpoints: bool = False,
         replay_tool_search: bool = False,
         include_namespace: bool = False,
+        replay_openai_history: bool = False,
     ) -> tuple[list[object], str | None]:
         converted_input_items, instructions = self._convert_chat_completion_messages_to_responses_input(
             messages,
             drop_params=drop_params,
             replay_tool_search=replay_tool_search,
             include_namespace=include_namespace,
+            replay_openai_history=replay_openai_history,
         )
         return (
             converted_input_items
@@ -825,10 +883,18 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         drop_params: bool = False,
         replay_tool_search: bool = False,
         include_namespace: bool = False,
+        replay_openai_history: bool = False,
     ) -> tuple[list[object], str | None]:
         input_items: Final[list[object]] = []
         instructions: str | None = None
-        custom_tool_call_ids: Final = frozenset(
+        replayed_turns: Final = tuple(
+            replay_items(msg, replay_tool_search=replay_tool_search) if replay_openai_history else None
+            for msg in messages
+        )
+        replayed_custom_call_ids: Final = frozenset(
+            chain.from_iterable(turn.custom_call_ids for turn in replayed_turns if turn is not None)
+        )
+        custom_tool_call_ids: Final = replayed_custom_call_ids | frozenset(
             tool_call["id"]
             for msg in messages
             if msg.get("role") == "assistant" and isinstance(msg.get("tool_calls"), list)
@@ -849,7 +915,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             tool_calls = msg.get("tool_calls")
             tool_call_id = msg.get("tool_call_id")
 
-            if role == "system":
+            if (replayed_turn := replayed_turns[index]) is not None:
+                input_items.extend(replayed_turn.items)
+            elif role == "system":
                 # Extract system message as instructions
                 if isinstance(content, str) and index < leading_system_count:
                     if instructions:
@@ -915,6 +983,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                             "content": self._convert_content_to_responses_format(
                                 content, "assistant", drop_params=drop_params
                             ),
+                            **_assistant_phase_field(msg, replay_openai_history, drop_params),
                         }
                     )
                 for tool_call in tool_calls:
@@ -955,6 +1024,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                         "content": self._convert_content_to_responses_format(
                             content, cast(str, role), drop_params=drop_params
                         ),
+                        **_assistant_phase_field(msg, replay_openai_history and role == "assistant", drop_params),
                     }
                 )
             elif role == "assistant":
@@ -1081,6 +1151,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint,
             replay_tool_search=_requests_tool_search(cast(object, optional_params)),  # cast-ok: upcast
             include_namespace=True,
+            replay_openai_history=provider == "openai",
         )
         # OpenAI's Responses API rejects an empty input. For a system-only
         # request, carry the system message as a system-role input item instead
@@ -1165,6 +1236,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         output_items: Sequence[object],
         handle_raw_dict_callback: Callable[..., tuple["Choices | None", int]] | None = None,
         structured_output: bool = False,
+        replay_openai_history: bool = False,
+        completed: bool = False,
     ) -> list["Choices"]:
         """
         Convert Responses API output items to chat completion choices.
@@ -1174,6 +1247,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             handle_raw_dict_callback: Optional callback for handling raw dict items
             structured_output: The chat request carried a json response_format, so only the
                 first message item becomes content
+            replay_openai_history: Record the output's item sequence for the next turn
+            completed: The response finished, so its item sequence is whole
 
         Returns:
             List of Choices objects
@@ -1196,7 +1271,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         choices: Final[list[Choices]] = []
         index = 0
         reasoning_content: str | None = None
-        pending_reasoning_item: _BuiltReasoningItem | None = None
+        pending_reasoning_items: tuple[_BuiltReasoningItem, ...] = ()
 
         # Collect all tool calls to put them in a single choice
         # (Chat Completions API expects all tool calls in one message)
@@ -1205,12 +1280,19 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
 
         for item in output_items:
             if isinstance(item, ResponseReasoningItem):
-                pending_reasoning_item = _build_reasoning_item(
-                    item_id=item.id,
-                    encrypted_content=getattr(item, "encrypted_content", None),
-                    summary_raw=item.summary,
+                pending_reasoning_items = (
+                    *(pending_reasoning_items if replay_openai_history else ()),
+                    _build_reasoning_item(
+                        item_id=item.id,
+                        encrypted_content=getattr(item, "encrypted_content", None),
+                        summary_raw=item.summary,
+                    ),
+                )
+                reasoning_content = " ".join(
+                    s["text"]
+                    for s in chain.from_iterable(built["summary"] for built in pending_reasoning_items)
+                    if s.get("text")
                 )
-                reasoning_content = " ".join(s["text"] for s in pending_reasoning_item["summary"] if s.get("text"))
 
             elif isinstance(item, ResponseOutputMessage):
                 for content in item.content:
@@ -1227,7 +1309,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                         annotations=annotations,
                         reasoning_items=cast(
                             list[ChatCompletionReasoningItem] | None,
-                            ([pending_reasoning_item] if pending_reasoning_item is not None else None),
+                            (list(pending_reasoning_items) or None),
                         ),
                     )
 
@@ -1240,7 +1322,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                     )
 
                     reasoning_content = None  # flush
-                    pending_reasoning_item = None  # flush
+                    pending_reasoning_items = ()  # flush
                     index += 1
 
             elif isinstance(item, ResponseFunctionToolCall):
@@ -1300,16 +1382,17 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                     output_items, "stop", tool_search_items=tool_search_items
                 )
             ]
-        return [
-            _fold_choices_into_one(
-                choices=choices,
-                tool_calls=accumulated_tool_calls,
-                trailing_reasoning_content=reasoning_content,
-                trailing_reasoning_item=pending_reasoning_item,
-                tool_search_items=tool_search_items,
-                structured_output=structured_output,
-            )
-        ]
+        folded: Final = _fold_choices_into_one(
+            choices=choices,
+            tool_calls=accumulated_tool_calls,
+            trailing_reasoning_content=reasoning_content,
+            trailing_reasoning_items=pending_reasoning_items,
+            tool_search_items=tool_search_items,
+            structured_output=structured_output,
+        )
+        if not replay_openai_history:
+            return [folded]
+        return [_with_replay_fields(folded, output_items, completed=completed, structured_output=structured_output)]
 
     @staticmethod
     def _build_empty_choice(
@@ -1443,6 +1526,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
             structured_output=is_structured_output_request(
                 cast("Mapping[str, object]", optional_params)  # cast-ok: optional_params is an untyped dict
             ),
+            replay_openai_history=_served_provider(
+                cast("Mapping[str, object]", litellm_params)  # cast-ok: litellm_params is an untyped dict
+            )
+            == "openai",
+            completed=raw_response.status == "completed",
         )
 
         response_is_incomplete: Final = raw_response.status == "incomplete" or (
@@ -1506,9 +1594,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         sync_stream: bool,
         json_mode: bool | None = False,
         structured_output: bool = False,
+        replays_openai_history: bool = False,
     ) -> BaseModelResponseIterator:
         return OpenAiResponsesToChatCompletionStreamIterator(
-            streaming_response, sync_stream, json_mode, structured_output=structured_output
+            streaming_response,
+            sync_stream,
+            json_mode,
+            structured_output=structured_output,
+            replays_openai_history=replays_openai_history,
         )
 
     def _convert_content_str_to_input_text(self, content: str, role: str) -> dict[str, object]:
@@ -1890,8 +1983,10 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
         sync_stream: bool,
         json_mode: bool | None = False,
         structured_output: bool = False,
+        replays_openai_history: bool = False,
     ):
         super().__init__(streaming_response, sync_stream, json_mode)
+        self._replays_openai_history: Final = replays_openai_history
         self._chat_completion_id: str | None = None
         self._served_service_tier: str | None = None
         self._tool_call_index_map: dict[int, int] = {}  # mutable-ok: per-stream accumulator state
@@ -1937,6 +2032,8 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
     def translate_responses_chunk_to_openai_stream(
         parsed_chunk: dict | BaseModel,
         tool_call_index_map: dict[int, int] | None = None,  # mutable-ok: per-stream state, remapped in place
+        replays_openai_history: bool = False,
+        structured_output: bool = False,
     ) -> "ModelResponseStream":
         """
         Translate a Responses API streaming chunk to OpenAI chat completion streaming format.
@@ -1944,6 +2041,8 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
         Args:
             parsed_chunk: Dict containing the Responses API event chunk
             tool_call_index_map: Per-stream output_index -> sequential tool_call index map
+            replays_openai_history: Record the completed output's item sequence for the next turn
+            structured_output: Only the first message item became content
 
         Returns:
             ModelResponseStream: OpenAI-formatted streaming chunk
@@ -2154,6 +2253,10 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
             terminal_reasoning_items_typed: Final = _as_chat_reasoning_items(
                 _reasoning_items_from_output_items(output_items)
             )
+            output_payload: Final = cast(object, output_items)  # cast-ok: untyped
+            tool_search_fields: Final = _tool_search_provider_fields(
+                _tool_search_items_from_output_items(output_payload)
+            )
 
             usage = None
             if response_data.get("usage"):
@@ -2167,10 +2270,17 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
                     StreamingChoices(
                         index=0,
                         delta=Delta(
-                            content="",
+                            content=None if replays_openai_history else "",
                             reasoning_items=terminal_reasoning_items_typed,
-                            provider_specific_fields=_tool_search_provider_fields(
-                                _tool_search_items_from_output_items(cast(object, output_items))  # cast-ok: untyped
+                            provider_specific_fields=(
+                                completed_stream_history(
+                                    output_payload,
+                                    tool_search_fields,
+                                    completed=event_type == "response.completed",
+                                    structured_output=structured_output,
+                                )
+                                if replays_openai_history
+                                else tool_search_fields
                             ),
                         ),
                         finish_reason=finish_reason,
@@ -2216,7 +2326,10 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
             self._empty_content_chunk()
             if self._is_text_delta_after_structured_answer(chunk)
             else OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream(
-                chunk, tool_call_index_map=self._tool_call_index_map
+                chunk,
+                tool_call_index_map=self._tool_call_index_map,
+                replays_openai_history=self._replays_openai_history,
+                structured_output=self._structured_output,
             )
         )
         if self._structured_output and _is_message_item_done(chunk):

litellm/constants.py

@@ -1064,6 +1064,7 @@ openai_compatible_endpoints: Final[list[str]] = [
     "https://api.reka.ai/v1",
     "https://gigachat.devices.sberbank.ru/api/v1",
     "https://api.openinfer.ai/v1",
+    "https://vanchin.streamlake.ai/api/gateway/v1/endpoints",
 ]
 
 
@@ -1145,6 +1146,7 @@ openai_compatible_providers: Final[list[str]] = [
     "prism",
     "sail",
     "reka",
+    "streamlake",
 ]
 
 OPENAI_AUDIO_TRANSCRIPTION_PROVIDERS: Final = frozenset({"openai"} | frozenset(openai_compatible_providers))

litellm/google_genai/adapters/transformation.py

@@ -198,6 +198,11 @@ def _genai_message_part(part: object) -> _GenAIMessagePart | None:
     return ChatCompletionTextObject(type="text", text=text) if isinstance(text, str) else _genai_media_part(fields)
 
 
+def _genai_content_parts(value: object) -> Sequence[object]:
+    single_part: Final = _validated(_CONFIG_FIELDS, value)
+    return (single_part,) if single_part is not None else _validated(_GENAI_CONTENT_PARTS, value) or ()
+
+
 def _genai_message_parts(parts: object) -> tuple[_GenAIMessagePart, ...]:
     items: Final = _validated(_GENAI_CONTENT_PARTS, parts) or ()
     return tuple(converted for part in items if (converted := _genai_message_part(part)) is not None)
@@ -248,7 +253,7 @@ def _genai_content_messages(content: object) -> tuple[AllMessageValues, ...]:
     fields: Final = _validated(_CONFIG_FIELDS, content)
     if fields is None:
         return ()
-    parts: Final = _validated(_GENAI_CONTENT_PARTS, fields.get("parts", ())) or ()
+    parts: Final = _genai_content_parts(fields.get("parts", ()))
     converted: Final = _genai_message_parts(parts)
     role: Final = fields.get("role", "user")
     if role == "user":

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -18,6 +18,7 @@ from litellm.litellm_core_utils.bug_report import (
 )
 from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none
 from litellm.litellm_core_utils.secret_redaction import redact_string as redact_secret_string
+from litellm.llms.bigmodel.common_utils import raise_bigmodel_business_error
 from litellm.types.utils import LlmProviders
 
 from ..exceptions import (
@@ -2762,6 +2763,8 @@ def exception_type(
                     exception_provider=exception_provider,
                     extra_information=extra_information,
                 )
+            if custom_llm_provider == "bigmodel":
+                raise_bigmodel_business_error(model, f"{exception_provider} - {error_str}", original_exception)  # pyright: ignore[reportUnknownArgumentType]  # exception_type takes untyped parameters
             if custom_llm_provider == "openrouter":
                 _map_openrouter_exception(
                     model=model,

litellm/litellm_core_utils/get_litellm_params.py

@@ -77,6 +77,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
             "otpm",
             "tpd",
             "rpd",
+            "daily_reset_time",
             "use_xai_oauth",
             "chatgpt_auth_file",
             "github_copilot_auth_type",

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -794,6 +794,13 @@ def _get_openai_compatible_provider_info(
             api_base,
             dynamic_api_key,
         ) = litellm.XAIChatConfig().get_openai_compatible_provider_info(api_base, api_key)
+    elif custom_llm_provider == "bigmodel":
+        from litellm.llms.bigmodel.common_utils import get_bigmodel_api_key, get_bigmodel_url
+
+        (
+            api_base,  # rebind-ok: existing provider-resolution pipeline resolves the API base in place
+            dynamic_api_key,  # rebind-ok: existing provider-resolution pipeline resolves the API key in place
+        ) = get_bigmodel_url(api_base, "chat"), get_bigmodel_api_key(api_key)
     elif custom_llm_provider == "zai":
         (
             api_base,

litellm/litellm_core_utils/internal_call_metadata.py

@@ -120,7 +120,7 @@ def is_unbilled_non_inference_call(
     origin: Final = metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY) if metadata is not None else None
     if origin == BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN:
         return False
-    if not is_background_response(response):
+    if not is_background_response(_finished_response(response)):
         return True
     return origin == BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN or _is_managed_background_call(metadata)
 

litellm/litellm_core_utils/litellm_logging.py

@@ -5052,6 +5052,23 @@ class Logging(LiteLLMLoggingBaseClass):
         return result_copy
 
 
+_SENSITIVE_KEYWORDS: Final = (
+    "authorization",
+    "token",
+    "key",
+    "secret",
+    "vertex_credentials",
+    "credentials",
+    "password",
+    "passwd",
+)
+
+
+def has_sensitive_keyword(key: str) -> bool:
+    lowered: Final = key.lower()
+    return any(keyword in lowered for keyword in _SENSITIVE_KEYWORDS)
+
+
 def get_masked_values(
     sensitive_object: Mapping[str, object],
     ignore_sensitive_values: bool = False,
@@ -5070,16 +5087,6 @@ def get_masked_values(
         masked_length: Optional length for the masked portion (number of *). If set, will use exactly this many *
                      regardless of original string length. The total length will be unmasked_length + masked_length.
     """
-    sensitive_keywords: Final = (
-        "authorization",
-        "token",
-        "key",
-        "secret",
-        "vertex_credentials",
-        "credentials",
-        "password",
-        "passwd",
-    )
 
     def _mask_value(v: object) -> object:
         if isinstance(v, dict):
@@ -5106,12 +5113,7 @@ def get_masked_values(
         return v[: unmasked_length // 2] + "*" * (len(v) - unmasked_length) + v[-unmasked_length // 2 :]
 
     return {
-        k: (
-            v
-            if ignore_sensitive_values
-            or not any(sensitive_keyword in k.lower() for sensitive_keyword in sensitive_keywords)
-            else _mask_value(v)
-        )
+        k: (v if ignore_sensitive_values or not has_sensitive_keyword(k) else _mask_value(v))
         for k, v in sensitive_object.items()
     }
 

litellm/litellm_core_utils/llm_request_utils.py

@@ -1,8 +1,9 @@
-from collections.abc import Mapping
+from collections.abc import Iterable, Iterator, Mapping
 from typing import Final, cast
 
 import litellm
 from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
+from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider
 
 
 def _form_field_value(value: object) -> str:
@@ -153,24 +154,33 @@ def pick_cheapest_chat_models_from_llm_provider(custom_llm_provider: str, n=1):
     if custom_llm_provider not in litellm.models_by_provider:
         return []
 
-    known_models: Final = litellm.models_by_provider.get(custom_llm_provider, [])
-    model_costs: Final = []
+    known_models: Final[Iterable[str]] = litellm.models_by_provider.get(custom_llm_provider, ())
+    ranked_rows: Final = (
+        (route_rank, cost, model)
+        for model, cost in _chat_row_costs(known_models, custom_llm_provider)
+        if (route_rank := _route_rank(model, custom_llm_provider)) is not None
+    )
+    return [model for _, _, model in sorted(ranked_rows, key=lambda row: row[:2])[:n]]
 
-    for model in known_models:
+
+def _chat_row_costs(models: Iterable[str], custom_llm_provider: str) -> Iterator[tuple[str, float]]:
+    for model in models:
         try:
             model_info = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
         except Exception:
             continue
         if model_info.get("mode") != "chat":
             continue
-        _cost = (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0)
-        model_costs.append((model, _cost))
+        yield model, (model_info.get("input_cost_per_token") or 0.0) + (model_info.get("output_cost_per_token") or 0.0)
 
-    # Sort by cost (ascending)
-    model_costs.sort(key=lambda x: x[1])
 
-    # Return the top n cheapest models
-    return [model for model, _ in model_costs[:n]]
+def _route_rank(model: str, custom_llm_provider: str) -> int | None:
+    routed_provider: Final = inferred_provider(model)
+    if routed_provider is None:
+        return 1
+    if routed_provider == inferred_provider(f"{custom_llm_provider}/{model}"):
+        return 0
+    return None
 
 
 def get_proxy_server_request_headers(litellm_params: dict | None) -> dict:

litellm/litellm_core_utils/streaming_chunk_builder_utils.py

@@ -18,6 +18,7 @@ from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import BEDROCK_TRAC
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     get_content_from_model_response,  # pyright: ignore[reportUnknownVariableType]  # response param accepts an untyped dict
 )
+from litellm.litellm_core_utils.token_counter import usage_tokenizer_model
 from litellm.types.llms.openai import (
     AllMessageValues,
     ChatCompletionAssistantContentValue,
@@ -51,7 +52,7 @@ from litellm.types.utils import (
     UnreportedUsage,
     Usage,
 )
-from litellm.utils import print_verbose, token_counter
+from litellm.utils import get_base_model_from_metadata, print_verbose, token_counter
 
 if TYPE_CHECKING:
     from openai.types.completion_usage import CompletionUsage
@@ -1552,6 +1553,9 @@ def stream_chunk_builder(
             return stream_chunk_builder_text_completion(chunks=chunks, messages=messages)
 
         model: Final = chunks[0]["model"]
+        usage_model: Final = usage_tokenizer_model(
+            model, get_base_model_from_metadata(logging_obj.model_call_details) if logging_obj is not None else None
+        )
         # Initialize the response dictionary
         response: Final = processor.build_base_response(chunks)
 
@@ -1575,7 +1579,7 @@ def stream_chunk_builder(
             completion_output = get_content_from_model_response(response)
             usage = processor.calculate_usage(
                 chunks=chunks,
-                model=model,
+                model=usage_model,
                 completion_output=completion_output,
                 messages=messages,
                 reasoning_tokens=0,
@@ -1766,7 +1770,7 @@ def stream_chunk_builder(
 
         usage = processor.calculate_usage(
             chunks=chunks,
-            model=model,
+            model=usage_model,
             completion_output=completion_output,
             messages=messages,
             reasoning_tokens=reasoning_tokens,

litellm/litellm_core_utils/streaming_handler.py

@@ -1938,6 +1938,16 @@ class CustomStreamWrapper:
         model_response: Final = self.model_response_creator(
             hidden_params=None if self.received_model_id is None else {"received_model_id": self.received_model_id}
         )
+        served_tier: Final = next(
+            (
+                tier
+                for chunk in reversed(self.chunks)
+                if isinstance(tier := getattr(chunk, "service_tier", None), str) and tier
+            ),
+            None,
+        )
+        if served_tier is not None:
+            setattr(model_response, "service_tier", served_tier)  # noqa: B010  # pydantic extra, not a declared field
         _finish_reason: Final = self.received_finish_reason or self.intermittent_finish_reason
         if _finish_reason is not None:
             model_response.choices[0].finish_reason = _finish_reason

litellm/litellm_core_utils/token_counter.py

@@ -15,6 +15,7 @@ import anyio
 import anyio.lowlevel
 import httpx
 import tiktoken
+from tiktoken.model import MODEL_TO_ENCODING
 from typing_extensions import ParamSpec, TypeVar
 
 import litellm
@@ -49,6 +50,7 @@ if TYPE_CHECKING:
     from litellm.litellm_core_utils.tokenizer import Encoding, HuggingFace
 from litellm.litellm_core_utils.url_utils import safe_get
 from litellm.llms.custom_httpx.http_handler import get_httpx_client, header_value
+from litellm.router_utils.deployment_pricing import deployment_pricing_models
 from litellm.rust_bridge.tokenizer import get_encoding
 from litellm.types.llms.anthropic import (
     AnthropicContentParamSource,
@@ -1005,17 +1007,144 @@ def openai_tokenizer_encoding(model: str) -> "Encoding":
 
 
 def openai_tokenizer_encoding_name(model: str) -> str:
-    """The tiktoken encoding name for `model`, without loading the encoding."""
-    from litellm.utils import print_verbose
+    """The tiktoken encoding name for `model`, without loading the encoding.
 
-    model_to_use: Final = _fix_model_name(model)
+    A listed OpenAI or Azure model that tiktoken does not know is either a suffixed variant of a model it does
+    know (`text-embedding-ada-002-v2`) or a newer release than its table, which gets o200k_base, the encoding every
+    OpenAI model since gpt-4o uses. Provider and region prefixes (`azure/eu/`, `azure_ai/`) are dropped, a name
+    listed only under Azure (`gpt-41`, `openai/o1-mini`) counts as listed through its `azure/` cost-map key, and an
+    unlisted name tiktoken counts with o200k_base (`openai/gpt-5.9`) or named like a gpt-5 or o-series release
+    (`openai/gpt-6.2`) gets it."""
+    model_to_use: Final = _openai_model_name(model) or _fix_model_name(model)
     if "gpt-4o" in model_to_use:
         return "o200k_base"
     try:
         return tiktoken.encoding_name_for_model(model_to_use)
     except KeyError:
-        print_verbose("Warning: model not found. Using cl100k_base encoding.")
-        return "cl100k_base"
+        return _suffixed_variant_encoding_name(model_to_use) or "o200k_base"
+
+
+def unresolved_tokenizer_model(model: str) -> str:
+    """`model`, or `gpt-3.5-turbo` when resolving the OpenAI model `model` names moves it off cl100k_base, the encoding
+    tiktoken gives the name as written, so a count with the result never drops below that one."""
+    as_written: Final = _fix_model_name(model)
+    as_written_encoding: Final = (
+        "o200k_base" if "gpt-4o" in as_written else (_tiktoken_encoding_name(as_written) or "cl100k_base")
+    )
+    moved_off_cl100k: Final = (
+        as_written_encoding == "cl100k_base" and openai_tokenizer_encoding_name(model) != "cl100k_base"
+    )
+    return "gpt-3.5-turbo" if moved_off_cl100k else model
+
+
+def usage_tokenizer_model(model: str, base_model: str | None) -> str:
+    """The model to estimate missing usage with: a deployment's `base_model` when it names an OpenAI model, since a
+    custom deployment name says nothing about its tokenizer, else `model`."""
+    return base_model if base_model is not None and is_openai_model(base_model) else model
+
+
+def is_openai_model(model: str) -> bool:
+    """Whether `model` names an OpenAI or Azure OpenAI model, listed in the cost map or counted with o200k_base by
+    tiktoken, with or without its prefixes."""
+    return _openai_model_name(model) is not None
+
+
+def deployment_openai_model(deployment: object) -> str | None:
+    """The OpenAI model a router deployment serves, read from its base model first, since a custom deployment name
+    (`azure/my-gpt-5`) says nothing about its tokenizer, or None when it serves no OpenAI model."""
+    return _as_openai_model(_served_model(deployment))
+
+
+def served_openai_models(deployments: Iterable[object]) -> tuple[str | None, ...]:
+    """The distinct OpenAI models `deployments` serve, with None standing for any deployment serving another model."""
+    served: Final = dict.fromkeys(_served_model(deployment) for deployment in deployments)
+    return tuple(dict.fromkeys(_as_openai_model(model) for model in served))
+
+
+def _served_model(deployment: object) -> str | None:
+    return next(iter(deployment_pricing_models(deployment)), None)
+
+
+def _as_openai_model(model: str | None) -> str | None:
+    return model if model is not None and is_openai_model(model) else None
+
+
+def single_encoding_openai_model(served: Sequence[str | None]) -> str | None:
+    """An OpenAI model to count a request with, when every `served_openai_models` entry is an OpenAI model on the same
+    encoding, else None."""
+    if not served or None in served:
+        return None
+    encodings: Final = frozenset(openai_tokenizer_encoding_name(model) for model in served if model is not None)
+    return served[0] if len(encodings) == 1 else None
+
+
+def _openai_model_name(model: str) -> str | None:
+    return _listed_openai_model_name(model) or _tiktoken_o200k_model_name(model) or _versioned_openai_model_name(model)
+
+
+_VERSIONED_OPENAI_MODEL: Final = re.compile(r"(?:gpt-(?:[5-9]|[1-9]\d+)|o[1-9]\d*)(?:[.-]|$)")
+
+
+def _versioned_openai_model_name(model: str) -> str | None:
+    name: Final = model.rsplit("/", 1)[-1]
+    return name if _VERSIONED_OPENAI_MODEL.match(name) else None
+
+
+def _tiktoken_o200k_model_name(model: str) -> str | None:
+    name: Final = model.rsplit("/", 1)[-1]
+    return name if _tiktoken_encoding_name(name) == "o200k_base" else None
+
+
+def _listed_openai_model_name(model: str) -> str | None:
+    segments: Final = model.split("/")
+    suffixes: Final = ("/".join(segments[start:]) for start in range(len(segments)))
+    listed: Final = any(
+        _is_openai_cost_map_model(suffix) or _is_openai_cost_map_model(f"azure/{suffix}") for suffix in suffixes
+    )
+    return segments[-1] if listed else None
+
+
+def _is_openai_cost_map_model(model: str) -> bool:
+    return (
+        model in litellm.azure_llms
+        or model in litellm.open_ai_chat_completion_models
+        or _is_azure_openai_text_model(model)
+    )
+
+
+_TEXT_MODEL_MODES: Final = frozenset({"chat", "completion", "responses", "realtime"})
+
+
+def _is_azure_openai_text_model(model: str) -> bool:
+    cost_info: Final[Mapping[str, object]] = litellm.model_cost.get(model) or {}
+    return (
+        cost_info.get("litellm_provider") == "azure"
+        and cost_info.get("mode") in _TEXT_MODEL_MODES
+        and _is_openai_model_name(model.rsplit("/", 1)[-1])
+    )
+
+
+def _is_openai_model_name(name: str) -> bool:
+    return (
+        name in litellm.open_ai_chat_completion_models
+        or _tiktoken_encoding_name(name) is not None
+        or any(
+            name.startswith(f"{listed}-") or listed.startswith(f"{name}-")
+            for listed in tuple(litellm.open_ai_chat_completion_models)
+        )
+    )
+
+
+def _tiktoken_encoding_name(model: str) -> str | None:
+    try:
+        return tiktoken.encoding_name_for_model(model)
+    except KeyError:
+        return None
+
+
+def _suffixed_variant_encoding_name(model: str) -> str | None:
+    known: Final = tuple(name for name in MODEL_TO_ENCODING if model.startswith(f"{name}-"))
+    return MODEL_TO_ENCODING[max(known, key=len)] if known else None
 
 
 def uses_legacy_message_accounting(model: str) -> bool:

litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py

@@ -31,6 +31,7 @@ from ..constants import (
 )
 from ..errors import AnthropicContextManagementError
 from ..placeholders import build_cleared_tool_result_content
+from .compact import count_system_tokens, history_after_client_compaction
 
 if TYPE_CHECKING:
     from litellm.types.llms.openai import ChatCompletionToolParam
@@ -203,9 +204,10 @@ def _trigger_met(
     model: str,
     messages: list[dict[str, object]],
     tools: list[dict[str, object]] | None,
+    system: str | Sequence[Mapping[str, object]] | None,
     counted_tool_uses: int,
 ) -> tuple[bool, int | None]:
-    """Return (trigger_met, input_tokens if counted for reuse)."""
+    """Return (trigger_met, message and tool tokens for reuse); like the API, input_tokens adds the system prompt."""
     trigger_type: Final = trigger.get("type", "input_tokens")
     threshold = trigger.get("value")
 
@@ -216,14 +218,17 @@ def _trigger_met(
 
     if not isinstance(threshold, int):
         threshold = DEFAULT_INPUT_TOKENS_TRIGGER
-    current_tokens: Final = litellm.token_counter(
+    message_tokens: Final = litellm.token_counter(
         model=model,
         messages=messages,
         tools=cast("list[ChatCompletionToolParam] | None", tools),
     )
+    if message_tokens > threshold:
+        return True, message_tokens
+    current_tokens: Final = message_tokens + count_system_tokens(model, system)
     verbose_logger.debug("context_management polyfill: current_tokens: %s", current_tokens)
     verbose_logger.debug("context_management polyfill: threshold: %s", threshold)
-    return current_tokens > threshold, current_tokens
+    return current_tokens > threshold, message_tokens
 
 
 def _resolve_keep_count(keep: dict[str, object]) -> int:
@@ -237,7 +242,7 @@ def _resolve_keep_count(keep: dict[str, object]) -> int:
 
 
 def _last_completed_tool_use_id(
-    messages: list[dict[str, object]],
+    messages: Sequence[Mapping[str, object]],
 ) -> str | None:
     """Latest completed tool_result id; never cleared."""
     last_id: str | None = None
@@ -253,7 +258,7 @@ def _last_completed_tool_use_id(
 
 
 def _clear_tool_results(
-    messages: list[dict[str, object]], ids_to_clear: set[str]
+    messages: list[dict[str, object]], ids_to_clear: Set[str]
 ) -> tuple[list[dict[str, object]], int]:
     """Clear matching tool_result content; return (messages, cleared_count)."""
     cleared = 0
@@ -311,7 +316,11 @@ def apply_clear_tool_uses_20250919(
     system: str | list[dict[str, object]] | None,
     edit_spec: ClearToolUsesEditSpec,
 ) -> tuple[list[dict[str, object]], AppliedEdit | None]:
-    """Apply clear_tool_uses to an edit ``check_clear_tool_uses_spec`` returned; return (messages, AppliedEdit or None)."""
+    """Apply clear_tool_uses to an edit ``check_clear_tool_uses_spec`` returned; return (messages, AppliedEdit or None).
+
+    Like the API, it reads only the history after a client compaction block and counts the block's summary as system.
+    """
+    kept, counted_system = history_after_client_compaction(messages, system)
     fields: Final = _ClearToolUsesFields.model_validate(edit_spec)
     clear_at_least: Final = 0 if fields.clear_at_least is None else fields.clear_at_least.value
     exclude_tools: Final = frozenset(fields.exclude_tools or ())
@@ -326,7 +335,7 @@ def apply_clear_tool_uses_20250919(
         "value": DEFAULT_KEEP_TOOL_USES,
     }
 
-    blocks: Final = _all_blocks(messages)
+    blocks: Final = _all_blocks(kept)
     results: Final = _blocks_by_id(blocks, "tool_result", "tool_use_id")
     uses: Final = _blocks_by_id(blocks, "tool_use", "id")
     tiny: Final = _tiny_result_ids(results)
@@ -335,15 +344,15 @@ def apply_clear_tool_uses_20250919(
         for tool_use_id, name in _tool_uses_in_order(blocks)
         if name not in exclude_tools and (tool_use_id not in tiny or _clears_input(name, input_tools))
     )
-    met, counted_tokens = _trigger_met(trigger, model, messages, tools, len(clearable))
-    if not met:
-        return messages, None
-
     keep_count: Final = _resolve_keep_count(keep)
     if len(clearable) <= keep_count:
         return messages, None
 
-    last_completed_id: Final = _last_completed_tool_use_id(messages)
+    met, counted_tokens = _trigger_met(trigger, model, kept, tools, counted_system, len(clearable))
+    if not met:
+        return messages, None
+
+    last_completed_id: Final = _last_completed_tool_use_id(kept)
     candidates: Final = tuple(
         (tool_use_id, name)
         for tool_use_id, name in clearable[: len(clearable) - keep_count]
@@ -358,7 +367,7 @@ def apply_clear_tool_uses_20250919(
         _select_in_batches(candidate_ids, tokens, clear_at_least) if clear_at_least > 0 else candidate_ids
     )
 
-    edited, cleared_count = _clear_tool_results(messages, set(ids_to_clear))
+    edited, cleared_count = _clear_tool_results(messages, ids_to_clear)
     verbose_logger.debug("context_management polyfill: edited: %s", edited)
     if cleared_count == 0:
         return messages, None
@@ -370,11 +379,12 @@ def apply_clear_tool_uses_20250919(
         counted_tokens
         if counted_tokens is not None
         else litellm.token_counter(
-            model=model, messages=messages, tools=cast("list[ChatCompletionToolParam] | None", tools)
+            model=model, messages=kept, tools=cast("list[ChatCompletionToolParam] | None", tools)
         )
     )
+    kept_after, _ = history_after_client_compaction(final_messages, system)
     tokens_after: Final = litellm.token_counter(
-        model=model, messages=final_messages, tools=cast("list[ChatCompletionToolParam] | None", tools)
+        model=model, messages=kept_after, tools=cast("list[ChatCompletionToolParam] | None", tools)
     )
     cleared_input_tokens: Final = max(tokens_before - tokens_after, 0)
 

litellm/llms/anthropic/messages/context_management/editors/compact.py

@@ -189,6 +189,21 @@ def _strip_compaction_blocks(
     return cleaned
 
 
+def history_after_client_compaction(
+    messages: Sequence[dict[str, object]],
+    system: str | Sequence[Mapping[str, object]] | None,
+) -> tuple[list[dict[str, object]], str | Sequence[Mapping[str, object]] | None]:
+    """Return the history the API keeps after the latest compaction block, without it, and the system with its summary."""
+    sliced, block = _slice_around_compaction_block(messages)
+    if block is None:
+        return list(messages), system
+    tail: Final = _strip_compaction_blocks(sliced)
+    summary: Final = block.get("content")
+    if not isinstance(summary, str) or not summary:
+        return tail, system
+    return tail, _augment_system_with_summary(_system_to_text(system) or None, summary)
+
+
 def _augment_system_with_summary(
     system: str | list[dict[str, object]] | None,
     summary_text: str,
@@ -320,14 +335,16 @@ def _count_effective_tokens(
         content: Final = compaction_block.get("content") or ""
         if content:
             total += litellm.token_counter(model=model, text=content)
+    return total + count_system_tokens(model, system)
+
+
+def count_system_tokens(model: str, system: str | Sequence[Mapping[str, object]] | None) -> int:
     system_text: Final = _system_to_text(system)
-    if system_text:
-        total += litellm.token_counter(model=model, text=system_text)
-    return total
+    return litellm.token_counter(model=model, text=system_text) if system_text else 0
 
 
 def _system_to_text(
-    system: str | list[dict[str, object]] | None,
+    system: str | Sequence[Mapping[str, object]] | None,
 ) -> str:
     """Flatten an Anthropic-style ``system`` value into a single string for
     token counting. Returns ``""`` when ``system`` carries no text."""
@@ -338,7 +355,7 @@ def _system_to_text(
     return "\n".join(
         text
         for block in system
-        if isinstance(block, dict)
+        if isinstance(block, Mapping)
         and block.get("type") == "text"
         and isinstance(text := block.get("text"), str)
         and text

litellm/llms/azure/azure.py

@@ -33,6 +33,7 @@ from litellm.types.utils import (
 from litellm.utils import (
     CustomStreamWrapper,
     convert_to_model_response_object,
+    get_base_model_from_metadata,
 )
 
 from ...types.llms.openai import HttpxBinaryResponseContent
@@ -145,6 +146,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
         azure_client: AzureOpenAI | OpenAI,
         data: dict,
         timeout: float | httpx.Timeout,
+        logging_obj: LiteLLMLoggingObj,
     ):
         """
         Helper to:
@@ -160,7 +162,12 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
         except BadRequestError as e:
             if not is_output_token_limit_error(e):
                 raise
-            return build_output_token_limit_response(e=e, data=data, is_async=False)
+            return build_output_token_limit_response(
+                e=e,
+                data=data,
+                is_async=False,
+                base_model=get_base_model_from_metadata(logging_obj.model_call_details),
+            )
         except Exception as e:
             raise e
 
@@ -192,7 +199,12 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
         except BadRequestError as e:
             if not is_output_token_limit_error(e):
                 raise
-            return build_output_token_limit_response(e=e, data=data, is_async=True)
+            return build_output_token_limit_response(
+                e=e,
+                data=data,
+                is_async=True,
+                base_model=get_base_model_from_metadata(logging_obj.model_call_details),
+            )
         except Exception as e:
             raise e
 
@@ -350,7 +362,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                     )
 
                 headers, response = self.make_sync_azure_openai_chat_completion_request(
-                    azure_client=azure_client, data=data, timeout=timeout
+                    azure_client=azure_client, data=data, timeout=timeout, logging_obj=logging_obj
                 )
                 if not isinstance(response, BaseModel):
                     raise AzureOpenAIError(
@@ -564,7 +576,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
             },
         )
         headers, response = self.make_sync_azure_openai_chat_completion_request(
-            azure_client=azure_client, data=data, timeout=timeout
+            azure_client=azure_client, data=data, timeout=timeout, logging_obj=logging_obj
         )
         logging_obj.model_call_details["response_headers"] = headers
         streamwrapper: Final = CustomStreamWrapper(

litellm/llms/azure/responses/transformation.py

@@ -142,7 +142,9 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
                 return filtered_item
         return item
 
-    def _validate_input_param(self, input: str | ResponseInputParam) -> str | ResponseInputParam:
+    def _validate_input_param(
+        self, input: str | ResponseInputParam, *, drops_set_created_by: bool = False
+    ) -> str | ResponseInputParam:
         """
         Override parent method to also filter out 'status' field from message items.
         Azure OpenAI API does not accept 'status' field in input messages.
@@ -150,7 +152,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
         from typing import cast
 
         # First call parent's validation
-        validated_input: Final = super()._validate_input_param(input)
+        validated_input: Final = super()._validate_input_param(input, drops_set_created_by=drops_set_created_by)
 
         # Then filter out status from message items
         if isinstance(validated_input, list):
@@ -333,7 +335,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
         api_base: str,
         litellm_params: GenericLiteLLMParams,
         headers: dict,
-    ) -> tuple[str, dict]:
+    ) -> tuple[str, dict[str, str]]:
         """
         Transform the get response API request into a URL and data
 
@@ -341,7 +343,7 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
         - GET /v1/responses/{response_id}
         """
         get_url: Final = self._construct_url_for_response_id_in_path(api_base=api_base, response_id=response_id)
-        data: Final[dict] = {}
+        data: Final[dict[str, str]] = {}
         verbose_logger.debug("get response url=%s", get_url)
         return get_url, data
 

litellm/llms/azure_ai/common_utils.py

@@ -140,6 +140,15 @@ def get_azure_ai_auth_headers(
 AZURE_MODEL_ROUTER_SELECTED_MODEL_KEY: Final = "azure_model_router_selected_model"
 
 
+FOUNDRY_CLASSIC_AGENT_ID_PREFIX: Final = "asst_"
+
+
+def is_foundry_project_agent(model: str | None) -> bool:
+    if model is None or AzureFoundryModelInfo.get_azure_ai_route(model) != "agents":
+        return False
+    return not model.split("agents/", 1)[1].startswith(FOUNDRY_CLASSIC_AGENT_ID_PREFIX)
+
+
 def azure_ai_supports_native_responses(model: str | None, api_base: str | None) -> bool:
     resolved_base: Final = AzureFoundryModelInfo.get_api_base(api_base)
     if resolved_base is not None and not is_azure_openai_v1_host(resolved_base):

litellm/llms/azure_ai/responses/transformation.py

@@ -1,26 +1,64 @@
-from typing import Final
+import os
+from collections.abc import Collection, Mapping, Sequence
+from typing import TYPE_CHECKING, Final, NoReturn
 
 import httpx
+from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
 
+import litellm
+from litellm.exceptions import UnsupportedParamsError
 from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig
 from litellm.llms.azure_ai.chat.transformation import AzureAIGPT5Config
 from litellm.llms.azure_ai.common_utils import (
+    AZURE_AI_AGENTS_SCOPE,
     AzureFoundryModelInfo,
     api_key_header_for_base,
     get_azure_ai_auth_headers,
 )
+from litellm.types.llms.azure_ai import FoundryAgentReference
+from litellm.types.llms.openai import (
+    ResponseInputParam,
+    ResponsesAPIOptionalRequestParams,
+    ResponsesAPIResponse,
+    ResponsesAPIStreamingResponse,
+)
 from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import LlmProviders
 
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+
 _PROJECT_PATH_PREFIX: Final = ("api", "projects")
 _RESPONSES_PATH: Final = ("openai", "v1", "responses")
+_AGENT_OWNED_PARAMS: Final = frozenset({"temperature", "top_p", "tools", "text", "reasoning"})
+_CALLER_AGENT_KEYS: Final = ("agent", "agent_reference")
+_RESPONSE_FIELDS: Final = TypeAdapter(dict[str, object])
+
+
+def _project_root(base_url: httpx.URL) -> tuple[str, ...]:
+    segments: Final = tuple(segment for segment in base_url.path.split("/") if segment)
+    return segments[:3] if segments[:2] == _PROJECT_PATH_PREFIX else ()
 
 
 def _responses_url(api_base: str) -> str:
     base_url: Final = httpx.URL(api_base)
-    segments: Final = tuple(segment for segment in base_url.path.split("/") if segment)
-    project_root: Final = segments[:3] if segments[:2] == _PROJECT_PATH_PREFIX else ()
-    return str(base_url.copy_with(path="/" + "/".join((*project_root, *_RESPONSES_PATH)), query=None))
+    return str(base_url.copy_with(path="/" + "/".join((*_project_root(base_url), *_RESPONSES_PATH)), query=None))
+
+
+def _without_agent_instructions(response: ResponsesAPIResponse) -> ResponsesAPIResponse:
+    if (response.model_extra or {}).get("agent_reference") is None:
+        return response
+    return response.model_copy(update={"instructions": None})
+
+
+def _event_without_agent_instructions(event: Mapping[str, object]) -> Mapping[str, object]:
+    response: Final = event.get("response")
+    if not isinstance(response, Mapping):
+        return event
+    fields: Final = _RESPONSE_FIELDS.validate_python(response)
+    if fields.get("agent_reference") is None:
+        return event
+    return {**event, "response": {**fields, "instructions": None}}
 
 
 class AzureAIResponsesAPIConfig(AzureOpenAIResponsesAPIConfig):
@@ -42,10 +80,19 @@ class AzureAIResponsesAPIConfig(AzureOpenAIResponsesAPIConfig):
 
     def validate_environment(self, headers: dict, model: str, litellm_params: GenericLiteLLMParams | None) -> dict:
         params: Final = litellm_params or GenericLiteLLMParams()
+        api_base: Final = AzureFoundryModelInfo.get_api_base(params.api_base)
+        on_project: Final = api_base is not None and bool(_project_root(httpx.URL(api_base)))
+        auth_params: Final = (
+            params.model_copy(
+                update={"azure_scope": params.azure_scope or os.getenv("AZURE_SCOPE") or AZURE_AI_AGENTS_SCOPE}
+            )
+            if on_project
+            else params
+        )
         auth_headers: Final = get_azure_ai_auth_headers(
             api_key=AzureFoundryModelInfo.get_api_key(params.api_key),
-            litellm_params=params.model_dump(),
-            api_key_header=api_key_header_for_base(AzureFoundryModelInfo.get_api_base(params.api_base)),
+            litellm_params=auth_params.model_dump(),
+            api_key_header="Authorization" if on_project else api_key_header_for_base(api_base),
         )
         return {
             **headers,
@@ -64,3 +111,207 @@ class AzureAIResponsesAPIConfig(AzureOpenAIResponsesAPIConfig):
                 "Set the api_base parameter or the AZURE_AI_API_BASE environment variable."
             )
         return _responses_url(resolved_base)
+
+    def transform_response_api_response(
+        self, model: str, raw_response: httpx.Response, logging_obj: "LiteLLMLoggingObj"
+    ) -> ResponsesAPIResponse:
+        return _without_agent_instructions(super().transform_response_api_response(model, raw_response, logging_obj))
+
+    def transform_streaming_response(
+        self,
+        model: str,
+        parsed_chunk: Mapping[str, object],
+        logging_obj: "LiteLLMLoggingObj",
+    ) -> ResponsesAPIStreamingResponse:
+        return super().transform_streaming_response(  # pyright: ignore[reportUnknownMemberType]  # parent types parsed_chunk as a bare dict
+            model=model, parsed_chunk=dict(_event_without_agent_instructions(parsed_chunk)), logging_obj=logging_obj
+        )
+
+    def transform_get_response_api_response(
+        self, raw_response: httpx.Response, logging_obj: "LiteLLMLoggingObj"
+    ) -> ResponsesAPIResponse:
+        return _without_agent_instructions(super().transform_get_response_api_response(raw_response, logging_obj))
+
+    def transform_cancel_response_api_response(
+        self, raw_response: httpx.Response, logging_obj: "LiteLLMLoggingObj"
+    ) -> ResponsesAPIResponse:
+        return _without_agent_instructions(super().transform_cancel_response_api_response(raw_response, logging_obj))
+
+
+def _agent_reference(model: str) -> FoundryAgentReference:
+    name, _, version = model.split("agents/", 1)[1].partition(":")
+    if version:
+        return FoundryAgentReference(type="agent_reference", name=name, version=version)
+    return FoundryAgentReference(type="agent_reference", name=name)
+
+
+class _CallerAgentReference(BaseModel):
+    model_config = ConfigDict(frozen=True, coerce_numbers_to_str=True)
+
+    name: str
+    version: str | None = None
+
+
+def _caller_agent_reference(value: object) -> _CallerAgentReference | None:
+    try:
+        return _CallerAgentReference.model_validate(value)
+    except ValidationError:
+        return None
+
+
+def _requested_agent(
+    deployment_agent: FoundryAgentReference, caller_agents: Sequence[_CallerAgentReference | None]
+) -> FoundryAgentReference:
+    requested_versions: Final = frozenset(
+        agent.version
+        for agent in caller_agents
+        if agent is not None and agent.name == deployment_agent["name"] and agent.version is not None
+    )
+    if "version" in deployment_agent or len(requested_versions) != 1:
+        return deployment_agent
+    return FoundryAgentReference(
+        type="agent_reference", name=deployment_agent["name"], version=next(iter(requested_versions))
+    )
+
+
+def _caller_agent_fields(fields: Mapping[str, object]) -> tuple[tuple[str, object], ...]:
+    return tuple((key, fields[key]) for key in _CALLER_AGENT_KEYS if key in fields)
+
+
+def _deployment_agent(
+    model: str, caller_fields: Sequence[tuple[str, object]], litellm_params: GenericLiteLLMParams
+) -> FoundryAgentReference:
+    caller_keys: Final = tuple(key for key, _ in caller_fields)
+    caller_agents: Final = tuple(_caller_agent_reference(value) for _, value in caller_fields)
+    agent: Final = _requested_agent(_agent_reference(model), caller_agents)
+    other_agents: Final = tuple(
+        key
+        for key, caller_agent in zip(caller_keys, caller_agents)
+        if caller_agent is None
+        or caller_agent.name != agent["name"]
+        or caller_agent.version not in (None, agent.get("version"))
+    )
+    if other_agents and not (litellm.drop_params is True or litellm_params.drop_params is True):
+        raise UnsupportedParamsError(
+            message=(
+                f"This model entry always calls Foundry agent {agent['name']!r}, so `{other_agents[0]}` "
+                "in the request cannot pick another agent or version. Remove it, call the model entry for that "
+                "agent, or set `drop_params: true` to drop it"
+            ),
+            llm_provider=LlmProviders.AZURE_AI.value,
+            model=model,
+        )
+    return agent
+
+
+_INPUT_ITEMS: Final = TypeAdapter(list[object])
+
+
+def _with_system_prompt(instructions: str, request_input: object) -> Sequence[object]:
+    conversation: Final = (
+        [{"type": "message", "role": "user", "content": request_input}]
+        if isinstance(request_input, str)
+        else _INPUT_ITEMS.validate_python(request_input)
+    )
+    return [{"type": "message", "role": "system", "content": instructions}, *conversation]
+
+
+class AzureAIAgentResponsesAPIConfig(AzureAIResponsesAPIConfig):
+    def should_fake_stream(
+        self, model: str | None, stream: bool | None, custom_llm_provider: str | None = None
+    ) -> bool:
+        return False
+
+    def get_supported_openai_params(self, model: str) -> list[str]:  # mutable-ok: override contract
+        inherited: Final[list[str]] = super().get_supported_openai_params(model)  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # parent returns a bare list  # mutable-ok: same
+        return [param for param in inherited if param not in _AGENT_OWNED_PARAMS]
+
+    def map_openai_params_for_deployment(
+        self,
+        response_api_optional_params: ResponsesAPIOptionalRequestParams,
+        model: str,
+        drop_params: bool,
+        *,
+        declared_reasoning_efforts: Collection[str],
+    ) -> Mapping[str, object]:
+        mapped: Final = super().map_openai_params_for_deployment(
+            response_api_optional_params, model, drop_params, declared_reasoning_efforts=declared_reasoning_efforts
+        )
+        resends_instructions: Final = bool(mapped.get("instructions")) and bool(mapped.get("previous_response_id"))
+        if resends_instructions and not drop_params:
+            raise UnsupportedParamsError(
+                message=(
+                    "azure_ai Foundry agents store a turn's system message with the conversation, so `instructions` "
+                    "with `previous_response_id` would add another copy each turn. Send `instructions` on the first "
+                    "turn only, or set `drop_params: true` to drop it"
+                ),
+                llm_provider=LlmProviders.AZURE_AI.value,
+                model=model,
+            )
+        dropped: Final = _AGENT_OWNED_PARAMS | (
+            frozenset({"instructions"}) if resends_instructions else frozenset[str]()
+        )
+        return {key: value for key, value in mapped.items() if key not in dropped}
+
+    def transform_responses_api_request(
+        self,
+        model: str,
+        input: str | ResponseInputParam,
+        response_api_optional_request_params: dict[str, object],  # mutable-ok: signature fixed by the override contract
+        litellm_params: GenericLiteLLMParams,
+        headers: dict[str, str],  # mutable-ok: same
+    ) -> dict[str, object]:  # mutable-ok: same
+        request: Final[dict[str, object]] = super().transform_responses_api_request(  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # parent returns a bare dict  # mutable-ok: same
+            model=model,
+            input=input,
+            response_api_optional_request_params=response_api_optional_request_params,
+            litellm_params=litellm_params,
+            headers=headers,
+        )
+        instructions: Final = request.get("instructions")
+        return {
+            **{key: value for key, value in request.items() if key not in ("model", "instructions")},
+            **(
+                {"input": _with_system_prompt(instructions, request.get("input"))}
+                if isinstance(instructions, str) and instructions
+                else {}
+            ),
+            "agent_reference": _deployment_agent(
+                model, _caller_agent_fields(litellm_params.model_extra or {}), litellm_params
+            ),
+        }
+
+    def transform_extra_body(
+        self,
+        extra_body: Mapping[str, object],
+        request: Mapping[str, object],
+        model: str,
+        litellm_params: GenericLiteLLMParams,
+    ) -> Mapping[str, object]:
+        passthrough: Final = {key: value for key, value in extra_body.items() if key not in _CALLER_AGENT_KEYS}
+        if len(passthrough) == len(extra_body):
+            return passthrough
+        return {
+            **passthrough,
+            "agent_reference": _deployment_agent(
+                model, (*_caller_agent_fields(request), *_caller_agent_fields(extra_body)), litellm_params
+            ),
+        }
+
+    def transform_compact_response_api_request(
+        self,
+        model: str,
+        input: str | ResponseInputParam,
+        response_api_optional_request_params: dict[str, object],  # mutable-ok: signature fixed by the override contract
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict[str, str],  # mutable-ok: same
+    ) -> NoReturn:
+        raise UnsupportedParamsError(
+            message=(
+                "azure_ai Foundry agents cannot be compacted, because the project's /responses/compact endpoint "
+                "takes a model deployment, not an agent. Call compact on a model entry for that deployment instead"
+            ),
+            llm_provider=LlmProviders.AZURE_AI.value,
+            model=model,
+        )

litellm/llms/base_llm/responses/transformation.py

@@ -72,6 +72,9 @@ class BaseResponsesAPIConfig(ABC):
     def supports_native_tool_search(self) -> bool:
         return False
 
+    def supports_streamed_retrieval(self) -> bool:
+        return self.custom_llm_provider in (LlmProviders.OPENAI, LlmProviders.AZURE, LlmProviders.AZURE_AI)
+
     def sign_request(
         self,
         headers: dict,
@@ -212,7 +215,7 @@ class BaseResponsesAPIConfig(ABC):
         api_base: str,
         litellm_params: GenericLiteLLMParams,
         headers: dict,
-    ) -> tuple[str, dict]:
+    ) -> tuple[str, dict[str, str]]:
         pass
 
     @abstractmethod
@@ -271,6 +274,9 @@ class BaseResponsesAPIConfig(ABC):
         """Returns True if litellm should fake a stream for the given model and stream value"""
         return False
 
+    def may_answer_stream_with_json(self) -> bool:
+        return False
+
     def supports_native_websocket(self) -> bool:
         """
         Returns True if the provider has a native WebSocket endpoint for Responses API.

litellm/llms/bigmodel/chat/transformation.py

@@ -0,0 +1,59 @@
+from collections.abc import Mapping
+from typing import Final
+
+from litellm.llms.zai.chat.transformation import ZAIChatConfig
+
+from ..common_utils import get_bigmodel_api_key, get_bigmodel_url, is_unsupported_disabled_thinking
+
+_SUPPORTED_PARAMS: Final = (
+    "max_tokens",
+    "max_completion_tokens",
+    "stream",
+    "stream_options",
+    "temperature",
+    "top_p",
+    "stop",
+    "tools",
+    "tool_choice",
+    "response_format",
+    "thinking",
+    "reasoning_effort",
+)
+
+
+class BigModelChatConfig(ZAIChatConfig):
+    @property
+    def custom_llm_provider(self) -> str:
+        return "bigmodel"
+
+    def get_openai_compatible_provider_info(self, api_base: str | None, api_key: str | None) -> tuple[str, str | None]:
+        return get_bigmodel_url(api_base, "chat"), get_bigmodel_api_key(api_key)
+
+    def get_supported_openai_params(self, model: str) -> list[str]:  # mutable-ok: inherited interface requires a list
+        return list(_SUPPORTED_PARAMS)
+
+    def map_openai_params(
+        self,
+        non_default_params: Mapping[str, object],
+        optional_params: Mapping[str, object],
+        model: str,
+        drop_params: bool,
+    ) -> dict[str, object]:  # mutable-ok: request pipeline adds parameters to the returned body
+        mapped: Final = {
+            **optional_params,
+            **{
+                ("max_tokens" if key == "max_completion_tokens" else key): value
+                for key, value in non_default_params.items()
+                if key in _SUPPORTED_PARAMS and key != "thinking"
+            },
+        }
+        if "thinking" not in non_default_params:
+            return mapped
+        thinking: Final = non_default_params["thinking"]
+        if is_unsupported_disabled_thinking(model, thinking):
+            return {
+                **mapped,
+                "reasoning_effort": mapped.get("reasoning_effort", "low"),
+                "extra_body": {"thinking": {"type": "enabled"}},
+            }
+        return {**mapped, "extra_body": {"thinking": thinking}}

litellm/llms/bigmodel/common_utils.py

@@ -0,0 +1,92 @@
+import re
+from collections.abc import Mapping
+from types import MappingProxyType
+from typing import Final, Literal
+from urllib.parse import urlsplit, urlunsplit
+
+import httpx
+from pydantic import JsonValue, TypeAdapter, ValidationError
+
+import litellm
+from litellm.exceptions import ContentPolicyViolationError, ContextWindowExceededError
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.secret_managers.main import get_secret_str
+
+BIGMODEL_API_BASE: Final = "https://open.bigmodel.cn"
+_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
+_BUSINESS_CODE_IN_ERROR: Final = re.compile(r"""['"]code['"]\s*:\s*['"]?(\d{4})\b""")
+_CODING_CHAT_PATH: Final = "/api/coding/paas/v4"
+_API_PATHS: Final = (
+    "/api/coding/paas/v4/chat/completions",
+    _CODING_CHAT_PATH,
+    "/api/paas/v4/chat/completions",
+    "/api/anthropic/v1/messages",
+    "/api/anthropic/v1",
+    "/api/v1/responses",
+    "/api/paas/v4",
+    "/api/anthropic",
+    "/api/v1",
+    "/api",
+)
+_ENDPOINT_PATHS: Final = MappingProxyType(
+    {
+        "chat": "/api/paas/v4",
+        "responses": "/api/v1/responses",
+        "messages": "/api/anthropic/v1/messages",
+    }
+)
+
+
+def get_bigmodel_api_key(api_key: str | None) -> str | None:
+    return api_key or get_secret_str("BIGMODEL_API_KEY")
+
+
+def get_bigmodel_url(api_base: str | None, endpoint: Literal["chat", "responses", "messages"]) -> str:
+    resolved_base: Final = api_base or get_secret_str("BIGMODEL_API_BASE") or BIGMODEL_API_BASE
+    parsed: Final = urlsplit(resolved_base)
+    path: Final = parsed.path.rstrip("/")
+    suffix: Final = next((candidate for candidate in _API_PATHS if path.endswith(candidate)), "")
+    endpoint_path: Final = (
+        _CODING_CHAT_PATH if endpoint == "chat" and suffix.startswith(_CODING_CHAT_PATH) else _ENDPOINT_PATHS[endpoint]
+    )
+    return urlunsplit(
+        (parsed.scheme, parsed.netloc, f"{path.removesuffix(suffix)}{endpoint_path}", parsed.query, parsed.fragment)
+    )
+
+
+def bigmodel_json_body(response: httpx.Response) -> Mapping[str, JsonValue]:
+    try:
+        return _JSON_OBJECT.validate_json(response.content)
+    except ValidationError:
+        raise BaseLLMException(message=response.text, status_code=response.status_code, headers=response.headers)
+
+
+def is_unsupported_disabled_thinking(model: str, thinking: object) -> bool:
+    return thinking == {"type": "disabled"} and litellm.utils.supports_factory(
+        model=model, custom_llm_provider="bigmodel", key="thinking_always_on"
+    )
+
+
+def bigmodel_business_error(
+    code: JsonValue, message: str, model: str, response: httpx.Response | None
+) -> ContextWindowExceededError | ContentPolicyViolationError | None:
+    match code:
+        case 1261 | "1261":
+            return ContextWindowExceededError(message=message, model=model, llm_provider="bigmodel", response=response)
+        case 1301 | "1301":
+            return ContentPolicyViolationError(message=message, model=model, llm_provider="bigmodel", response=response)
+        case _:
+            return None
+
+
+def raise_bigmodel_business_error(model: object, error_str: str, original_exception: object) -> None:
+    found: Final = _BUSINESS_CODE_IN_ERROR.search(error_str)
+    response: Final = getattr(original_exception, "response", None)
+    error: Final = bigmodel_business_error(
+        found.group(1) if found else None,
+        error_str,
+        str(model),
+        response if isinstance(response, httpx.Response) else None,
+    )
+    if error is not None:
+        raise error

litellm/llms/bigmodel/messages/transformation.py

@@ -0,0 +1,206 @@
+import json
+from collections.abc import AsyncIterator, Mapping, Sequence
+from typing import Final
+
+import httpx
+from openai._streaming import ServerSentEvent, SSEDecoder
+from pydantic import JsonValue, TypeAdapter, ValidationError
+
+from litellm.exceptions import AuthenticationError
+from litellm.litellm_core_utils.litellm_logging import Logging
+from litellm.llms.openai_like.messages.transformation import (
+    DEFAULT_ANTHROPIC_API_VERSION,
+    OpenAILikeAnthropicMessagesConfig,
+)
+from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse
+from litellm.types.router import GenericLiteLLMParams
+
+from ..common_utils import (
+    bigmodel_json_body,
+    get_bigmodel_api_key,
+    get_bigmodel_url,
+    is_unsupported_disabled_thinking,
+)
+
+_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
+_OUTPUT_CONFIG: Final = TypeAdapter(dict[str, object])
+
+
+def _with_lowest_effort_thinking(
+    params: Mapping[str, object],
+) -> dict[str, object]:  # mutable-ok: the parent transform takes a request dict
+    requested_output_config: Final = params.get("output_config")
+    output_config: Final = {
+        **({} if "reasoning_effort" in params else {"effort": "low"}),
+        **(
+            _OUTPUT_CONFIG.validate_python(requested_output_config) if isinstance(requested_output_config, dict) else {}
+        ),
+    }
+    return {**params, "thinking": {"type": "adaptive"}, **({"output_config": output_config} if output_config else {})}
+
+
+def _normalize_message(body: Mapping[str, JsonValue]) -> dict[str, JsonValue]:  # mutable-ok: native JSON response
+    usage: Final = body.get("usage")
+    if not isinstance(usage, dict):
+        return dict(body)
+    server_tools: Final = usage.get("server_tool_use")
+    if not isinstance(server_tools, dict) or "web_fetch_requests" in server_tools:
+        return dict(body)
+    normalized: Final[JsonValue] = {
+        **body,
+        "usage": {**usage, "server_tool_use": {"web_fetch_requests": 0, **server_tools}},
+    }
+    return normalized
+
+
+class _BigModelMessagesStream(httpx.AsyncByteStream):
+    def __init__(self, response: httpx.Response) -> None:
+        self._response = response
+
+    async def __aiter__(self) -> AsyncIterator[bytes]:
+        async for event in SSEDecoder().aiter_bytes(self._response.aiter_bytes()):
+            yield _normalize_event(event)
+
+    async def aclose(self) -> None:
+        await self._response.aclose()
+
+
+def _json_object(data: str) -> Mapping[str, JsonValue] | None:
+    try:
+        return _JSON_OBJECT.validate_json(data)
+    except ValidationError:
+        return None
+
+
+def _normalized_data(data: str) -> str:
+    body: Final = _json_object(data)
+    if body is None:
+        return data
+    message: Final = body.get("message")
+    normalized: Final[JsonValue] = (
+        {**body, "message": _normalize_message(message)}
+        if body.get("type") == "message_start" and isinstance(message, dict)
+        else _normalize_message(body)
+    )
+    return data if normalized == body else json.dumps(normalized, ensure_ascii=False)
+
+
+def _normalize_event(event: ServerSentEvent) -> bytes:
+    metadata: Final = "".join(
+        f"{name}: {value}\n"
+        for name, value in (("event", event.event), ("id", event.id), ("retry", event.retry))
+        if value is not None
+    )
+    data_lines: Final = "".join(f"data: {line}\n" for line in _normalized_data(event.data).split("\n"))
+    return f"{metadata}{data_lines}\n".encode()
+
+
+class BigModelMessagesConfig(OpenAILikeAnthropicMessagesConfig):
+    @property
+    def custom_llm_provider(self) -> str:
+        return "bigmodel"
+
+    def is_native_messages_route(self) -> bool:
+        return True
+
+    def should_strip_billing_metadata(self) -> bool:
+        return True
+
+    def transform_anthropic_messages_request(
+        self,
+        model: str,
+        messages: list[dict[str, object]],  # mutable-ok: matches dict-typed base signature
+        anthropic_messages_optional_request_params: dict[str, object],  # mutable-ok: matches dict-typed base signature
+        litellm_params: GenericLiteLLMParams,
+        headers: dict[str, str],  # mutable-ok: matches dict-typed base signature
+    ) -> dict[str, object]:  # mutable-ok: matches dict-typed base signature
+        request_params: Final = (
+            _with_lowest_effort_thinking(anthropic_messages_optional_request_params)
+            if is_unsupported_disabled_thinking(model, anthropic_messages_optional_request_params.get("thinking"))
+            else anthropic_messages_optional_request_params
+        )
+        return super().transform_anthropic_messages_request(  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # the parent transform returns a bare dict
+            model, messages, request_params, litellm_params, headers
+        )
+
+    def transform_anthropic_messages_response(
+        self, model: str, raw_response: httpx.Response, logging_obj: Logging
+    ) -> AnthropicMessagesResponse:
+        normalized: Final = httpx.Response(
+            raw_response.status_code,
+            json=_normalize_message(bigmodel_json_body(raw_response)),
+            request=raw_response.request,
+        )
+        return super().transform_anthropic_messages_response(model, normalized, logging_obj)
+
+    def get_async_streaming_response_iterator(
+        self,
+        model: str,
+        httpx_response: httpx.Response,
+        request_body: dict[str, object],  # mutable-ok: inherited native stream API takes a request dict
+        litellm_logging_obj: Logging,
+    ) -> AsyncIterator[bytes]:
+        normalized: Final = httpx.Response(
+            httpx_response.status_code,
+            headers={
+                key: value
+                for key, value in httpx_response.headers.items()
+                if key.lower() not in ("content-encoding", "content-length")
+            },
+            stream=_BigModelMessagesStream(httpx_response),
+            request=httpx_response.request,
+        )
+        return super().get_async_streaming_response_iterator(  # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType]  # native parent yields bytes but its iterator annotation omits the element type
+            model, normalized, request_body, litellm_logging_obj
+        )
+
+    @staticmethod
+    def _translate_reasoning_effort_to_anthropic(
+        model: str, optional_params: Mapping[str, object], max_tokens: int | None, custom_llm_provider: str
+    ) -> None:
+        return None
+
+    @staticmethod
+    def _translate_adaptive_effort_for_non_adaptive_model(
+        model: str, optional_params: Mapping[str, object], max_tokens: int | None, custom_llm_provider: str
+    ) -> None:
+        return None
+
+    @staticmethod
+    def _drop_incompatible_temperature_for_thinking(
+        model: str, optional_params: Mapping[str, object], custom_llm_provider: str
+    ) -> None:
+        return None
+
+    def validate_anthropic_messages_environment(
+        self,
+        headers: Mapping[str, str],
+        model: str,
+        messages: Sequence[object],
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        api_key: str | None = None,
+        api_base: str | None = None,
+    ) -> tuple[dict[str, str], str | None]:  # mutable-ok: HTTP handler merges forwarded headers
+        resolved_key: Final = get_bigmodel_api_key(api_key)
+        if resolved_key is None:
+            raise AuthenticationError(
+                message="Set BIGMODEL_API_KEY or pass api_key", llm_provider="bigmodel", model=model
+            )
+        return {
+            **{key: value for key, value in headers.items() if key.lower() not in ("x-api-key", "authorization")},
+            "x-api-key": resolved_key,
+            "anthropic-version": DEFAULT_ANTHROPIC_API_VERSION,
+            "content-type": "application/json",
+        }, api_base
+
+    def get_complete_url(
+        self,
+        api_base: str | None,
+        api_key: str | None,
+        model: str,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        stream: bool | None = None,
+    ) -> str:
+        return get_bigmodel_url(api_base, "messages")

litellm/llms/bigmodel/responses/transformation.py

@@ -0,0 +1,263 @@
+from collections.abc import Mapping
+from types import MappingProxyType
+from typing import Final
+
+import httpx
+from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError
+from typing_extensions import Never
+
+from litellm.exceptions import AuthenticationError, UnsupportedParamsError
+from litellm.litellm_core_utils.litellm_logging import Logging
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
+from litellm.types.llms.openai import (
+    ResponseInputParam,
+    ResponsesAPIOptionalRequestParams,
+    ResponsesAPIResponse,
+    ResponsesAPIStreamingResponse,
+)
+from litellm.types.router import GenericLiteLLMParams
+from litellm.types.utils import LlmProviders
+
+from ..common_utils import bigmodel_business_error, bigmodel_json_body, get_bigmodel_api_key, get_bigmodel_url
+
+_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
+_SUPPORTED_PARAMS: Final = (
+    "input",
+    "model",
+    "instructions",
+    "stream",
+    "temperature",
+    "top_p",
+    "max_output_tokens",
+    "tools",
+    "tool_choice",
+    "parallel_tool_calls",
+    "include",
+    "reasoning",
+    "text",
+    "prompt_cache_key",
+    "previous_response_id",
+    "store",
+    "extra_headers",
+    "extra_query",
+    "extra_body",
+    "timeout",
+)
+
+
+_STATUS_BY_BUSINESS_CODE: Final = MappingProxyType(
+    {
+        **dict.fromkeys((1000, 1001, 1003, 1005), 401),
+        **dict.fromkeys((1210, 1211, 1212, 1213, 1214, 1215, 1221, 1222, 1261, 1301), 400),
+        1220: 403,
+        **dict.fromkeys(
+            (1113, 1302, 1305, 1308, 1309, 1310, 1311, 1313, 1314, 1315, 1316, 1317, 1318, 1319, 1320, 1321),
+            429,
+        ),
+    }
+)
+
+
+class _RequestedTools(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+
+    tools: JsonValue = None
+    tool_choice: JsonValue = None
+    parallel_tool_calls: JsonValue = None
+
+
+def _requested_tools(logging_obj: Logging) -> _RequestedTools:
+    try:
+        return _RequestedTools.model_validate(logging_obj.model_call_details.get("optional_params") or {})
+    except ValidationError:
+        return _RequestedTools()
+
+
+def _envelope_status_code(code: JsonValue) -> int:
+    if not isinstance(code, int) or isinstance(code, bool):
+        return 500
+    if 400 <= code <= 599:
+        return code
+    return _STATUS_BY_BUSINESS_CODE.get(code, 500)
+
+
+def _normalize_output_item(item: JsonValue) -> JsonValue:
+    if not isinstance(item, dict) or item.get("type") not in ("message", "reasoning"):
+        return item
+    content: Final = item.get("content")
+    normalized: Final[JsonValue] = {"summary": [], **item} if item["type"] == "reasoning" else item
+    if not isinstance(content, dict):
+        return normalized
+    return {**normalized, "content": [content]}
+
+
+def _normalize_response(
+    body: Mapping[str, JsonValue],
+    requested: _RequestedTools,
+) -> dict[str, JsonValue]:  # mutable-ok: JSON serialization boundary
+    usage: Final = body.get("usage")
+    output: Final = body.get("output")
+    tools: Final = body.get("tools")
+    tool_choice: Final = body.get("tool_choice")
+    parallel_tool_calls: Final = body.get("parallel_tool_calls")
+    input_tokens: Final = usage.get("input_tokens") if isinstance(usage, dict) else None
+    output_tokens: Final = usage.get("output_tokens") if isinstance(usage, dict) else None
+    normalized_usage: Final = (
+        {
+            "total_tokens": input_tokens + output_tokens,
+            **usage,
+        }
+        if isinstance(usage, dict) and isinstance(input_tokens, int) and isinstance(output_tokens, int)
+        else usage
+    )
+    normalized: Final[JsonValue] = {
+        **body,
+        "parallel_tool_calls": (
+            requested.parallel_tool_calls is not False if parallel_tool_calls is None else parallel_tool_calls
+        ),
+        "tools": (requested.tools or []) if tools is None else tools,
+        "tool_choice": (requested.tool_choice or "auto") if tool_choice is None else tool_choice,
+        **({"usage": normalized_usage} if "usage" in body else {}),
+        **({"output": [_normalize_output_item(item) for item in output]} if isinstance(output, list) else {}),
+    }
+    return normalized
+
+
+def _normalize_stream_part(part: JsonValue) -> JsonValue:
+    if not isinstance(part, dict):
+        return part
+    if part.get("type") == "output_text":
+        return {
+            "annotations": [],
+            "logprobs": None,
+            **part,
+        }
+    if part.get("type") == "reasoning_text":
+        return {"reasoning": part.get("text", ""), **part}
+    return part
+
+
+class BigModelResponsesAPIConfig(OpenAIResponsesAPIConfig):
+    @property
+    def custom_llm_provider(self) -> LlmProviders:
+        return LlmProviders.BIGMODEL
+
+    def supports_native_file_search(self) -> bool:
+        return False
+
+    def supports_native_websocket(self) -> bool:
+        return False
+
+    def should_fake_stream(
+        self, model: str | None, stream: bool | None, custom_llm_provider: str | None = None
+    ) -> bool:
+        return False
+
+    def may_answer_stream_with_json(self) -> bool:
+        return True
+
+    def get_supported_openai_params(self, model: str) -> list[str]:  # mutable-ok: inherited API requires a list
+        return list(_SUPPORTED_PARAMS)
+
+    def map_openai_params(
+        self,
+        response_api_optional_params: ResponsesAPIOptionalRequestParams,
+        model: str,
+        drop_params: bool,
+    ) -> dict[str, object]:  # mutable-ok: request pipeline adds parameters to the returned body
+        return {key: value for key, value in response_api_optional_params.items() if key in _SUPPORTED_PARAMS}
+
+    def validate_environment(
+        self, headers: Mapping[str, str], model: str, litellm_params: GenericLiteLLMParams | None
+    ) -> dict[str, str]:  # mutable-ok: HTTP handler merges forwarded headers
+        resolved_key: Final = get_bigmodel_api_key(litellm_params.api_key if litellm_params else None)
+        if resolved_key is None:
+            raise AuthenticationError(
+                message="Set BIGMODEL_API_KEY or pass api_key", llm_provider="bigmodel", model=model
+            )
+        return {
+            **headers,
+            "Authorization": f"Bearer {resolved_key}",
+            "Content-Type": "application/json",
+        }
+
+    def get_complete_url(self, api_base: str | None, litellm_params: Mapping[str, object]) -> str:
+        return get_bigmodel_url(api_base, "responses")
+
+    def transform_response_api_response(
+        self, model: str, raw_response: httpx.Response, logging_obj: Logging
+    ) -> ResponsesAPIResponse:
+        body: Final = bigmodel_json_body(raw_response)
+        if body.get("success") is False:
+            message: Final = str(body.get("msg") or raw_response.text)
+            raise bigmodel_business_error(body.get("code"), message, model, None) or BaseLLMException(
+                message=message,
+                status_code=_envelope_status_code(body.get("code")),
+                headers=raw_response.headers,
+            )
+        normalized: Final = _normalize_response(body, _requested_tools(logging_obj))
+        normalized_response: Final = httpx.Response(
+            status_code=raw_response.status_code,
+            headers={
+                key: value
+                for key, value in raw_response.headers.items()
+                if key.lower() not in ("content-encoding", "content-length")
+            },
+            json=normalized,
+            request=raw_response.request,
+        )
+        return super().transform_response_api_response(model, normalized_response, logging_obj)
+
+    def transform_get_response_api_response(
+        self, raw_response: httpx.Response, logging_obj: Logging
+    ) -> ResponsesAPIResponse:
+        return self.transform_response_api_response("", raw_response, logging_obj)
+
+    def transform_streaming_response(
+        self, model: str, parsed_chunk: Mapping[str, object], logging_obj: Logging
+    ) -> ResponsesAPIStreamingResponse:
+        chunk: Final = _JSON_OBJECT.validate_python(parsed_chunk)
+        response: Final = chunk.get("response")
+        item: Final = chunk.get("item")
+        normalized: Final[JsonValue] = {
+            **chunk,
+            **(
+                {"response": _normalize_response(response, _requested_tools(logging_obj))}
+                if isinstance(response, dict)
+                else {}
+            ),
+            **({"item": _normalize_output_item(item)} if "item" in chunk else {}),
+        }
+        event_model: Final = self.get_event_model_class(str(normalized.get("type")))
+        match chunk.get("type"):
+            case "response.output_text.done":
+                return event_model.model_validate({"logprobs": [], **normalized})
+            case "response.content_part.done":
+                return event_model.model_validate({**normalized, "part": _normalize_stream_part(chunk.get("part"))})
+            case "error":
+                error: Final = chunk.get("error")
+                error_fields: Final = error if isinstance(error, dict) else chunk
+                normalized_error: Final[JsonValue] = {
+                    **normalized,
+                    "error": {"type": "api_error", **error_fields, "code": error_fields.get("code") or "unknown_error"},
+                }
+                return event_model.model_validate(normalized_error)
+            case _:
+                return event_model.model_validate(normalized)
+
+    def transform_cancel_response_api_request(
+        self, response_id: str, api_base: str, litellm_params: GenericLiteLLMParams, headers: Mapping[str, str]
+    ) -> Never:
+        raise UnsupportedParamsError(message="BigModel does not support cancelling responses", status_code=400)
+
+    def transform_compact_response_api_request(
+        self,
+        model: str,
+        input: str | ResponseInputParam,
+        response_api_optional_request_params: Mapping[str, object],
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: Mapping[str, str],
+    ) -> Never:
+        raise UnsupportedParamsError(message="BigModel does not support compacting responses", status_code=400)

litellm/llms/custom_httpx/http_handler.py

@@ -764,18 +764,23 @@ class AsyncHTTPHandler:
         follow_redirects: bool | None = None,
         timeout: float | httpx.Timeout | None = None,
         max_response_bytes: int | None = None,
+        stream: bool = False,
     ):
         # Set follow_redirects to UseClientDefault if None
         _follow_redirects: Final = follow_redirects if follow_redirects is not None else USE_CLIENT_DEFAULT
+        query: Final = httpx.QueryParams(params).merge(HTTPHandler.extract_query_params(url))
+        request_headers: Final = httpx.Headers(headers)
 
-        params = params or {}
-        params.update(HTTPHandler.extract_query_params(url))
+        if stream:
+            return await self._send_streaming_get(
+                url, params=query, headers=request_headers, follow_redirects=follow_redirects, timeout=timeout
+            )
 
         if max_response_bytes is not None:
             return await self._get_with_response_limit(
                 url,
-                params=httpx.QueryParams(params),
-                headers=httpx.Headers(headers),
+                params=query,
+                headers=request_headers,
                 max_bytes=max_response_bytes,
                 follow_redirects=self.client.follow_redirects if follow_redirects is None else follow_redirects,
                 timeout=self.client.timeout if timeout is None else httpx.Timeout(timeout),
@@ -783,13 +788,41 @@ class AsyncHTTPHandler:
 
         response: Final = await self.client.get(
             url,
-            params=params,
-            headers=headers,
+            params=query,
+            headers=request_headers,
             follow_redirects=_follow_redirects,
             timeout=timeout if timeout is not None else USE_CLIENT_DEFAULT,
         )
         return response
 
+    async def _send_streaming_get(
+        self,
+        url: str,
+        *,
+        params: httpx.QueryParams,
+        headers: httpx.Headers,
+        follow_redirects: bool | None,
+        timeout: float | httpx.Timeout | None,
+    ) -> httpx.Response:
+        start_time: Final = time.time()
+        request: Final = self.client.build_request(
+            "GET", url, params=params, headers=headers, timeout=USE_CLIENT_DEFAULT if timeout is None else timeout
+        )
+        try:
+            response: Final = await self.client.send(
+                request,
+                stream=True,
+                follow_redirects=USE_CLIENT_DEFAULT if follow_redirects is None else follow_redirects,
+            )
+            _anchor_handler_to(response, self)
+            response.raise_for_status()
+        except httpx.TimeoutException as e:
+            _raise_timeout(e, start_time)
+        except httpx.HTTPStatusError as e:
+            await _raise_masked_async_error(e, True)
+        else:
+            return response
+
     async def _get_with_response_limit(
         self,
         url: str,
@@ -1468,15 +1501,20 @@ class HTTPHandler:
         headers: Mapping[str, Any] | None = None,
         follow_redirects: bool | None = None,
         timeout: float | httpx.Timeout | None = None,
+        stream: bool = False,
     ):
         # Set follow_redirects to UseClientDefault if None
         _follow_redirects: Final = follow_redirects if follow_redirects is not None else USE_CLIENT_DEFAULT
-        params = params or {}
-        params.update(self.extract_query_params(url))
+        query: Final = httpx.QueryParams(params).merge(self.extract_query_params(url))
+
+        if stream:
+            return self._send_streaming_get(
+                url, params=query, headers=headers, follow_redirects=follow_redirects, timeout=timeout
+            )
 
         response: Final = self.client.get(
             url,
-            params=params,
+            params=query,
             headers=headers,
             follow_redirects=_follow_redirects,
             timeout=timeout if timeout is not None else USE_CLIENT_DEFAULT,
@@ -1484,6 +1522,34 @@ class HTTPHandler:
 
         return response
 
+    def _send_streaming_get(
+        self,
+        url: str,
+        *,
+        params: httpx.QueryParams,
+        headers: Mapping[str, str] | None,
+        follow_redirects: bool | None,
+        timeout: float | httpx.Timeout | None,
+    ) -> httpx.Response:
+        start_time: Final = time.time()
+        request: Final = self.client.build_request(
+            "GET", url, params=params, headers=headers, timeout=USE_CLIENT_DEFAULT if timeout is None else timeout
+        )
+        try:
+            response: Final = self.client.send(
+                request,
+                stream=True,
+                follow_redirects=USE_CLIENT_DEFAULT if follow_redirects is None else follow_redirects,
+            )
+            _anchor_handler_to(response, self)
+            response.raise_for_status()
+        except httpx.TimeoutException as e:
+            _raise_timeout(e, start_time)
+        except httpx.HTTPStatusError as e:
+            _raise_masked_sync_error(e, True)
+        else:
+            return response
+
     @staticmethod
     def extract_query_params(url: str) -> dict[str, str]:
         """

litellm/llms/custom_httpx/llm_http_handler.py

@@ -367,12 +367,37 @@ class _SignedBodyKwargs(TypedDict, total=False):
     json: ReadOnly[dict[str, object]]
 
 
+def _is_sized_json_body(response: httpx.Response) -> bool:
+    headers: Final = httpx.Headers(response.headers)
+    media_types: Final = (value.split(";")[0].strip().lower() for value in headers.get_list("content-type"))
+    return "content-length" in headers and "application/json" in media_types
+
+
+def _is_json_document(content: bytes) -> bool:
+    try:
+        return isinstance(json.loads(content), dict)
+    except ValueError:
+        return False
+
+
 def _signed_body_kwargs(*, signed_body: bytes | None, data: dict[str, object]) -> _SignedBodyKwargs:
     if signed_body is not None:
         return {"data": signed_body}
     return {"json": data}
 
 
+def _answered_json(response: httpx.Response) -> bool:
+    return "application/json" in response.headers.get("content-type", "").lower()
+
+
+def _stream_query_params(stream: bool, starting_after: int | None) -> Mapping[str, object]:
+    if not stream:
+        return {}
+    if starting_after is None:
+        return {"stream": "true"}
+    return {"stream": "true", "starting_after": starting_after}
+
+
 def _google_genai_streaming_hidden_params(
     *,
     api_base: str,
@@ -3025,6 +3050,17 @@ class BaseLLMHTTPHandler:
             },
         )
 
+        replay_as_stream: Final = partial(
+            MockResponsesAPIStreamingIterator,
+            model=model,
+            logging_obj=logging_obj,
+            responses_api_provider_config=responses_api_provider_config,
+            litellm_metadata=litellm_metadata,
+            custom_llm_provider=custom_llm_provider,
+            request_data=request_context,
+            call_type=CallTypes.responses.value,
+        )
+
         try:
             if is_stream_request:
                 response = sync_httpx_client.post(
@@ -3036,7 +3072,13 @@ class BaseLLMHTTPHandler:
                 )
                 if caller_requested_stream:
                     if fake_stream is True:
-                        return MockResponsesAPIStreamingIterator(
+                        return replay_as_stream(response=response)
+                    if not (
+                        responses_api_provider_config.may_answer_stream_with_json()
+                        and _is_sized_json_body(response)
+                        and _is_json_document(response.read())
+                    ):
+                        return SyncResponsesAPIStreamingIterator(
                             response=response,
                             model=model,
                             logging_obj=logging_obj,
@@ -3046,17 +3088,6 @@ class BaseLLMHTTPHandler:
                             request_data=request_context,
                             call_type=CallTypes.responses.value,
                         )
-
-                    return SyncResponsesAPIStreamingIterator(
-                        response=response,
-                        model=model,
-                        logging_obj=logging_obj,
-                        responses_api_provider_config=responses_api_provider_config,
-                        litellm_metadata=litellm_metadata,
-                        custom_llm_provider=custom_llm_provider,
-                        request_data=request_context,
-                        call_type=CallTypes.responses.value,
-                    )
                 response.read()
             else:
                 response = sync_httpx_client.post(
@@ -3071,6 +3102,9 @@ class BaseLLMHTTPHandler:
                 provider_config=responses_api_provider_config,
             )
 
+        if is_stream_request and caller_requested_stream:
+            return replay_as_stream(response=response)
+
         initial_response: Final = responses_api_provider_config.transform_response_api_response(
             model=model,
             raw_response=response,
@@ -3226,6 +3260,17 @@ class BaseLLMHTTPHandler:
             },
         )
 
+        replay_as_stream: Final = partial(
+            MockResponsesAPIStreamingIterator,
+            model=model,
+            logging_obj=logging_obj,
+            responses_api_provider_config=responses_api_provider_config,
+            litellm_metadata=litellm_metadata,
+            custom_llm_provider=custom_llm_provider,
+            request_data=request_context,
+            call_type=CallTypes.responses.value,
+        )
+
         try:
             if is_stream_request:
                 response = await async_httpx_client.post(
@@ -3239,7 +3284,13 @@ class BaseLLMHTTPHandler:
 
                 if caller_requested_stream:
                     if fake_stream is True:
-                        return MockResponsesAPIStreamingIterator(
+                        return replay_as_stream(response=response)
+                    if not (
+                        responses_api_provider_config.may_answer_stream_with_json()
+                        and _is_sized_json_body(response)
+                        and _is_json_document(await response.aread())
+                    ):
+                        return ResponsesAPIStreamingIterator(
                             response=response,
                             model=model,
                             logging_obj=logging_obj,
@@ -3249,17 +3300,6 @@ class BaseLLMHTTPHandler:
                             request_data=request_context,
                             call_type=CallTypes.responses.value,
                         )
-
-                    return ResponsesAPIStreamingIterator(
-                        response=response,
-                        model=model,
-                        logging_obj=logging_obj,
-                        responses_api_provider_config=responses_api_provider_config,
-                        litellm_metadata=litellm_metadata,
-                        custom_llm_provider=custom_llm_provider,
-                        request_data=request_context,
-                        call_type=CallTypes.responses.value,
-                    )
                 await response.aread()
             else:
                 response = await async_httpx_client.post(
@@ -3276,6 +3316,9 @@ class BaseLLMHTTPHandler:
                 provider_config=responses_api_provider_config,
             )
 
+        if is_stream_request and caller_requested_stream:
+            return replay_as_stream(response=response)
+
         initial_response: Final = responses_api_provider_config.transform_response_api_response(
             model=model,
             raw_response=response,
@@ -3493,12 +3536,20 @@ class BaseLLMHTTPHandler:
         logging_obj: LiteLLMLoggingObj,
         custom_llm_provider: str | None = None,
         extra_headers: dict[str, object] | None = None,
+        extra_query: Mapping[str, object] | None = None,
         extra_body: dict[str, object] | None = None,
+        stream: bool = False,
+        starting_after: int | None = None,
+        litellm_metadata: Mapping[str, object] = MappingProxyType({}),
         timeout: float | httpx.Timeout | None = None,
         client: HTTPHandler | AsyncHTTPHandler | None = None,
         _is_async: bool = False,
         shared_session: Optional["ClientSession"] = None,
-    ) -> ResponsesAPIResponse | Coroutine[object, object, ResponsesAPIResponse]:
+    ) -> (
+        ResponsesAPIResponse
+        | SyncResponsesAPIStreamingIterator
+        | Coroutine[object, object, ResponsesAPIResponse | ResponsesAPIStreamingIterator]
+    ):
         """
         Get a response by ID
         Uses GET /v1/responses/{response_id} endpoint in the responses API
@@ -3511,7 +3562,11 @@ class BaseLLMHTTPHandler:
                 logging_obj=logging_obj,
                 custom_llm_provider=custom_llm_provider,
                 extra_headers=extra_headers,
+                extra_query=extra_query,
                 extra_body=extra_body,
+                stream=stream,
+                starting_after=starting_after,
+                litellm_metadata=litellm_metadata,
                 timeout=timeout,
                 client=client,
                 shared_session=shared_session,
@@ -3540,20 +3595,23 @@ class BaseLLMHTTPHandler:
             litellm_params=litellm_params,
             headers=headers,
         )
+        query: Final = {**data, **(extra_query or {}), **_stream_query_params(stream, starting_after)}
 
         ## LOGGING
         logging_obj.pre_call(
             input="",
             api_key="",
             additional_args={
-                "complete_input_dict": data,
+                "complete_input_dict": query,
                 "api_base": api_base,
                 "headers": headers,
             },
         )
 
         try:
-            response: Final = sync_httpx_client.get(url=url, headers=headers, params=data)
+            response: Final = sync_httpx_client.get(
+                url=url, headers=headers, params=query, stream=stream, timeout=timeout if stream else None
+            )
             response.raise_for_status()
         except Exception as e:
             raise self._handle_error(
@@ -3561,6 +3619,20 @@ class BaseLLMHTTPHandler:
                 provider_config=responses_api_provider_config,
             )
 
+        if stream and not _answered_json(response):
+            return SyncResponsesAPIStreamingIterator(
+                response=response,
+                model="",
+                logging_obj=logging_obj,
+                responses_api_provider_config=responses_api_provider_config,
+                litellm_metadata=dict(litellm_metadata),
+                custom_llm_provider=custom_llm_provider,
+                request_data={"litellm_params": dict(litellm_params)},
+                replays_stored_stream=True,
+            )
+
+        if stream:
+            response.read()
         return responses_api_provider_config.transform_get_response_api_response(
             raw_response=response,
             logging_obj=logging_obj,
@@ -3574,11 +3646,15 @@ class BaseLLMHTTPHandler:
         logging_obj: LiteLLMLoggingObj,
         custom_llm_provider: str | None = None,
         extra_headers: dict[str, object] | None = None,
+        extra_query: Mapping[str, object] | None = None,
         extra_body: dict[str, object] | None = None,
+        stream: bool = False,
+        starting_after: int | None = None,
+        litellm_metadata: Mapping[str, object] = MappingProxyType({}),
         timeout: float | httpx.Timeout | None = None,
         client: HTTPHandler | AsyncHTTPHandler | None = None,
         shared_session: Optional["ClientSession"] = None,
-    ) -> ResponsesAPIResponse:
+    ) -> ResponsesAPIResponse | ResponsesAPIStreamingIterator:
         """
         Async version of get_responses
         """
@@ -3616,20 +3692,23 @@ class BaseLLMHTTPHandler:
             litellm_params=litellm_params,
             headers=headers,
         )
+        query: Final = {**data, **(extra_query or {}), **_stream_query_params(stream, starting_after)}
 
         ## LOGGING
         logging_obj.pre_call(
             input="",
             api_key="",
             additional_args={
-                "complete_input_dict": data,
+                "complete_input_dict": query,
                 "api_base": api_base,
                 "headers": headers,
             },
         )
 
         try:
-            response: Final = await async_httpx_client.get(url=url, headers=headers, params=data)
+            response: Final = await async_httpx_client.get(
+                url=url, headers=headers, params=query, stream=stream, timeout=timeout if stream else None
+            )
             response.raise_for_status()
         except Exception as e:
             verbose_logger.debug("Error retrieving response: %s", e)
@@ -3638,6 +3717,20 @@ class BaseLLMHTTPHandler:
                 provider_config=responses_api_provider_config,
             )
 
+        if stream and not _answered_json(response):
+            return ResponsesAPIStreamingIterator(
+                response=response,
+                model="",
+                logging_obj=logging_obj,
+                responses_api_provider_config=responses_api_provider_config,
+                litellm_metadata=dict(litellm_metadata),
+                custom_llm_provider=custom_llm_provider,
+                request_data={"litellm_params": dict(litellm_params)},
+                replays_stored_stream=True,
+            )
+
+        if stream:
+            await response.aread()
         return responses_api_provider_config.transform_get_response_api_response(
             raw_response=response,
             logging_obj=logging_obj,

litellm/llms/manus/responses/transformation.py

@@ -241,7 +241,7 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig):
         api_base: str,
         litellm_params: GenericLiteLLMParams,
         headers: dict,
-    ) -> tuple[str, dict]:
+    ) -> tuple[str, dict[str, str]]:
         """
         Transform the get response API request into a URL and data.
 
@@ -252,7 +252,7 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig):
         """
         encoded_response_id: Final = encode_url_path_segment(response_id, field_name="response_id")
         url: Final = f"{api_base}/{encoded_response_id}"
-        data: Final[dict] = {}
+        data: Final[dict[str, str]] = {}
         return url, data
 
     def transform_get_response_api_response(

litellm/llms/openai/common_utils.py

@@ -31,7 +31,7 @@ if TYPE_CHECKING:
 import litellm
 from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
 from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object
-from litellm.litellm_core_utils.token_counter import token_counter
+from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.custom_httpx.http_handler import (
     _DEFAULT_TTL_FOR_HTTPX_CLIENTS,
@@ -242,7 +242,7 @@ async def _aiter_once(chunk: ChatCompletionChunk) -> AsyncIterator[ChatCompletio
 
 
 def build_output_token_limit_response(
-    e: openai.BadRequestError, data: Mapping[str, object], is_async: bool
+    e: openai.BadRequestError, data: Mapping[str, object], is_async: bool, base_model: str | None
 ) -> tuple[httpx.Headers, ChatCompletion | Iterator[ChatCompletionChunk] | AsyncIterator[ChatCompletionChunk]]:
     """Synthesize the length-truncated response the provider itself returns for slightly larger output budgets.
 
@@ -251,7 +251,11 @@ def build_output_token_limit_response(
     """
     model: Final[str] = str(data.get("model", ""))
     messages: Final = data.get("messages")
-    prompt_tokens: Final = token_counter(model=model, messages=messages) if isinstance(messages, list) else 0
+    prompt_tokens: Final = (
+        token_counter(model=usage_tokenizer_model(model, base_model), messages=messages)
+        if isinstance(messages, list)
+        else 0
+    )
     if not data.get("stream"):
         return e.response.headers, _output_token_limit_completion(model, prompt_tokens)
     chunk: Final = _output_token_limit_chunk(model)

litellm/llms/openai/openai.py

@@ -44,6 +44,7 @@ from litellm.utils import (
     CustomStreamWrapper,
     ProviderConfigManager,
     convert_to_model_response_object,
+    get_base_model_from_metadata,
 )
 
 from ...types.llms.openai import *
@@ -557,7 +558,12 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         except openai.BadRequestError as e:
             if not is_output_token_limit_error(e):
                 raise
-            return build_output_token_limit_response(e=e, data=data, is_async=True)
+            return build_output_token_limit_response(
+                e=e,
+                data=data,
+                is_async=True,
+                base_model=get_base_model_from_metadata(logging_obj.model_call_details),
+            )
         except Exception as e:
             raise e
 
@@ -604,7 +610,12 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
         except openai.BadRequestError as e:
             if not is_output_token_limit_error(e):
                 raise
-            return build_output_token_limit_response(e=e, data=data, is_async=False)
+            return build_output_token_limit_response(
+                e=e,
+                data=data,
+                is_async=False,
+                base_model=get_base_model_from_metadata(logging_obj.model_call_details),
+            )
         except Exception as e:
             if raw_response is not None:
                 raise Exception(

litellm/llms/openai/realtime/handler.py

@@ -4,11 +4,17 @@ This file contains the calling OpenAI's `/v1/realtime` endpoint.
 This requires websockets, and is currently only supported on LiteLLM Proxy.
 """
 
+import asyncio
 import ssl
-from typing import TYPE_CHECKING, Final, cast
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Final, Literal, cast
+
+from pydantic import TypeAdapter
 
 from litellm._logging import redact_string, verbose_logger
 from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
+from litellm.exceptions import AuthenticationError
+from litellm.responses.streaming_iterator import map_stream_error_to_exception
 from litellm.types.realtime import RealtimeQueryParams
 
 from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
@@ -25,6 +31,54 @@ from ..openai import OpenAIChatCompletion
 
 if TYPE_CHECKING:
     from fastapi import WebSocket
+    from openai.types.realtime import RealtimeError
+    from websockets.asyncio.client import ClientConnection
+
+_SERVER_EVENT_FIELDS: Final = TypeAdapter(Mapping[str, object])
+_CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL: Final = "invalid_model"
+_CREDENTIALS_REJECTED: Final = "invalid_api_key"
+
+
+def first_event_error(first_event: str | bytes) -> "RealtimeError | None":
+    event: Final = _SERVER_EVENT_FIELDS.validate_json(first_event)
+    if event.get("type") != "error":
+        return None
+    from openai.types.realtime import RealtimeErrorEvent
+
+    return RealtimeErrorEvent.model_validate(event).error
+
+
+def _confirm_first_event(
+    first_event: str | bytes, model: str, sent_credentials: bool, model_listed_outside_realtime: bool
+) -> Literal[True]:
+    error: Final = first_event_error(first_event)
+    if error is None or (
+        error.code == _CREDENTIALS_ACCEPTED_FOR_A_NON_REALTIME_MODEL and not model_listed_outside_realtime
+    ):
+        return True
+    if error.code == _CREDENTIALS_REJECTED or not sent_credentials:
+        raise AuthenticationError(message=error.message, llm_provider="openai", model=model)
+    raise map_stream_error_to_exception(error, model=model, custom_llm_provider="openai")
+
+
+async def confirm_session_started(
+    connection: "ClientConnection", model: str, sent_credentials: bool, model_listed_outside_realtime: bool
+) -> Literal[True]:
+    return _confirm_first_event(await connection.recv(), model, sent_credentials, model_listed_outside_realtime)
+
+
+async def confirm_gateway_session_started(
+    connection: "ClientConnection",
+    model: str,
+    sent_credentials: bool,
+    model_listed_outside_realtime: bool,
+    first_event_wait_seconds: float,
+) -> Literal[True]:
+    try:
+        first_event: Final = await asyncio.wait_for(connection.recv(), first_event_wait_seconds)
+    except asyncio.TimeoutError:
+        return True
+    return _confirm_first_event(first_event, model, sent_credentials, model_listed_outside_realtime)
 
 
 class OpenAIRealtime(OpenAIChatCompletion):

litellm/llms/openai/responses/count_tokens/transformation.py

@@ -22,7 +22,7 @@ from litellm.llms.anthropic.messages.responses_adapters.transformation import (
 )
 from litellm.responses.utils import ResponsesAPIRequestUtils
 from litellm.types.llms.anthropic import AnthropicMessagesRequest, AnthropicOutputSchema
-from litellm.types.llms.openai import ResponsesAPIRequestParams
+from litellm.types.llms.openai import ResponsesAPIRequestParams, replayable_responses_input
 from litellm.types.utils import all_litellm_params
 
 
@@ -387,10 +387,13 @@ def _forwards(field: str, policy: DropPolicy) -> bool:
     return field in _COUNT_REQUEST_FIELDS or (not policy.drop_params and field in _INFERENCE_REQUEST_FIELDS)
 
 
-def _restored(field: str, value: object) -> object:
+def _restored(field: str, value: object, policy: DropPolicy) -> object:
     match field, value:
         case "input", _:
-            return ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(copy.deepcopy(value))
+            return replayable_responses_input(
+                ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(copy.deepcopy(value)),
+                drops_set_created_by=policy.drop_params,
+            )
         case "previous_response_id", str():
             return ResponsesAPIRequestUtils.decode_previous_response_id_to_original_previous_response_id(value)
         case _:
@@ -398,5 +401,7 @@ def _restored(field: str, value: object) -> object:
 
 
 def native_count_request(request: Mapping[str, object], model: str, policy: DropPolicy) -> Mapping[str, object]:
-    forwarded: Final = {field: _restored(field, value) for field, value in request.items() if _forwards(field, policy)}
+    forwarded: Final = {
+        field: _restored(field, value, policy) for field, value in request.items() if _forwards(field, policy)
+    }
     return {**forwarded, "model": model}

litellm/llms/openai/responses/guardrail_translation/handler.py

@@ -32,11 +32,11 @@ import copy
 import json
 import time
 import uuid
-from collections.abc import Mapping, MutableSequence, Sequence
+from collections.abc import Iterable, Mapping, MutableSequence, Sequence
 from dataclasses import dataclass
-from itertools import accumulate, chain, repeat
+from itertools import accumulate, chain, groupby, repeat
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, Union, cast
+from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, TypeVar, Union, cast
 
 from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
 from typing_extensions import ReadOnly, TypedDict
@@ -55,7 +55,9 @@ from litellm.llms.base_llm.guardrail_translation.base_translation import (
 )
 from litellm.llms.base_llm.guardrail_translation.utils import (
     blocked_responses_stream_usage,
+    effective_scan_only_tool_results_for_guardrail,
     effective_skip_system_message_for_guardrail,
+    effective_skip_tool_message_for_guardrail,
     list_field,
     merge_guardrailed_scoped_messages,
     role_out_of_guardrail_scope,
@@ -65,7 +67,10 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
     stream_item_items,
     unappliable_request_rewrite,
 )
-from litellm.llms.openai.responses.guardrail_translation.tool_merge import merge_guardrailed_tools
+from litellm.llms.openai.responses.guardrail_translation.tool_merge import (
+    append_returned_tools,
+    merge_guardrailed_tools,
+)
 from litellm.responses.litellm_completion_transformation.compaction_items import (
     compaction_summary,
     decode_compaction_item,
@@ -74,7 +79,9 @@ from litellm.responses.litellm_completion_transformation.compaction_items import
 from litellm.responses.litellm_completion_transformation.custom_tools import without_malformed_tool_flags
 from litellm.responses.litellm_completion_transformation.tool_search import is_server_tool_search_tool
 from litellm.responses.litellm_completion_transformation.transformation import (
+    TOOL_CALL_OUTPUT_ITEM_TYPES,
     LiteLLMCompletionResponsesConfig,
+    tool_output_to_chat_content,
 )
 from litellm.types.llms.base import LiteLLMBaseModel
 from litellm.types.llms.openai import (
@@ -270,6 +277,7 @@ _TOOL_CALL_PAYLOAD_EVENT_TYPES: Final = _TOOL_CALL_PAYLOAD_DELTA_EVENT_TYPES | f
 )
 _OUTPUT_ITEM_EVENT_TYPES: Final = frozenset({"response.output_item.added", "response.output_item.done"})
 _OUTPUT_TEXT_EVENT_TYPES: Final = frozenset({"response.output_text.delta", "response.output_text.done"})
+_TOOL_OUTPUT_TEXT_PART_TYPES: Final = frozenset({"input_text", "output_text", "text"})
 _PATCHABLE_ITEM_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
     {"function_call_output": "output", "custom_tool_call_output": "output", "message": "content"}
 )
@@ -519,10 +527,109 @@ def scannable_instructions(data: Mapping[str, object], *, skip_system: bool = Fa
 
 
 def _input_item_role(item: object) -> str:
+    if stream_item_field(item, "type") in TOOL_CALL_OUTPUT_ITEM_TYPES:
+        return "tool"
     role: Final = item.get("role") if isinstance(item, Mapping) else None
     return role.lower() if isinstance(role, str) else ""
 
 
+def _scanned_text_field(item: Mapping[str, object]) -> str:
+    return "output" if item.get("type") in TOOL_CALL_OUTPUT_ITEM_TYPES else "content"
+
+
+def _tool_output_part_text(part: object) -> str | None:
+    if not isinstance(part, dict):
+        return None
+    fields: Final = _item_fields_adapter.validate_python(part)
+    text: Final = fields.get("text")
+    return text if fields.get("type") in _TOOL_OUTPUT_TEXT_PART_TYPES and isinstance(text, str) else None
+
+
+def tool_output_texts(item: object) -> tuple[tuple[int | None, str], ...]:
+    if not stream_item_field(item, "call_id") or stream_item_field(item, "type") == "computer_call_output":
+        return ()
+    parts: Final = stream_item_items(item, "output")
+    part_texts: Final = tuple(
+        (index, text) for index, part in enumerate(parts) if (text := _tool_output_part_text(part))
+    )
+    if part_texts:
+        return part_texts
+    tool_content: Final = tool_output_to_chat_content(stream_item_field(item, "output"))
+    return ((None, tool_content),) if isinstance(tool_content, str) and tool_content else ()
+
+
+_Grouped: Final = TypeVar("_Grouped")
+
+
+def _pair_call_id(pair: tuple[str, object]) -> str:
+    return pair[0]
+
+
+def _grouped_by_call_id(pairs: Iterable[tuple[str, _Grouped]]) -> Mapping[str, tuple[_Grouped, ...]]:
+    ordered: Final = sorted(pairs, key=_pair_call_id)
+    return MappingProxyType(
+        {call_id: tuple(value for _, value in group) for call_id, group in groupby(ordered, key=_pair_call_id)}
+    )
+
+
+def _texts_by_tool_row(
+    messages: Sequence[Mapping[str, object]], input_data: "str | ResponseInputParam"
+) -> Mapping[int, tuple[str, ...]]:
+    if isinstance(input_data, str):
+        return MappingProxyType({})
+    output_texts_by_call_id: Final = _grouped_by_call_id(
+        (str(stream_item_field(item, "call_id")), tuple(text for _, text in tool_output_texts(item)))
+        for item in input_data
+        if stream_item_field(item, "type") in TOOL_CALL_OUTPUT_ITEM_TYPES
+    )
+    row_indices_by_call_id: Final = _grouped_by_call_id(
+        (str(message.get("tool_call_id")), index)
+        for index, message in enumerate(messages)
+        if message.get("role") == "tool"
+    )
+    return MappingProxyType(
+        dict(
+            chain.from_iterable(
+                zip(indices, output_texts_by_call_id[call_id], strict=True)
+                for call_id, indices in row_indices_by_call_id.items()
+                if len(indices) == len(output_texts_by_call_id.get(call_id, ()))
+            )
+        )
+    )
+
+
+def _is_string_tool_row(message: Mapping[str, object]) -> bool:
+    return message.get("role") == "tool" and isinstance(message.get("content"), str)
+
+
+def _split_into_text_parts(message: AllMessageValues, texts: tuple[str, ...]) -> AllMessageValues:
+    if not _is_string_tool_row(message) or len(texts) < 2:
+        return message
+    return cast(  # cast-ok: a tool row whose content is its text parts as chat text blocks
+        "AllMessageValues", {**message, "content": [{"type": "text", "text": text} for text in texts]}
+    )
+
+
+def _with_split_rows_restored(
+    rows: Sequence[AllMessageValues], split_rows: Sequence[tuple[AllMessageValues, AllMessageValues]]
+) -> Sequence[AllMessageValues]:
+    if not isinstance(rows, list):
+        return rows
+    return [next((original for split, original in split_rows if row == split), row) for row in rows]
+
+
+def _is_textless_tool_row(message: Mapping[str, object], texts: tuple[str, ...] | None) -> bool:
+    return _is_string_tool_row(message) and (not message.get("content") or texts == ())
+
+
+def _rewritten_whole_text(item: object, field: str, guardrail_response: str) -> object | None:
+    if isinstance(stream_item_field(item, field), str):
+        return guardrail_response
+    if tool_output_texts(item) == ((None, guardrail_response),):
+        return None
+    return guardrail_response
+
+
 def _patched_request_fields(
     raw_input: object,
     instructions: object,
@@ -636,17 +743,28 @@ class OpenAIResponsesHandler(BaseTranslation):
 
         Handles both string input and list of message objects.
         """
-        input_data: Final[str | ResponseInputParam | None] = data.get("input")
-        if not isinstance(input_data, (str, list)):
+        raw_input: Final[str | ResponseInputParam | None] = data.get("input")
+        if not isinstance(raw_input, (str, list)):
             return data
+        input_data: Final = cast("str | ResponseInputParam", raw_input)  # cast-ok: fields read via stream_item_field
         skip_system: Final = effective_skip_system_message_for_guardrail(guardrail_to_apply)
+        skip_tool: Final = effective_skip_tool_message_for_guardrail(guardrail_to_apply)
+        scan_only: Final = effective_scan_only_tool_results_for_guardrail(guardrail_to_apply)
         structured_messages: Final = self.get_structured_messages(data)
-        scoped_indices: Final = scoped_structured_message_indices(
-            structured_messages or [], scan_only_tool_results=False, skip_system=skip_system, skip_tool=False
+        all_messages: Final = structured_messages or []
+        texts_by_tool_row: Final = _texts_by_tool_row(all_messages, input_data)
+        scoped_indices: Final = tuple(
+            index
+            for index in scoped_structured_message_indices(
+                all_messages, scan_only_tool_results=scan_only, skip_system=skip_system, skip_tool=skip_tool
+            )
+            if not _is_textless_tool_row(all_messages[index], texts_by_tool_row.get(index))
         )
-        scoped_structured_messages: Final = (
-            [structured_messages[index] for index in scoped_indices] if structured_messages else None
+        split_rows: Final = tuple(
+            (_split_into_text_parts(all_messages[index], texts_by_tool_row.get(index, ())), all_messages[index])
+            for index in scoped_indices
         )
+        scoped_structured_messages: Final = [split for split, _ in split_rows] if structured_messages else None
         raw_tools: Final = data.get("tools")
         original_tools: Final[tuple[Mapping[str, object], ...]] = (
             tuple(raw_tools) if isinstance(raw_tools, list) else ()
@@ -661,7 +779,12 @@ class OpenAIResponsesHandler(BaseTranslation):
             )
         )
         extracted: Final = self._extract_guardrail_inputs(
-            data, input_data, flattened_tool_groups, skip_system=skip_system
+            data,
+            input_data,
+            () if scan_only else flattened_tool_groups,
+            skip_system=skip_system,
+            skip_tool=skip_tool,
+            scan_only=scan_only,
         )
         if not extracted.inputs.get("texts"):
             return data
@@ -674,13 +797,14 @@ class OpenAIResponsesHandler(BaseTranslation):
             logging_obj=litellm_logging_obj,
         )
         self._apply_guardrailed_tools_to_data(
-            data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools")
+            data, original_tools, flattened_tool_groups, guardrailed_inputs.get("tools"), scan_only=scan_only
         )
         written_back: Final = self._written_back_request_fields(
             data,
             structured_messages or (),
             scoped_indices,
             scoped_structured_messages,
+            tuple((split, original) for split, original in split_rows if split is not original),
             guardrail_to_apply,
             guardrailed_inputs,
         )
@@ -730,8 +854,10 @@ class OpenAIResponsesHandler(BaseTranslation):
         flattened_tool_groups: Sequence[Sequence[Mapping[str, object]]],
         *,
         skip_system: bool = False,
+        skip_tool: bool = False,
+        scan_only: bool = False,
     ) -> _ExtractedInputs:
-        instructions: Final = scannable_instructions(data, skip_system=skip_system)
+        instructions: Final = scannable_instructions(data, skip_system=skip_system or scan_only)
         texts_to_check: Final[list[str]] = [] if instructions is None else [instructions]
         images_to_check: Final[list[str]] = []
         task_mappings: Final[list[tuple[int, int | None]]] = []
@@ -745,11 +871,15 @@ class OpenAIResponsesHandler(BaseTranslation):
             )
         )
         if isinstance(input_data, str):
-            texts_to_check.append(input_data)
+            if not scan_only:
+                texts_to_check.append(input_data)
         else:
             for msg_idx, message in enumerate(input_data):
                 if role_out_of_guardrail_scope(
-                    _input_item_role(message), skip_system_message=skip_system, skip_tool_message=False
+                    _input_item_role(message),
+                    skip_system_message=skip_system,
+                    skip_tool_message=skip_tool,
+                    scan_only_tool_results=scan_only,
                 ):
                     continue
                 self._extract_input_text_and_images(
@@ -775,12 +905,14 @@ class OpenAIResponsesHandler(BaseTranslation):
         structured_messages: Sequence[AllMessageValues],
         scoped_indices: Sequence[int],
         scoped_structured_messages: Sequence[AllMessageValues] | None,
+        split_rows: Sequence[tuple[AllMessageValues, AllMessageValues]],
         guardrail_to_apply: "CustomGuardrail",
         guardrailed_inputs: GenericGuardrailAPIInputs,
     ) -> _RequestFields | None:
-        guardrailed: Final = guardrailed_inputs.get("structured_messages")
-        if guardrailed is None or guardrailed is scoped_structured_messages:
+        returned: Final = guardrailed_inputs.get("structured_messages")
+        if returned is None or returned is scoped_structured_messages:
             return None
+        guardrailed: Final = _with_split_rows_restored(returned, split_rows)
         covers_full_request: Final = len(scoped_indices) == len(structured_messages) or (
             guardrail_to_apply.structured_messages_cover_full_request() and len(guardrailed) == len(structured_messages)
         )
@@ -814,11 +946,14 @@ class OpenAIResponsesHandler(BaseTranslation):
         original_tools: Sequence[Mapping[str, object]],
         flattened_tool_groups: Sequence[Sequence[Mapping[str, object]]],
         guardrailed_tools: Sequence[ChatCompletionToolParam] | None,
+        *,
+        scan_only: bool,
     ) -> None:
         if guardrailed_tools is None:
             return
+        merge: Final = append_returned_tools if scan_only else merge_guardrailed_tools
         data["tools"] = list(  # rebind-ok: in-place request rewrite
-            merge_guardrailed_tools(original_tools, flattened_tool_groups, guardrailed_tools)
+            merge(original_tools, flattened_tool_groups, guardrailed_tools)
         )
 
     def _extract_input_text_and_images(
@@ -839,6 +974,11 @@ class OpenAIResponsesHandler(BaseTranslation):
             texts_to_check.append(summary)
             task_mappings.append((msg_idx, None))
             return
+        if message.get("type") in TOOL_CALL_OUTPUT_ITEM_TYPES:
+            for content_idx, text in tool_output_texts(message):
+                texts_to_check.append(text)
+                task_mappings.append((msg_idx, content_idx))
+            return
         content: Final = message.get("content", None)
         if content is None:
             return
@@ -890,18 +1030,20 @@ class OpenAIResponsesHandler(BaseTranslation):
                 )
                 continue
 
-            content = messages[msg_idx].get("content", None)
+            field = _scanned_text_field(messages[msg_idx])
+            content = messages[msg_idx].get(field, None)
             if content is None:
                 continue
 
-            if isinstance(content, str) and content_idx_optional is None:
-                # Replace string content with guardrail response
-                messages[msg_idx]["content"] = guardrail_response
+            if content_idx_optional is None:
+                rewritten = _rewritten_whole_text(messages[msg_idx], field, guardrail_response)
+                if rewritten is not None:
+                    messages[msg_idx][field] = rewritten
 
-            elif isinstance(content, list) and content_idx_optional is not None:
+            elif isinstance(content, list):
                 # Replace specific text item in list content
-                if isinstance(messages[msg_idx]["content"][content_idx_optional], dict):
-                    messages[msg_idx]["content"][content_idx_optional]["text"] = guardrail_response
+                if isinstance(messages[msg_idx][field][content_idx_optional], dict):
+                    messages[msg_idx][field][content_idx_optional]["text"] = guardrail_response
 
     async def process_output_response(
         self,

litellm/llms/openai/responses/guardrail_translation/tool_merge.py

@@ -6,6 +6,7 @@ from typing import Final, TypeAlias
 from pydantic import BaseModel, TypeAdapter, ValidationError
 
 from litellm._logging import verbose_logger
+from litellm.llms.base_llm.guardrail_translation.utils import merge_returned_tools_into_request_tools, openai_tool_name
 from litellm.responses.litellm_completion_transformation.custom_tools import custom_tool_grammar_suffix
 from litellm.responses.litellm_completion_transformation.transformation import (
     NAMESPACE_DESCRIPTION_SEPARATOR,
@@ -183,3 +184,18 @@ def merge_guardrailed_tools(
         tuple(tool for key, tool in zip(guardrailed_keys, guardrailed) if key not in owned_keys)
     )
     return tuple(chain(merged_originals, appended))
+
+
+def append_returned_tools(
+    original_tools: Sequence[Tool],
+    flattened_groups: Sequence[Sequence[Tool]],
+    returned_tools: Iterable[object],
+) -> tuple[Tool, ...]:
+    request_chat_tools: Final = tuple(chain.from_iterable(flattened_groups))
+    additions: Final = merge_returned_tools_into_request_tools(
+        request_tools=request_chat_tools, returned_tools=_validated_tools(returned_tools), tool_name=openai_tool_name
+    )[len(request_chat_tools) :]
+    return (
+        *original_tools,
+        *LiteLLMCompletionResponsesConfig.transform_chat_completion_tool_params_to_responses_api_tools(additions),
+    )

litellm/llms/openai/responses/transformation.py

@@ -108,7 +108,7 @@ class _DropParamsSetting(BaseModel):
     drop_params: object = None
 
 
-def _drops_params(litellm_params: GenericLiteLLMParams) -> bool:
+def drops_params(litellm_params: GenericLiteLLMParams) -> bool:
     setting: Final = _DropParamsSetting.model_validate(litellm_params)
     return litellm.drop_params is True or normalize_drop_params(setting.drop_params) is True
 
@@ -544,7 +544,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         litellm_params: GenericLiteLLMParams,
         prompt_cache_options: object = None,
     ) -> tuple[str | ResponseInputParam, Sequence[ALL_RESPONSES_API_TOOL_PARAMS] | None]:
-        validated_input: Final = self._validate_input_param(self._drop_bridge_minted_reasoning_items(input))
+        validated_input: Final = self._validate_input_param(
+            self._drop_bridge_minted_reasoning_items(input), drops_set_created_by=drops_params(litellm_params)
+        )
         marked_input: Final = (
             self._input_with_prompt_cache_breakpoints(validated_input)
             if self._reads_prompt_cache_breakpoints(
@@ -585,7 +587,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         if tools is None or not any(is_anthropic_tool_search_chat_tool(tool) for tool in tools):
             return tools
         callers_own_search: Final = any(is_openai_tool_search_tool(tool) for tool in tools)
-        if callers_own_search or not _drops_params(litellm_params) or _runs_tool_search(model, tools):
+        if callers_own_search or not drops_params(litellm_params) or _runs_tool_search(model, tools):
             return tools_with_anthropic_tool_search_mapped(tools)
         verbose_logger.warning(
             "Dropping tool search under drop_params: OpenAI model '%s' cannot run it or no tool is deferred, so every "
@@ -778,7 +780,9 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         ]
         return tools if all(new is old for new, old in zip(sanitized, tools, strict=True)) else sanitized
 
-    def _validate_input_param(self, input: str | ResponseInputParam) -> str | ResponseInputParam:
+    def _validate_input_param(
+        self, input: str | ResponseInputParam, *, drops_set_created_by: bool = False
+    ) -> str | ResponseInputParam:
         """
         Ensure all input fields if pydantic are converted to dict
 
@@ -790,18 +794,20 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
             for item in input:
                 # if it's pydantic, convert to dict
                 if isinstance(item, BaseModel):
-                    validated_input.append(item.model_dump(exclude_none=True))
+                    validated_input.append(
+                        replayable_responses_item(
+                            item.model_dump(exclude_none=True), drops_set_created_by=drops_set_created_by
+                        )
+                    )
                 elif isinstance(item, dict):
-                    # Handle reasoning items specifically to filter out status=None
-                    if item.get("type") == "reasoning":
-                        verbose_logger.debug("Handling reasoning item: %s", item)
-                        # Type assertion since we know it's a dict at this point
-                        dict_item = cast(dict[str, object], item)
-                        filtered_item = self._handle_reasoning_item(dict_item)
-                    else:
-                        # For other dict items, just pass through
-                        filtered_item = cast(dict[str, object], item)
-                    validated_input.append(filtered_item)
+                    validated_input.append(
+                        replayable_responses_item(
+                            self._handle_reasoning_item(cast(dict[str, object], item))
+                            if item.get("type") == "reasoning"
+                            else item,
+                            drops_set_created_by=drops_set_created_by,
+                        )
+                    )
                 else:
                     validated_input.append(item)
             return validated_input
@@ -1091,7 +1097,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         api_base: str,
         litellm_params: GenericLiteLLMParams,
         headers: dict,
-    ) -> tuple[str, dict]:
+    ) -> tuple[str, dict[str, str]]:
         """
         Transform the get response API request into a URL and data
 
@@ -1100,7 +1106,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         """
         encoded_response_id: Final = encode_url_path_segment(response_id, field_name="response_id")
         url: Final = f"{api_base}/{encoded_response_id}"
-        data: Final[dict] = {}
+        data: Final[dict[str, str]] = {}
         return url, data
 
     def transform_get_response_api_response(
@@ -1215,7 +1221,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
     def compact_dropped_params(self, litellm_params: GenericLiteLLMParams) -> frozenset[str]:
         if self.custom_llm_provider not in _PROVIDERS_WITH_STRICT_COMPACT_PARAMS:
             return frozenset()
-        if _drops_params(litellm_params):
+        if drops_params(litellm_params):
             return _COMPACT_TRACKING_PARAMS | _COMPACT_REJECTED_PARAMS
         return _COMPACT_TRACKING_PARAMS
 

litellm/llms/openai_like/dynamic_config.py

@@ -55,6 +55,15 @@ def create_config_class(provider: SimpleProviderConfig):
             else:
                 return super()._transform_messages(messages=messages, model=model, is_async=False)
 
+        def _should_preserve_cache_control_for_endpoint(
+            self,
+            custom_llm_provider: str | None,
+            api_base: str | None,
+        ) -> bool:
+            return provider.special_handling.get(
+                "preserve_cache_control", False
+            ) or super()._should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base)
+
         def _get_openai_compatible_provider_info(
             self, api_base: str | None, api_key: str | None
         ) -> tuple[str | None, str | None]:
@@ -177,7 +186,7 @@ def create_config_class(provider: SimpleProviderConfig):
     return JSONProviderConfig
 
 
-_responses_config_cache: Final[dict] = {}
+_responses_config_cache: Final[dict[str, "type[OpenAILikeResponsesConfig]"]] = {}
 
 
 def create_responses_config_class(provider: SimpleProviderConfig) -> "type[OpenAILikeResponsesConfig]":

litellm/llms/openai_like/json_loader.py

@@ -3,6 +3,7 @@ JSON-based provider configuration loader for OpenAI-compatible providers.
 """
 
 import json
+from collections.abc import Mapping
 from pathlib import Path
 from typing import Final
 
@@ -12,6 +13,7 @@ from litellm._logging import verbose_logger
 from litellm.types.utils import ProviderSpecificModelInfo
 
 _PROVIDER_INFO_ADAPTER: Final = TypeAdapter(ProviderSpecificModelInfo)
+_SPECIAL_HANDLING_ADAPTER: Final = TypeAdapter(Mapping[str, bool])
 
 
 class SimpleProviderConfig:
@@ -20,12 +22,14 @@ class SimpleProviderConfig:
     def __init__(self, slug: str, data: dict):
         self.slug = slug
         self.base_url = data["base_url"]
-        self.api_key_env = data["api_key_env"]
-        self.api_base_env = data.get("api_base_env")
+        self.api_key_env: str = data["api_key_env"]
+        self.api_base_env: str | None = data.get("api_base_env")
         self.base_class = data.get("base_class", "openai_gpt")
         self.param_mappings = data.get("param_mappings", {})
         self.constraints = data.get("constraints", {})
-        self.special_handling = data.get("special_handling", {})
+        self.special_handling: Mapping[str, bool] = _SPECIAL_HANDLING_ADAPTER.validate_python(
+            data.get("special_handling", {})
+        )
         self.supported_endpoints = data.get("supported_endpoints", [])
         self.provider_info: ProviderSpecificModelInfo = (
             _PROVIDER_INFO_ADAPTER.validate_python(data["provider_info"]) if "provider_info" in data else {}

litellm/llms/openai_like/providers.json

@@ -269,5 +269,21 @@
       "/v1/chat/completions",
       "/v1/responses"
     ]
+  },
+  "streamlake": {
+    "base_url": "https://vanchin.streamlake.ai/api/gateway/v1/endpoints",
+    "api_key_env": "STREAMLAKE_API_KEY",
+    "api_base_env": "STREAMLAKE_API_BASE",
+    "param_mappings": {
+      "max_tokens": "max_completion_tokens"
+    },
+    "provider_info": {
+      "supports_function_calling": true,
+      "supports_reasoning": true
+    },
+    "special_handling": {
+      "preserve_cache_control": true
+    },
+    "supported_endpoints": ["/v1/chat/completions"]
   }
 }

litellm/llms/perplexity/responses/transformation.py

@@ -17,7 +17,7 @@ from pydantic import TypeAdapter
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.url_utils import join_api_base
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
-from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
+from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig, drops_params
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import ResponseInputParam, ResponsesAPIResponse
 from litellm.types.router import GenericLiteLLMParams
@@ -84,7 +84,7 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig):
         """Handle preset/ model prefix: send as {"preset": name} instead of {"model": name}."""
         input = self._ensure_message_type(input)
         if model.startswith("preset/"):
-            input = self._validate_input_param(input)
+            input = self._validate_input_param(input, drops_set_created_by=drops_params(litellm_params))
             data: Final[dict] = {
                 "preset": model[len("preset/") :],
                 "input": input,

litellm/llms/volcengine/responses/transformation.py

@@ -313,10 +313,10 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig):
         api_base: str,
         litellm_params: GenericLiteLLMParams,
         headers: dict,
-    ) -> tuple[str, dict]:
+    ) -> tuple[str, dict[str, str]]:
         encoded_response_id: Final = encode_url_path_segment(response_id, field_name="response_id")
         url: Final = f"{api_base}/{encoded_response_id}"
-        data: Final[dict] = {}
+        data: Final[dict[str, str]] = {}
         return url, data
 
     def transform_get_response_api_response(

litellm/main.py

@@ -2672,10 +2672,13 @@ def _complete_custom_openai(
 
     # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there
     api_base = _resolve_openai_api_base(api_base)
+    if custom_llm_provider == "bigmodel" and not api_key:
+        raise litellm.AuthenticationError(
+            message="Set BIGMODEL_API_KEY or pass api_key", llm_provider="bigmodel", model=model
+        )
     organization = (
         organization
-        or litellm.organization
-        or get_secret("OPENAI_ORGANIZATION")
+        or (None if custom_llm_provider == "bigmodel" else litellm.organization or get_secret("OPENAI_ORGANIZATION"))
         or None  # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105
     )
     openai.organization = organization
@@ -6299,6 +6302,7 @@ def completion(
             or custom_llm_provider == "nebius"
             or custom_llm_provider == "wandb"
             or custom_llm_provider == "clarifai"
+            or custom_llm_provider == "bigmodel"
             or custom_llm_provider in litellm.openai_compatible_providers
             or JSONProviderRegistry.exists(custom_llm_provider)  # JSON-configured providers
             or "ft:gpt-3.5-turbo" in model  # finetune gpt-3.5-turbo

litellm/model_prices_and_context_window_backup.json

@@ -28071,7 +28071,7 @@
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
-        "supports_vision": false
+        "supports_vision": true
     },
     "fireworks_ai/accounts/fireworks/models/mixtral-8x22b-instruct-hf": {
         "input_cost_per_token": 1.2e-06,
@@ -28345,7 +28345,7 @@
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
-        "supports_vision": false
+        "supports_vision": true
     },
     "fireworks_ai/qwen3p7-plus": {
         "cache_read_input_token_cost": 8e-08,
@@ -61389,6 +61389,359 @@
         "supports_tool_choice": true,
         "supports_vision": true
     },
+    "streamlake/deepseek-v4-flash-0731": {
+        "cache_read_input_token_cost": 1.4e-08,
+        "input_cost_per_token": 4.4e-07,
+        "litellm_provider": "streamlake",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 384000,
+        "mode": "chat",
+        "output_cost_per_token": 1.32e-06,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_prompt_caching": true,
+        "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
+    },
+    "streamlake/DeepSeek-V4-Pro-0813": {
+        "cache_read_input_token_cost": 4.4e-08,
+        "input_cost_per_token": 1.32e-06,
+        "litellm_provider": "streamlake",
+        "max_input_tokens": 1048566,
+        "max_output_tokens": 384000,
+        "mode": "chat",
+        "output_cost_per_token": 3.96e-06,
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
+    },
+    "streamlake/DeepSeek-V3": {
+        "cache_read_input_token_cost": 1.14e-07,
+        "input_cost_per_token": 2.86e-07,
+        "litellm_provider": "streamlake",
+        "max_input_tokens": 56000,
+        "max_output_tokens": 8000,
+        "mode": "chat",
+        "output_cost_per_token": 1.143e-06,
+        "supports_function_calling": true,
+        "supports_reasoning": false,
+        "supports_prompt_caching": true,
+        "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
+    },
+    "streamlake/GLM-5.3": {
+        "cache_read_input_token_cost": 2.6e-07,
+        "input_cost_per_token": 1.4e-06,
+        "litellm_provider": "streamlake",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 4.4e-06,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_assistant_prefill": true,
+        "supports_prompt_caching": true,
+        "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
+    },
+    "streamlake/GLM-5.3-Flash": {
+        "cache_read_input_token_cost": 3e-08,
+        "input_cost_per_token": 1.5e-07,
+        "litellm_provider": "streamlake",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-07,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_assistant_prefill": true,
+        "supports_prompt_caching": true,
+        "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
+    },
+    "streamlake/Kimi-K2-Instruct-0905": {
+        "cache_read_input_token_cost": 1.43e-07,
+        "input_cost_per_token": 5.71e-07,
+        "litellm_provider": "streamlake",
+        "max_input_tokens": 224000,
+        "max_output_tokens": 32000,
+        "mode": "chat",
+        "output_cost_per_token": 2.286e-06,
+        "supports_function_calling": true,
+        "supports_reasoning": false,
+        "supports_prompt_caching": true,
+        "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
+    },
+    "streamlake/deepseek-r1": {
+        "cache_read_input_token_cost": 2.29e-07,
+        "input_cost_per_token": 5.71e-07,
+        "litellm_provider": "streamlake",
+        "max_input_tokens": 98304,
+        "max_output_tokens": 16000,
+        "mode": "chat",
+        "output_cost_per_token": 2.286e-06,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_prompt_caching": true,
+        "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
+    },
+    "streamlake/deepseek-v3.1": {
+        "cache_read_input_token_cost": 2.29e-07,
+        "input_cost_per_token": 5.71e-07,
+        "litellm_provider": "streamlake",
+        "max_input_tokens": 98304,
+        "max_output_tokens": 32000,
+        "mode": "chat",
+        "output_cost_per_token": 1.714e-06,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_prompt_caching": true,
+        "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
+    },
+    "streamlake/deepseek-v4-flash": {
+        "cache_read_input_token_cost": 2.8e-08,
+        "input_cost_per_token": 1.4e-07,
+        "litellm_provider": "streamlake",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 384000,
+        "mode": "chat",
+        "output_cost_per_token": 2.8e-07,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_prompt_caching": true,
+        "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
+    },
+    "streamlake/deepseek-v4-pro": {
+        "cache_read_input_token_cost": 1.45e-07,
+        "input_cost_per_token": 1.74e-06,
+        "litellm_provider": "streamlake",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 384000,
+        "mode": "chat",
+        "output_cost_per_token": 3.48e-06,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_prompt_caching": true,
+        "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
+    },
+    "streamlake/deepseek-v4.1-flash-1017": {
+        "cache_read_input_token_cost": 6e-09,
+        "input_cost_per_token": 3e-07,
+        "litellm_provider": "streamlake",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 384000,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-06,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_vision": true,
+        "supports_prompt_caching": true,
+        "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
+    },
+    "streamlake/deepseek-v4.1-flash-260910": {
+        "cache_read_input_token_cost": 6e-09,
+        "input_cost_per_token": 3e-07,
+        "litellm_provider": "streamlake",
+        "max_input_tokens": 1048568,
+        "max_output_tokens": 384000,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-06,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_vision": true,
+        "supports_prompt_caching": true,
+        "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
+    },
+    "streamlake/glm-5": {
+        "cache_read_input_token_cost": 2e-07,
+        "input_cost_per_token": 1e-06,
+        "litellm_provider": "streamlake",
+        "max_input_tokens": 202752,
+        "max_output_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 3.2e-06,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_prompt_caching": true,
+        "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
+    },
+    "streamlake/glm-5.1": {
+        "cache_read_input_token_cost": 2.6e-07,
+        "input_cost_per_token": 1.4e-06,
+        "litellm_provider": "streamlake",
+        "max_input_tokens": 202752,
+        "max_output_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 4.4e-06,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_prompt_caching": true,
+        "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
+    },
+    "streamlake/glm-5.2": {
+        "cache_read_input_token_cost": 2.6e-07,
+        "input_cost_per_token": 1.4e-06,
+        "litellm_provider": "streamlake",
+        "max_input_tokens": 1048566,
+        "max_output_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 4.4e-06,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_prompt_caching": true,
+        "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
+    },
+    "streamlake/kimi-k2.6": {
+        "cache_read_input_token_cost": 1.6e-07,
+        "input_cost_per_token": 9.5e-07,
+        "litellm_provider": "streamlake",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 256000,
+        "mode": "chat",
+        "output_cost_per_token": 4e-06,
+        "supports_function_calling": true,
+        "supports_vision": true,
+        "supports_reasoning": false,
+        "supports_prompt_caching": true,
+        "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
+    },
+    "streamlake/kimi-k2.7-code": {
+        "cache_read_input_token_cost": 1.9e-07,
+        "input_cost_per_token": 9.5e-07,
+        "litellm_provider": "streamlake",
+        "max_input_tokens": 262144,
+        "max_output_tokens": 32000,
+        "mode": "chat",
+        "output_cost_per_token": 4e-06,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_vision": true,
+        "supports_prompt_caching": true,
+        "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
+    },
+    "streamlake/mimo-v2.6-flash": {
+        "cache_read_input_token_cost": 2.8e-09,
+        "input_cost_per_token": 1.4e-07,
+        "litellm_provider": "streamlake",
+        "mode": "chat",
+        "output_cost_per_token": 2.8e-07,
+        "supports_vision": true,
+        "supports_prompt_caching": true,
+        "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
+    },
+    "streamlake/mimo-v2.6-pro": {
+        "cache_read_input_token_cost": 3.6e-09,
+        "input_cost_per_token": 4.35e-07,
+        "litellm_provider": "streamlake",
+        "mode": "chat",
+        "output_cost_per_token": 8.7e-07,
+        "supports_vision": true,
+        "supports_prompt_caching": true,
+        "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
+    },
+    "streamlake/minimax-m2.5": {
+        "cache_creation_input_token_cost": 3.75e-07,
+        "cache_read_input_token_cost": 3e-08,
+        "input_cost_per_token": 3e-07,
+        "litellm_provider": "streamlake",
+        "max_input_tokens": 196608,
+        "max_output_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-06,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_prompt_caching": true,
+        "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
+    },
+    "streamlake/minimax-m3": {
+        "cache_read_input_token_cost": 1.2e-07,
+        "cache_read_input_token_cost_above_512k_tokens": 2.4e-07,
+        "input_cost_per_token": 6e-07,
+        "input_cost_per_token_above_512k_tokens": 1.2e-06,
+        "litellm_provider": "streamlake",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 512000,
+        "mode": "chat",
+        "output_cost_per_token": 2.4e-06,
+        "output_cost_per_token_above_512k_tokens": 4.8e-06,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_vision": true,
+        "supports_prompt_caching": true,
+        "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
+    },
+    "streamlake/qwen3.5-397b-a17b": {
+        "cache_read_input_token_cost": 1.2e-07,
+        "input_cost_per_token": 6e-07,
+        "litellm_provider": "streamlake",
+        "max_input_tokens": 258048,
+        "max_output_tokens": 64000,
+        "mode": "chat",
+        "output_cost_per_token": 3.6e-06,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_vision": true,
+        "supports_prompt_caching": true,
+        "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
+    },
+    "streamlake/qwen3.6-27b": {
+        "input_cost_per_token": 6e-07,
+        "litellm_provider": "streamlake",
+        "max_input_tokens": 229376,
+        "max_output_tokens": 64000,
+        "mode": "chat",
+        "output_cost_per_token": 3.6e-06,
+        "supports_function_calling": true,
+        "supports_vision": true,
+        "supports_reasoning": false,
+        "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
+    },
+    "streamlake/qwen3.6-35b-a3b": {
+        "input_cost_per_token": 3.75e-07,
+        "litellm_provider": "streamlake",
+        "max_input_tokens": 229376,
+        "max_output_tokens": 64000,
+        "mode": "chat",
+        "output_cost_per_token": 2.25e-06,
+        "supports_function_calling": true,
+        "supports_vision": true,
+        "supports_reasoning": false,
+        "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
+    },
+    "streamlake/qwen3.8-2.4t-a95b": {
+        "cache_read_input_token_cost": 2.5e-07,
+        "input_cost_per_token": 2e-06,
+        "litellm_provider": "streamlake",
+        "max_input_tokens": 983616,
+        "max_output_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 6e-06,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_vision": true,
+        "cache_creation_input_token_cost": 2.5e-06,
+        "explicit_cache_read_input_token_cost": 1.7e-07,
+        "supports_prompt_caching": true,
+        "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
+    },
+    "streamlake/qwen3.8-27b": {
+        "cache_read_input_token_cost": 1e-07,
+        "input_cost_per_token": 5e-07,
+        "litellm_provider": "streamlake",
+        "max_input_tokens": 983616,
+        "max_output_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 3e-06,
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_vision": true,
+        "cache_creation_input_token_cost": 6.25e-07,
+        "explicit_cache_read_input_token_cost": 5e-08,
+        "supports_prompt_caching": true,
+        "source": "https://www.streamlake.ai/document/DOC/mgrnm4xm362hvp5wyce"
+    },
     "deepseek-v4-flash": {
         "cache_creation_input_token_cost": 0.0,
         "cache_read_input_token_cost": 6e-09,
@@ -62704,6 +63057,14 @@
                     "litellm_provider": "anthropic"
                 }
             },
+            {
+                "name": "typesafe-jev-ids",
+                "pattern": "^jev-(?:latest|preview|1\\.13\\.0|1\\.14\\.0)$",
+                "description": "A bare Jev id that has a typesafe/ price entry (jev-latest, jev-preview, jev-1.13.0, jev-1.14.0), the name the /typesafe pass-through forwards to TypeSafe. Listed by name, matched case-insensitively like every rule here, so foo/jev-latest or a model group named like a Jev id TypeSafe does not serve, such as jev-1.0 or jev-internal-router, never resolves to typesafe. A new Jev version goes here together with its price entry. Provider inference feeds the proxy's provider/* wildcard access checks, so this rule is what lets typesafe/* on a key, team or project allowlist cover a jev-latest request.",
+                "model_info": {
+                    "litellm_provider": "typesafe"
+                }
+            },
             {
                 "name": "claude-family-baseline",
                 "pattern": "claude-[a-z]+-\\d+(?:[-.]\\d+)?(?:-\\d{8})?",
@@ -78634,5 +78995,81 @@
         "supports_response_schema": true,
         "supports_tool_choice": true,
         "supports_vision": true
+    },
+    "bigmodel/glm-5.3": {
+        "input_cost_per_token": 1.1428571428571428e-06,
+        "output_cost_per_token": 4e-06,
+        "cache_read_input_token_cost": 2.857142857142857e-07,
+        "litellm_provider": "bigmodel",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "mode": "chat",
+        "source": "https://docs.bigmodel.cn/cn/guide/start/pricing",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true,
+        "thinking_always_on": true,
+        "metadata": {
+            "comment": "Official API list price checked 2026-10-27: input 8, output 28, cached input 2 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded."
+        },
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses",
+            "/v1/messages"
+        ],
+        "supports_native_streaming": true
+    },
+    "bigmodel/glm-5.3-flash": {
+        "input_cost_per_token": 1.142857142857143e-07,
+        "output_cost_per_token": 4e-07,
+        "cache_read_input_token_cost": 3.2857142857142855e-08,
+        "litellm_provider": "bigmodel",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "mode": "chat",
+        "source": "https://docs.bigmodel.cn/cn/guide/start/pricing",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true,
+        "thinking_always_on": true,
+        "supports_vision": true,
+        "supports_video_input": true,
+        "metadata": {
+            "comment": "Official API list price checked 2026-10-27: input 0.8, output 2.8, cached input 0.23 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded."
+        },
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses",
+            "/v1/messages"
+        ],
+        "supports_native_streaming": true
+    },
+    "bigmodel/glm-5.3-flashx": {
+        "input_cost_per_token": 2.857142857142857e-07,
+        "output_cost_per_token": 1e-06,
+        "cache_read_input_token_cost": 8.142857142857142e-08,
+        "litellm_provider": "bigmodel",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "mode": "chat",
+        "source": "https://docs.bigmodel.cn/cn/guide/start/pricing",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true,
+        "thinking_always_on": true,
+        "supports_vision": true,
+        "supports_video_input": true,
+        "metadata": {
+            "comment": "Official API list price checked 2026-10-27: input 2, output 7, cached input 0.57 CNY per million tokens; converted at 7.0 CNY/USD. Cache storage is temporarily free and is not encoded."
+        },
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses",
+            "/v1/messages"
+        ],
+        "supports_native_streaming": true
     }
 }

litellm/provider_endpoints_support_backup.json

@@ -3785,6 +3785,32 @@
         "generateContent": true
       }
     },
+    "streamlake": {
+      "display_name": "StreamLake (`streamlake`)",
+      "url": "https://docs.litellm.ai/docs/providers/openai_compatible",
+      "endpoints": {
+        "chat_completions": true,
+        "responses": true,
+        "text_completion": true,
+        "embeddings": false,
+        "image_generations": false,
+        "image_edits": false,
+        "audio_transcriptions": false,
+        "audio_speech": false,
+        "rerank": false,
+        "ocr": false,
+        "search": false,
+        "moderations": false,
+        "realtime": false,
+        "video_generations": false,
+        "batches": false,
+        "messages": true,
+        "interactions": true,
+        "systemone": false,
+        "a2a": true,
+        "generateContent": true
+      }
+    },
     "tencent": {
       "display_name": "Tencent TokenHub (`tencent`)",
       "url": "https://docs.litellm.ai/docs/providers/tencent",
@@ -5300,6 +5326,32 @@
         "a2a": false,
         "generateContent": false
       }
+    },
+    "bigmodel": {
+      "display_name": "BigModel (Zhipu AI) (`bigmodel`)",
+      "url": "https://docs.bigmodel.cn/cn/guide/develop/responses/introduction",
+      "endpoints": {
+        "chat_completions": true,
+        "responses": true,
+        "text_completion": true,
+        "embeddings": false,
+        "image_generations": false,
+        "image_edits": false,
+        "audio_transcriptions": false,
+        "audio_speech": false,
+        "rerank": false,
+        "ocr": false,
+        "search": false,
+        "moderations": false,
+        "realtime": false,
+        "video_generations": false,
+        "batches": false,
+        "messages": true,
+        "interactions": true,
+        "systemone": false,
+        "a2a": true,
+        "generateContent": true
+      }
     }
   },
   "endpoints": {

litellm/proxy/_lazy_openapi_snapshot.json

@@ -33639,7 +33639,7 @@
           ]
         },
         "get": {
-          "description": "Get a response by ID.\n\nSupports both:\n- Polling IDs (litellm_poll_*): Returns cumulative cached content from background responses\n- Provider response IDs: Passes through to provider API\n\nFollows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/get\n\n```bash\n# Get polling response\ncurl -X GET http://localhost:4000/v1/responses/litellm_poll_abc123     -H \"Authorization: Bearer $LITELLM_MASTER_KEY\"\n\n# Get provider response\ncurl -X GET http://localhost:4000/v1/responses/resp_abc123     -H \"Authorization: Bearer $LITELLM_MASTER_KEY\"\n```",
+          "description": "Get a response by ID.\n\nSupports both:\n- Polling IDs (litellm_poll_*): Returns cumulative cached content from background responses\n- Provider response IDs: Passes through to provider API\n\nFollows the OpenAI Responses API spec: https://platform.openai.com/docs/api-reference/responses/get\n\n```bash\n# Get polling response\ncurl -X GET http://localhost:4000/v1/responses/litellm_poll_abc123     -H \"Authorization: Bearer $LITELLM_MASTER_KEY\"\n\n# Get provider response\ncurl -X GET http://localhost:4000/v1/responses/resp_abc123     -H \"Authorization: Bearer $LITELLM_MASTER_KEY\"\n\n# Resume a background response's event stream after sequence number 7\ncurl -N \"http://localhost:4000/v1/responses/resp_abc123?stream=true&starting_after=7\"     -H \"Authorization: Bearer $LITELLM_MASTER_KEY\"\n```",
           "operationId": "get_response_openai_v1_responses__response_id__get",
           "parameters": [
             {
@@ -33650,6 +33650,46 @@
                 "title": "Response Id",
                 "type": "string"
               }
+            },
+            {
+              "description": "Replay the response's events as server-sent events (OpenAI, Azure, and Azure AI Foundry)",
+              "in": "query",
+              "name": "stream",
+              "required": false,
+              "schema": {
+                "type": "boolean"
+              }
+            },
+            {
+              "description": "With stream=true, replay only the events after this sequence number",
+              "in": "query",
+              "name": "starting_after",
+              "required": false,
+              "schema": {
+                "minimum": 0,
+                "type": "integer"
+              }
+            },
+            {
+              "description": "Additional output data to include, forwarded to the provider",
+              "in": "query",
+              "name": "include",
+              "required": false,
+              "schema": {
+                "items": {
+                  "type": "string"
+                },
+                "type": "array"
+              }
+            },
+            {
+              "description": "With stream=true, whether delta events carry obfuscation padding",
+              "in": "query",
+              "name": "include_obfuscation",
+              "required": false,
+              "schema": {
+                "type": "boolean"
+              }
             }
           ],
           "responses": {
@@ -33659,6 +33699,12 @@
                   "schema": {
                     "$ref": "#/components/schemas/ResponsesAPIResponse"
                   }
+                },
+                "text/event-stream": {
+                  "schema": {
+                    "description": "Server sent events when stream=true",
+                    "type": "string"
+                  }
                 }
               },
               "description": "Successful Response"

litellm/proxy/_types.py

@@ -3816,6 +3816,16 @@ class ManagementEndpointLoggingPayload(LiteLLMPydanticObjectBase):
     end_time: datetime | None = None
 
 
+def proxy_exception_status_code(message: str, code: str) -> str:
+    # Litellm router.py returns "No healthy deployment available" when there are no deployments available
+    # Should map to 429 errors https://github.com/BerriAI/litellm/issues/2487
+    if "No healthy deployment available" in message or "No deployments available" in message:
+        return "429"
+    if RouterErrors.no_deployments_with_tag_routing.value in message:
+        return "401"
+    return code
+
+
 class ProxyException(Exception):
     # NOTE: DO NOT MODIFY THIS
     # This is used to map exactly to OPENAI Exceptions
@@ -3844,13 +3854,7 @@ class ProxyException(Exception):
                     headers[k] = str(v)
         self.headers = headers or {}
         self.provider_specific_fields = provider_specific_fields
-        # rules for proxyExceptions
-        # Litellm router.py returns "No healthy deployment available" when there are no deployments available
-        # Should map to 429 errors https://github.com/BerriAI/litellm/issues/2487
-        if "No healthy deployment available" in self.message or "No deployments available" in self.message:
-            self.code = "429"
-        elif RouterErrors.no_deployments_with_tag_routing.value in self.message:
-            self.code = "401"
+        self.code = proxy_exception_status_code(self.message, self.code)
 
     def to_dict(self) -> dict:
         """Converts the ProxyException instance to a dictionary."""

litellm/proxy/auth/auth_checks.py

@@ -46,6 +46,11 @@ from litellm.constants import (
 from litellm.litellm_core_utils.dd_tracing import tracer
 from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
+from litellm.litellm_core_utils.tool_fields import DropPolicy
+from litellm.llms.base_llm.managed_resources.utils import (
+    extract_provider_resource_id_from_unified_id,
+    is_base64_encoded_unified_id,
+)
 from litellm.models.project import LiteLLM_ProjectTable
 from litellm.proxy._types import (
     MODEL_LIST_ROUTES,
@@ -165,6 +170,7 @@ from litellm.types.proxy.auth.auth_checks import UserNotFoundError
 from litellm.types.proxy.model_access_group_budget import ModelAccessGroupBudget
 from litellm.types.router import DeploymentTypedDict
 from litellm.utils import get_utc_datetime
+from litellm.vector_stores.vector_store_registry import listed_vector_store_ids, vector_store_ids_in_tool
 
 from .auth_checks_organization import (
     add_team_org_context_to_request_body,
@@ -1063,6 +1069,15 @@ def is_mcp_discovery_request(route: str, request_body: Mapping[str, object]) ->
     return isinstance(method, str) and method in MCP_ZERO_SPEND_JSONRPC_METHODS
 
 
+class ModelMaxBudgetHolder(Protocol):
+    @property
+    def model_max_budget(self) -> Mapping[str, object] | None: ...
+
+
+def model_max_budget_of(holder: ModelMaxBudgetHolder) -> Mapping[str, object] | None:
+    return holder.model_max_budget
+
+
 def route_skips_budget_checks(route: str) -> bool:
     return route not in BUDGET_ENFORCED_SIDE_EFFECT_ROUTES and (
         route in MODEL_DISCOVERY_ROUTES or not RouteChecks.is_llm_api_route(route=route)
@@ -1505,6 +1520,7 @@ async def common_checks(
             valid_token=valid_token,
             user_object=user_object,
             deny_by_default=_vector_store_deny_by_default(_typed_request_body(general_settings)),
+            llm_router=llm_router,
         )
 
     # 12. [OPTIONAL] Tool allowlist - key/team allowed_tools (no DB in hot path)
@@ -7976,6 +7992,57 @@ def _strict_requested_vector_store_ids(request_body: Mapping[str, object]) -> tu
     )
 
 
+@dataclass(frozen=True, slots=True)
+class _UnreadableVectorStoreIds:
+    pass
+
+
+def _top_level_vector_store_ids(
+    vector_store_ids: object, drop_policies: Callable[[], tuple[DropPolicy, ...]]
+) -> tuple[str, ...] | _UnreadableVectorStoreIds:
+    listed: Final = listed_vector_store_ids(vector_store_ids)
+    if vector_store_ids is None or vector_store_ids == [*listed]:
+        return listed
+    if any(policy.drops("vector_store_ids") for policy in drop_policies()):
+        return listed
+    if isinstance(vector_store_ids, str):
+        return (vector_store_ids,)
+    return _UnreadableVectorStoreIds()
+
+
+def _legacy_requested_vector_store_ids(
+    request_body: Mapping[str, object], drop_policies: Callable[[], tuple[DropPolicy, ...]]
+) -> tuple[str, ...] | _UnreadableVectorStoreIds:
+    """
+    Reads tools the way VectorStoreRegistry does and leaves a malformed tool to the proxy's tool
+    validation. A top-level vector_store_ids string names one store, since a query parameter always
+    arrives as a string. Any other value that is not a list of strings names no readable stores unless a
+    drop policy the request can resolve to drops it, and then only the string entries of a list name
+    stores, as in registry retrieval.
+    """
+    top_level: Final = _top_level_vector_store_ids(request_body.get("vector_store_ids"), drop_policies)
+    if isinstance(top_level, _UnreadableVectorStoreIds):
+        return top_level
+    return tuple(
+        chain(
+            top_level,
+            chain.from_iterable(vector_store_ids_in_tool(tool) for tool in _tools(request_body.get("tools"))),
+        )
+    )
+
+
+def _restricts_vector_stores(grants: Sequence[tuple[GrantLayer, LiteLLM_ObjectPermissionTable | None]]) -> bool:
+    return any(grant is not None and bool(grant.vector_stores) for _, grant in grants)
+
+
+def _request_drop_policies(
+    request_body: Mapping[str, object], llm_router: Router | None, valid_token: UserAPIKeyAuth | None
+) -> tuple[DropPolicy, ...]:
+    from litellm.proxy.common_request_processing import counted_drop_policies
+
+    return counted_drop_policies(request_body, llm_router, valid_token.team_id if valid_token is not None else None)
+
+
 def _require_vector_store_grant(
     object_type: GrantLayer,
     vector_store_ids_to_run: Sequence[str],
@@ -8023,6 +8090,7 @@ async def vector_store_access_check(
     *,
     user_object: LiteLLM_UserTable | None = None,
     deny_by_default: bool = False,
+    llm_router: Router | None = None,
 ):
     """
     Checks whether the caller may use every vector store the request names.
@@ -8031,11 +8099,19 @@ async def vector_store_access_check(
     `retrieval_config.vector_store_id`. Grants come from each identity's
     `object_permission.vector_stores`.
 
-    With `deny_by_default=False` (legacy), a key or team only restricts access when its list is
-    nonempty, and stores are only read from the request when a vector store registry is loaded.
+    Both modes read the same request fields, with or without a vector store registry, and answer 400
+    when the top-level `vector_store_ids` is not a list of strings, which legacy mode does only when the
+    key or team has an allowlist and the value is not a string, which it checks as one store id. A
+    LiteLLM-managed store id is compared by the provider store id it decodes to, on both sides. Legacy mode skips that value instead when the request, `litellm_settings`, the router defaults or a deployment of the requested
+    model drops it (`drop_params`, or `additional_drop_params` naming it), the same settings the proxy's
+    tool validation reads, and reads tools the way the registry does, where strict mode answers 400 on a
+    malformed one.
 
-    With `deny_by_default=True` (`general_settings.vector_store_deny_by_default`), stores are read
-    even without a registry, and a missing record, `null` or `[]` grants nothing:
+    With `deny_by_default=False` (legacy), a key or team only restricts access when its list is
+    nonempty.
+
+    With `deny_by_default=True` (`general_settings.vector_store_deny_by_default`), a missing record,
+    `null` or `[]` grants nothing:
 
     - virtual key: the key must grant every store, and so must its team when it has one, even if
       the team failed to load
@@ -8047,8 +8123,8 @@ async def vector_store_access_check(
     rescue nor restrict a key or team request.
 
     Raises ProxyException (401, `{key,team,user}_vector_store_access_denied`) on the first identity
-    that does not grant a requested store, and with the flag on, ProxyException (400,
-    `invalid_request_error`) when a `vector_store_ids` field is not a list of strings.
+    that does not grant a requested store, and ProxyException (400, `invalid_request_error`) on a
+    malformed `vector_store_ids` the request does not drop and, in legacy mode, an allowlist applies to.
     """
     from litellm.proxy.proxy_server import prisma_client, user_api_key_cache
 
@@ -8059,21 +8135,23 @@ async def vector_store_access_check(
         verbose_proxy_logger.debug("Prisma client not found, skipping vector store access check")
         return True
 
-    registry_ids: Final = (
-        _strict_requested_vector_store_ids(_typed_request_body(request_body))
+    typed_request_body: Final = _typed_request_body(request_body)
+    requested_ids: Final = (
+        _strict_requested_vector_store_ids(typed_request_body)
         if deny_by_default
-        else (
-            litellm.vector_store_registry.get_vector_store_ids_to_run(
-                non_default_params=request_body, tools=request_body.get("tools", None)
-            )
-            if litellm.vector_store_registry is not None
-            else None
+        else _legacy_requested_vector_store_ids(
+            typed_request_body, lambda: _request_drop_policies(typed_request_body, llm_router, valid_token)
         )
-        or ()
     )
-    rag_vector_store_id: Final = _get_rag_query_vector_store_id(_typed_request_body(request_body))
+    if isinstance(requested_ids, _UnreadableVectorStoreIds):
+        if _restricts_vector_stores(
+            await _identity_grants(valid_token, team_object, None, prisma_client, user_api_key_cache)
+        ):
+            raise _malformed_vector_store_ids()
+        return True
+    rag_vector_store_id: Final = _get_rag_query_vector_store_id(typed_request_body)
     rag_ids: Final = (rag_vector_store_id,) if rag_vector_store_id is not None else ()
-    vector_store_ids_to_run: Final = tuple(dict.fromkeys((*registry_ids, *rag_ids)))
+    vector_store_ids_to_run: Final = tuple(dict.fromkeys((*requested_ids, *rag_ids)))
     if not vector_store_ids_to_run:
         return True
 
@@ -8100,6 +8178,12 @@ async def vector_store_access_check(
     return True
 
 
+def _provider_vector_store_id(vector_store_id: str) -> str:
+    if not is_base64_encoded_unified_id(vector_store_id):
+        return vector_store_id
+    return extract_provider_resource_id_from_unified_id(vector_store_id) or vector_store_id
+
+
 def _can_object_call_vector_stores(
     object_type: Literal["key", "team", "org", "user"],
     vector_store_ids_to_run: Sequence[str],
@@ -8118,8 +8202,9 @@ def _can_object_call_vector_stores(
     if len(object_permissions.vector_stores) == 0:
         return True
 
+    allowed_stores: Final = frozenset(map(_provider_vector_store_id, object_permissions.vector_stores))
     for vector_store_id in vector_store_ids_to_run:
-        if vector_store_id not in object_permissions.vector_stores:
+        if _provider_vector_store_id(vector_store_id) not in allowed_stores:
             raise ProxyException(
                 message=f"User not allowed to access vector store. Tried to access {vector_store_id}. Only allowed to access {object_permissions.vector_stores}",
                 type=ProxyErrorTypes.get_vector_store_access_error_type_for_object(object_type),

litellm/proxy/auth/model_checks.py

@@ -20,6 +20,8 @@ from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params
 from litellm.types.utils import LlmProviders
 from litellm.utils import ProviderConfigManager, get_valid_models
 
+LLM_PROVIDER_PREFIXES: Final = frozenset(llm_provider.value for llm_provider in LlmProviders)
+
 _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields)
 
 
@@ -458,23 +460,21 @@ def get_known_models_from_wildcard(
             # add model prefix to wildcard models
             wildcard_models = [f"{model_prefix}{model.removeprefix(provider_id_prefix)}" for model in wildcard_models]
 
-    known_providers: Final = {provider.value for provider in LlmProviders}
-    suffix_appended_wildcard_models: Final = []
-    for model in wildcard_models:
-        if not model.startswith(f"{wildcard_provider_prefix}/"):
-            # `get_provider_models` returns provider-prefixed ids (e.g. "ollama/gemma3:1b").
-            # When the wildcard uses a custom prefix (e.g. "ollama_server1/*" to distinguish
-            # multiple instances), replace that existing provider prefix instead of stacking
-            # both, which would otherwise yield an uncallable "ollama_server1/ollama/gemma3:1b".
-            # Only strip the leading segment when it is a known provider, so ids whose first
-            # segment is an org rather than a provider (e.g. "meta-llama/Llama-3-8B") keep it.
-            leading, sep, model_suffix = model.partition("/")
-            if sep and leading in known_providers:
-                model = f"{wildcard_provider_prefix}/{model_suffix}"
-            else:
-                model = f"{wildcard_provider_prefix}/{model}"
-        suffix_appended_wildcard_models.append(model)
-    return list(dict.fromkeys(suffix_appended_wildcard_models))
+    return list(
+        dict.fromkeys(_under_wildcard_prefix(model, wildcard_provider_prefix, provider) for model in wildcard_models)
+    )
+
+
+def _under_wildcard_prefix(model: str, wildcard_provider_prefix: str, listed_provider: str) -> str:
+    if model.startswith(f"{wildcard_provider_prefix}/"):
+        return model
+    leading, sep, model_suffix = model.partition("/")
+    leading_is_listed_providers_own_prefix: Final = (
+        sep != ""
+        and leading in LLM_PROVIDER_PREFIXES
+        and (leading == listed_provider or listed_provider.startswith(f"{leading}_"))
+    )
+    return f"{wildcard_provider_prefix}/{model_suffix if leading_is_listed_providers_own_prefix else model}"
 
 
 def expand_wildcard_deployments_for_model_info(

litellm/proxy/auth/user_api_key_auth.py

@@ -40,6 +40,7 @@ from litellm.integrations.otel.model.config import is_otel_v2_enabled
 from litellm.integrations.otel.runtime import phase_event, phase_span, seed_request_identity
 from litellm.litellm_core_utils.dd_tracing import tracer
 from litellm.litellm_core_utils.dot_notation_indexing import get_nested_value
+from litellm.models.end_user import LiteLLM_EndUserTable
 from litellm.proxy._types import *
 from litellm.proxy.agent_endpoints.auth.agent_caller import agent_caller_from_headers
 from litellm.proxy.auth.auth_checks import (  # noqa: F401  # legacy module exports
@@ -85,6 +86,7 @@ from litellm.proxy.auth.auth_checks import (  # noqa: F401  # legacy module expo
     jwt_key_mapping_cache_key,
     key_access_group_grants_model,
     key_model_aliases_for_auth_check,
+    model_max_budget_of,
     resolve_and_validate_end_user_id,
     resolve_default_end_user_budget,
     route_skips_budget_checks,
@@ -384,6 +386,16 @@ class _TeamModelBudgetLimiter(Protocol):
     ) -> bool: ...
 
 
+class _EndUserModelBudgetLimiter(Protocol):
+    async def is_end_user_within_model_budget(
+        self,
+        end_user_id: str,
+        end_user_model_max_budget: Mapping[str, object],
+        model: str,
+        team_id: str | None = None,
+    ) -> bool: ...
+
+
 class _EveryModelBudgetLimiter(Protocol):
     async def is_key_within_model_budget(self, user_api_key_dict: UserAPIKeyAuth, model: str) -> bool: ...
 
@@ -530,6 +542,49 @@ async def _check_team_model_budget(
         )
 
 
+async def _end_user_model_max_budget(
+    end_user_id: str | None,
+    end_user_object: LiteLLM_EndUserTable | None,
+    key_end_user_budget_id: str | None,
+    prisma_client: PrismaClient | None,
+    user_api_key_cache: UserApiKeyCache,
+    parent_otel_span: Span | None,
+) -> Mapping[str, object] | None:
+    budget_table: Final = (
+        end_user_object.litellm_budget_table
+        if end_user_object is not None
+        else (
+            await resolve_default_end_user_budget(
+                prisma_client=prisma_client,
+                user_api_key_cache=user_api_key_cache,
+                key_end_user_budget_id=key_end_user_budget_id,
+                parent_otel_span=parent_otel_span,
+            )
+            if end_user_id and prisma_client is not None
+            else None
+        )
+    )
+    return model_max_budget_of(budget_table) if budget_table is not None else None
+
+
+async def _check_end_user_model_budget(
+    end_user_id: str | None,
+    end_user_model_max_budget: Mapping[str, object] | None,
+    model_max_budget_limiter: _EndUserModelBudgetLimiter,
+    models: Sequence[str],
+    team_id: str | None,
+) -> None:
+    if not end_user_id or not end_user_model_max_budget:
+        return
+    for model_name in models:
+        await model_max_budget_limiter.is_end_user_within_model_budget(
+            end_user_id=end_user_id,
+            end_user_model_max_budget=end_user_model_max_budget,
+            model=model_name,
+            team_id=team_id,
+        )
+
+
 async def _is_within_every_model_budget(
     valid_token: UserAPIKeyAuth,
     model_max_budget_limiter: _EveryModelBudgetLimiter,
@@ -2094,6 +2149,10 @@ async def _resolve_object_permission_for_unresolvable_team(
     )
 
 
+def _custom_auth_token_for_this_request(response: object) -> UserAPIKeyAuth:
+    return UserAPIKeyAuth.model_validate(response).model_copy()
+
+
 async def user_api_key_auth_builder(
     request: Request,
     api_key: str,
@@ -2172,7 +2231,7 @@ async def user_api_key_auth_builder(
                     request=request, api_key=api_key, user_custom_auth=user_custom_auth
                 )
             if response is not None and isinstance(response, UserAPIKeyAuth):
-                validated = UserAPIKeyAuth.model_validate(response)
+                validated = _custom_auth_token_for_this_request(response)
                 if getattr(litellm, "enable_post_custom_auth_checks", False):
                     validated = await _run_post_custom_auth_checks(
                         valid_token=validated,
@@ -2188,7 +2247,7 @@ async def user_api_key_auth_builder(
                 custom_auth_api_key = True
         elif user_custom_auth is not None:
             response = await user_custom_auth(request=request, api_key=api_key)
-            validated = UserAPIKeyAuth.model_validate(response)
+            validated = _custom_auth_token_for_this_request(response)
             if getattr(litellm, "enable_post_custom_auth_checks", False):
                 validated = await _run_post_custom_auth_checks(
                     valid_token=validated,
@@ -3129,7 +3188,7 @@ async def validate_resolved_virtual_key(  # noqa: C901  # Preserve ordering of e
                         )
 
                     # Recompute after a potential budget-fallback rewrite so
-                    # the end-user check below validates the final model
+                    # the user check below validates the final model
                     current_model = _get_model_from_request_context(
                         request_data=request_data,
                         route=route,
@@ -3147,23 +3206,6 @@ async def validate_resolved_virtual_key(  # noqa: C901  # Preserve ordering of e
                         models=current_models,
                     )
 
-                # Check 5b. End-user model max budget
-                end_user_mmb: Final = valid_token.end_user_model_max_budget
-                if (
-                    end_user_mmb is not None
-                    and isinstance(end_user_mmb, dict)
-                    and len(end_user_mmb) > 0
-                    and current_models
-                    and valid_token.end_user_id is not None
-                ):
-                    for model_name in current_models:
-                        await model_max_budget_limiter.is_end_user_within_model_budget(
-                            end_user_id=valid_token.end_user_id,
-                            end_user_model_max_budget=end_user_mmb,
-                            model=model_name,
-                            team_id=valid_token.team_id,
-                        )
-
         # Check 6: Additional Common Checks across jwt + key auth
         if valid_token.team_id is not None:
             try:
@@ -3733,9 +3775,7 @@ async def run_centralized_common_checks(
     project_object: Final[LiteLLM_ProjectTableCachedObj | None] = (
         None if isinstance(project_result, BaseException) else project_result
     )
-    end_user_object: Final[LiteLLM_EndUserTable | None] = (
-        None if isinstance(end_user_result, BaseException) else end_user_result
-    )
+    end_user_object: Final = end_user_result if isinstance(end_user_result, LiteLLM_EndUserTable) else None
     global_proxy_spend: float | None = None if isinstance(global_spend_result, BaseException) else global_spend_result
     carry_team_and_user_budget_state(
         valid_token=user_api_key_auth_obj,
@@ -3795,6 +3835,21 @@ async def run_centralized_common_checks(
             parent_otel_span=parent_otel_span,
             keep_token_limits=user_custom_auth is not None,
         )
+    end_user_model_max_budget: Final = await _end_user_model_max_budget(
+        end_user_id=end_user_id,
+        end_user_object=end_user_object,
+        key_end_user_budget_id=key_end_user_budget_id,
+        prisma_client=prisma_client,
+        user_api_key_cache=user_api_key_cache,
+        parent_otel_span=parent_otel_span,
+    )
+    if end_user_id and (
+        not user_api_key_auth_obj.authenticated_by_custom_auth
+        or user_api_key_auth_obj.end_user_model_max_budget is None
+    ):
+        user_api_key_auth_obj.end_user_model_max_budget = (  # rebind-ok: the spend hook charges from this token
+            dict(end_user_model_max_budget) if end_user_model_max_budget else None
+        )
 
     skip_budget_checks: Final = _should_skip_budget_checks(
         request_data=request_data,
@@ -3887,6 +3942,21 @@ async def run_centralized_common_checks(
                     user_api_key_cache=user_api_key_cache,
                     proxy_logging_obj=proxy_logging_obj,
                 )
+                await _check_end_user_model_budget(
+                    end_user_id=end_user_id,
+                    end_user_model_max_budget=user_api_key_auth_obj.end_user_model_max_budget,
+                    model_max_budget_limiter=model_max_budget_limiter,
+                    team_id=user_api_key_auth_obj.team_id,
+                    models=_get_model_names_for_budget_checks(
+                        model=_get_model_from_request_context(
+                            request_data=request_data,
+                            route=route,
+                            request=request,
+                            llm_router=llm_router,
+                            team_id=user_api_key_auth_obj.team_id,
+                        )
+                    ),
+                )
             if sub_call and RouteChecks.is_llm_api_route(route=route):
                 await virtual_key_max_budget_check(
                     valid_token=user_api_key_auth_obj,

litellm/proxy/client/cli/commands/agents.py

@@ -5,16 +5,18 @@ import shutil
 import subprocess
 import sys
 import tempfile
-from collections.abc import Callable, Mapping, Sequence
+from collections import Counter
+from collections.abc import Callable, Hashable, Iterable, Mapping, Sequence
 from dataclasses import dataclass
 from itertools import chain
 from pathlib import Path
 from types import MappingProxyType
 from typing import Final, TypeAlias, TypeVar
+from urllib.parse import urlsplit
 
 import click
 import requests
-from pydantic import JsonValue, TypeAdapter, ValidationError
+from pydantic import ConfigDict, JsonValue, TypeAdapter, ValidationError
 
 from litellm.proxy.common_utils.codex_model_catalog import (
     CODEX_BASE_INSTRUCTIONS_PATH,
@@ -497,6 +499,113 @@ def _fetch_model_listing(
     return listing.data
 
 
+class _DeploymentParams(LiteLLMBaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    model: str
+    custom_llm_provider: str | None = None
+    api_base: str | None = None
+    litellm_credential_name: str | None = None
+    reasoning_effort: JsonValue = None
+
+
+class _DeploymentInfo(LiteLLMBaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    team_public_model_name: str | None = None
+    blocked: bool | None = None
+
+
+class _Deployment(LiteLLMBaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    model_name: str
+    litellm_params: _DeploymentParams
+    model_info: _DeploymentInfo | None = None
+
+
+class _DeploymentListing(LiteLLMBaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    data: tuple[_Deployment, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class ModelUpstream:
+    """The OpenAI model every deployment behind a public name routes to, and the reasoning effort they all pin."""
+
+    model: str
+    reasoning_effort: str | None = None
+
+
+_NO_UPSTREAMS: Final[Mapping[str, ModelUpstream]] = MappingProxyType({})
+_Pinned: Final = TypeVar("_Pinned", bound=Hashable)
+
+
+def _public_name(deployment: _Deployment) -> str:
+    team_name: Final = deployment.model_info.team_public_model_name if deployment.model_info is not None else None
+    return team_name or deployment.model_name
+
+
+def _is_paused(deployment: _Deployment) -> bool:
+    return deployment.model_info is not None and deployment.model_info.blocked is True
+
+
+def _is_openai_host(params: _DeploymentParams) -> bool:
+    if params.api_base is None:
+        return not params.litellm_credential_name
+    host: Final = urlsplit(params.api_base).hostname or ""
+    return host == "api.openai.com" or host.endswith(".api.openai.com")
+
+
+def _openai_model(params: _DeploymentParams) -> str | None:
+    if params.custom_llm_provider not in (None, "openai") or not _is_openai_host(params):
+        return None
+    provider, slash, name = params.model.partition("/")
+    if not slash:
+        return params.model
+    return name if provider == "openai" else None
+
+
+def _pinned_effort(params: _DeploymentParams) -> str | None:
+    return params.reasoning_effort if isinstance(params.reasoning_effort, str) else None
+
+
+def _shared_by_every_deployment(values: Iterable[tuple[str, _Pinned]]) -> Mapping[str, _Pinned]:
+    distinct: Final = frozenset(values)
+    values_per_name: Final = Counter(name for name, _ in distinct)
+    return MappingProxyType({name: value for name, value in distinct if values_per_name[name] == 1})
+
+
+def _upstreams_by_public_name(deployments: Sequence[_Deployment]) -> Mapping[str, ModelUpstream]:
+    live: Final = tuple(d for d in deployments if not _is_paused(d))
+    models: Final = _shared_by_every_deployment((_public_name(d), _openai_model(d.litellm_params)) for d in live)
+    efforts: Final = _shared_by_every_deployment((_public_name(d), _pinned_effort(d.litellm_params)) for d in live)
+    return MappingProxyType(
+        {name: ModelUpstream(model, efforts.get(name)) for name, model in models.items() if model is not None}
+    )
+
+
+def _fetch_model_upstreams(
+    base_url: str,
+    api_key: str,
+    *,
+    get: Callable[..., requests.Response],
+) -> Mapping[str, ModelUpstream]:
+    url: Final = base_url.rstrip("/") + "/v1/model/info"
+    try:
+        resp: Final = get(url, headers=MappingProxyType({"Authorization": f"Bearer {api_key}"}), timeout=10)
+    except requests.RequestException:
+        return _NO_UPSTREAMS
+    if resp.status_code != 200:
+        return _NO_UPSTREAMS
+    try:
+        listing: Final = _DeploymentListing.model_validate_json(resp.content)
+    except ValidationError:
+        return _NO_UPSTREAMS
+    return _upstreams_by_public_name(listing.data)
+
+
 class _OpenCodeLimit(LiteLLMBaseModel):
     context: int
     output: int
@@ -576,8 +685,21 @@ def opencode_model_sync_env(
     return MappingProxyType({OPENCODE_CONFIG_CONTENT_ENV: opencode_provider_config(base_url, listing)})
 
 
+def _codex_row(model: ListedModel, upstream: ModelUpstream | None) -> CodexCatalogRow:
+    return CodexCatalogRow(
+        id=model.id,
+        mode=model.mode,
+        max_input_tokens=model.max_input_tokens,
+        upstream_model=upstream.model if upstream is not None else None,
+        reasoning_effort=upstream.reasoning_effort if upstream is not None else None,
+    )
+
+
 def codex_model_catalog(
-    models: Sequence[ListedModel], stock: Sequence[CodexStockModel], instructions: str
+    models: Sequence[ListedModel],
+    stock: Sequence[CodexStockModel],
+    instructions: str,
+    upstreams: Mapping[str, ModelUpstream],
 ) -> str | None:
     """The `model_catalog_json` body listing the proxy's chat models, or None if there are none.
 
@@ -586,14 +708,23 @@ def codex_model_catalog(
     installed Codex knows keeps that Codex's own entry and the proxy only
     decides its place in the picker: the listing orders it, lists it even when
     Codex hides it or keeps it off the API, and keeps Codex's upgrade nudge only
-    when the model it points at is served too. A model Codex does not know gets the fallback
+    when the model it points at is served too. An alias of such a model, per
+    `upstreams`, takes that entry too, minus the defaults Codex picks for the
+    model: no service tier or upgrade nudge, the reasoning effort its
+    deployments pin, and a window no larger than its listed max_input_tokens.
+    A model Codex does not know gets the fallback
     entry, with the same base instructions Codex itself uses so the agent never
-    runs without a system prompt. The catalog is the one `/v1/models?client_version=...`
-    serves, built from the installed Codex's own entries instead of the proxy's vendored ones.
+    runs without a system prompt. Apart from aliases, the catalog is the one
+    `/v1/models?client_version=...` serves, built from the installed Codex's own
+    entries instead of the proxy's vendored ones.
     """
-    rows: Final = tuple(CodexCatalogRow(id=m.id, mode=m.mode, max_input_tokens=m.max_input_tokens) for m in models)
+    rows: Final = tuple(_codex_row(m, upstreams.get(m.id)) for m in models)
     body: Final = codex_models_response_json(
-        rows, stock=MappingProxyType({m.slug: m for m in stock}), instructions=instructions, byte_limit=None
+        rows,
+        stock=MappingProxyType({m.slug: m for m in stock}),
+        instructions=instructions,
+        byte_limit=None,
+        alias_defaults="deployment",
     )
     return body.json if body.listed else None
 
@@ -678,12 +809,14 @@ def codex_model_sync_args(
     Codex has no env or inline equivalent of OPENCODE_CONFIG_CONTENT: the catalog
     must be a file, so it is written under $CODEX_HOME (default ~/.codex) and
     atomically replaced on every launch. The Codex at `binary` first lists its
-    own models, so the ones the proxy serves keep that Codex's entries, and then
+    own models, so the ones the proxy serves keep that Codex's entries, an alias
+    included when every unpaused deployment /v1/model/info lists for it routes
+    to the same OpenAI model and Codex knows that model, and then
     reads the file back once before it is handed over. The key never lands in
     the file. A failed fetch, read, listing, write or read-back is reported
     rather than raised: Codex still launches with its built-in catalog and takes
     a proxy model by name via -m, and a rejected file stays on disk to be looked
-    at.
+    at. An unavailable /v1/model/info only leaves every alias on the fallback entry.
     """
     listing: Final = _fetch_model_listing(base_url, api_key, get=get)
     if isinstance(listing, ModelSyncSkipped):
@@ -700,7 +833,8 @@ def codex_model_sync_args(
     stock: Final = _stock_codex_models(binary, base_env, run=run)
     if isinstance(stock, ModelSyncSkipped):
         return stock
-    catalog: Final = codex_model_catalog(listing, stock, instructions)
+    upstreams: Final = _fetch_model_upstreams(base_url, api_key, get=get)
+    catalog: Final = codex_model_catalog(listing, stock, instructions, upstreams)
     if catalog is None:
         return ModelSyncSkipped(f"{base_url.rstrip('/')}/v1/models lists no chat models")
     try:

litellm/proxy/common_request_processing.py

@@ -724,7 +724,7 @@ def _router_defaults(llm_router: Router | None) -> Mapping[str, object]:
     return {name: value for name, value in _REQUEST_FIELDS.validate_python(configured).items() if value is not None}
 
 
-def _counted_drop_policies(
+def counted_drop_policies(
     data: Mapping[str, object], llm_router: Router | None, team_id: str | None
 ) -> tuple[DropPolicy, ...]:
     requested: Final = {**_router_defaults(llm_router), **data}
@@ -740,7 +740,7 @@ def _counted_drop_policies(
 def resolve_counted_tool_fields(
     data: Mapping[str, object], llm_router: Router | None, team_id: str | None
 ) -> Mapping[str, object] | ProxyException:
-    resolved: Final = resolve_tool_fields(data, lambda: _counted_drop_policies(data, llm_router, team_id))
+    resolved: Final = resolve_tool_fields(data, lambda: counted_drop_policies(data, llm_router, team_id))
     if isinstance(resolved, MalformedField):
         return ProxyException(
             message=resolved.message,
@@ -3206,7 +3206,7 @@ class ProxyBaseLLMRequestProcessing:
                         request_data=self.data,
                         request=request,
                     )
-                    if route_type == "aresponses":
+                    if route_type in ("aresponses", "aget_responses"):
                         selected_data_generator = (
                             ProxyBaseLLMRequestProcessing._wrap_responses_stream_for_container_ownership(
                                 original_stream_response=response,

litellm/proxy/common_utils/codex_model_catalog.py

@@ -24,7 +24,7 @@ from functools import cache
 from itertools import accumulate, pairwise
 from pathlib import Path
 from types import MappingProxyType, NoneType
-from typing import TYPE_CHECKING, Annotated, Final, Literal
+from typing import TYPE_CHECKING, Annotated, Final, Literal, TypeAlias
 
 from pydantic import ConfigDict, StringConstraints, TypeAdapter, ValidationError
 
@@ -102,6 +102,7 @@ class CodexFallbackModel(LiteLLMBaseModel):
     experimental_supported_tools: tuple[()] = ()
     context_window: int | None
     base_instructions: str
+    supports_search_tool: bool = False
 
 
 class CodexStockUpgrade(LiteLLMBaseModel):
@@ -110,6 +111,12 @@ class CodexStockUpgrade(LiteLLMBaseModel):
     model: str
 
 
+class CodexReasoningLevel(LiteLLMBaseModel):
+    model_config = ConfigDict(frozen=True, extra="allow")
+
+    effort: str
+
+
 class CodexStockModel(LiteLLMBaseModel):
     """One `ModelInfo` entry as Codex ships it or prints it from `codex debug models`.
 
@@ -128,6 +135,9 @@ class CodexStockModel(LiteLLMBaseModel):
     upgrade: CodexStockUpgrade | None = None
     service_tiers: tuple[CodexServiceTier, ...] = ()
     default_service_tier: str | None = None
+    supports_search_tool: bool = False
+    supported_reasoning_levels: tuple[CodexReasoningLevel, ...] = ()
+    context_window: int | None = None
 
 
 class CodexStockCatalog(LiteLLMBaseModel):
@@ -135,21 +145,24 @@ class CodexStockCatalog(LiteLLMBaseModel):
 
 
 CodexCatalogEntry = CodexFallbackModel | CodexStockModel
+CodexAliasDefaults: TypeAlias = Literal["codex", "deployment"]
 
 
 @dataclass(frozen=True, slots=True)
 class CodexCatalogRow:
     """What the catalog needs to know about one listed model: its public id, the listing's mode and
-    input limit, the upstream model behind it, and the `model_info` values the operator set;
-    `service_tiers` holds one raw `model_info.service_tiers` value per deployment behind the id, None
-    where a deployment sets none."""
+    input limit, the upstream model behind it, the reasoning effort its deployments pin, and the
+    `model_info` values the operator set; `service_tiers` holds one raw `model_info.service_tiers`
+    value per deployment behind the id, None where a deployment sets none."""
 
     id: str
     mode: str | None = None
     max_input_tokens: int | None = None
     upstream_model: str | None = None
+    reasoning_effort: str | None = None
     display_name: str | None = None
     service_tiers: tuple[object, ...] = ()
+    search_tool_support: tuple[bool | None, ...] = ()
 
 
 @cache
@@ -242,6 +255,12 @@ def configured_service_tiers(
     return offered
 
 
+def offered_search_tool(per_deployment: Sequence[bool | None]) -> bool | None:
+    if False in per_deployment:
+        return False
+    return True if per_deployment and None not in per_deployment else None
+
+
 def _codex_can_drive(row: CodexCatalogRow) -> bool:
     return "*" not in row.id and (row.mode is None or row.mode in CODEX_CHAT_MODES)
 
@@ -254,18 +273,49 @@ def _stock_for(row: CodexCatalogRow, stock: Mapping[str, CodexStockModel]) -> Co
     return upstream if upstream is not None else stock.get(row.upstream_model.partition("/")[2])
 
 
+def _alias_context_window(row: CodexCatalogRow, stock_model: CodexStockModel) -> Mapping[str, int]:
+    if row.max_input_tokens is None or stock_model.context_window is None:
+        return MappingProxyType({})
+    return MappingProxyType({"context_window": min(stock_model.context_window, row.max_input_tokens)})
+
+
+def _alias_reasoning_level(row: CodexCatalogRow, stock_model: CodexStockModel) -> str | None:
+    efforts: Final = frozenset(level.effort for level in stock_model.supported_reasoning_levels)
+    return row.reasoning_effort if row.reasoning_effort in efforts else None
+
+
+def _deployment_defaults(row: CodexCatalogRow, stock_model: CodexStockModel) -> Mapping[str, object]:
+    """What an alias's deployments decide in place of the defaults Codex picks for the model they route to:
+    no service tier, verbosity, upgrade nudge or startup promo of Codex's own, so what the deployment
+    pins (or OpenAI's API default) stays in force, the pinned reasoning effort when Codex offers it and
+    none otherwise, and a window no larger than the alias's own limit."""
+    return MappingProxyType(
+        {
+            "service_tiers": (),
+            "default_service_tier": None,
+            "default_verbosity": None,
+            "upgrade": None,
+            "availability_nux": None,
+            "default_reasoning_level": _alias_reasoning_level(row, stock_model),
+            **_alias_context_window(row, stock_model),
+        }
+    )
+
+
 def _entry(
     index: int,
     row: CodexCatalogRow,
     stock: Mapping[str, CodexStockModel],
     served: frozenset[str],
     instructions: str,
+    alias_defaults: CodexAliasDefaults,
 ) -> CodexCatalogEntry:
     stock_model: Final = _stock_for(row, stock)
     stock_tiers: Final = MappingProxyType(
         {tier.id: tier for tier in (stock_model.service_tiers if stock_model is not None else ())}
     )
     configured: Final = configured_service_tiers(row.service_tiers, row.id, stock_tiers)
+    search_tool: Final = offered_search_tool(row.search_tool_support)
     if stock_model is None:
         return CodexFallbackModel(
             slug=row.id,
@@ -274,11 +324,13 @@ def _entry(
             service_tiers=configured if configured is not None else (),
             context_window=row.max_input_tokens,
             base_instructions=instructions,
+            supports_search_tool=search_tool is True,
         )
     upgrade: Final = (
         stock_model.upgrade if stock_model.upgrade is not None and stock_model.upgrade.model in served else None
     )
-    stock_name: Final = stock_model.display_name if row.id == stock_model.slug else row.id
+    is_alias: Final = row.id != stock_model.slug
+    stock_name: Final = row.id if is_alias else stock_model.display_name
     listing: Final = {
         "slug": row.id,
         "display_name": row.display_name if row.display_name is not None else stock_name,
@@ -286,6 +338,8 @@ def _entry(
         "visibility": "list",
         "supported_in_api": True,
         "upgrade": upgrade,
+        **({} if search_tool is None else {"supports_search_tool": search_tool}),
+        **(_deployment_defaults(row, stock_model) if is_alias and alias_defaults == "deployment" else {}),
     }
     if configured is None:
         return stock_model.model_copy(update=listing)
@@ -351,6 +405,7 @@ def codex_models_response_json(
     stock: Mapping[str, CodexStockModel] | None = None,
     instructions: str | None = None,
     byte_limit: int | None = CODEX_CATALOG_BYTE_LIMIT,
+    alias_defaults: CodexAliasDefaults = "codex",
 ) -> CodexCatalogBody:
     """Codex's `ModelsResponse` body for `rows`, in listing order, cut to Codex's byte limit.
 
@@ -359,13 +414,18 @@ def codex_models_response_json(
     byte limit silently and keeps its bundled one, so the body holds only the entries that fit,
     the ones offering a service tier taken first and the rest in listing order, an entry too large
     for the bytes left passed over, every kept entry at its listing position; `byte_limit=None`
-    keeps every entry.
+    keeps every entry. A row listed under another name than the Codex model its `upstream_model`
+    names (an alias) takes that model's entry; `alias_defaults="deployment"` drops the defaults
+    Codex picks for that model in favor of what the alias's deployments pin, while `"codex"` keeps
+    them, as Codex would use them for the model's own name.
     """
     known: Final = bundled_codex_models() if stock is None else stock
     prompt: Final = codex_base_instructions() if instructions is None else instructions
     drivable: Final = tuple(row for row in rows if _codex_can_drive(row))
     served: Final = frozenset(row.id for row in drivable)
-    entries: Final = tuple(_entry(index, row, known, served, prompt) for index, row in enumerate(drivable))
+    entries: Final = tuple(
+        _entry(index, row, known, served, prompt, alias_defaults) for index, row in enumerate(drivable)
+    )
     serialized: Final = tuple(_serialized(entry) for entry in entries)
     kept: Final = _kept_positions(entries, serialized, byte_limit)
     return CodexCatalogBody(
@@ -385,6 +445,7 @@ def _catalog_row(
         upstream_model=llm_router.get_routable_upstream_model(lookup_id, team_id) if llm_router is not None else None,
         display_name=llm_router.get_configured_display_name(label_id) if llm_router is not None else None,
         service_tiers=llm_router.get_configured_service_tiers(lookup_id, team_id) if llm_router is not None else (),
+        search_tool_support=llm_router.get_search_tool_support(lookup_id, team_id) if llm_router is not None else (),
     )
 
 

litellm/proxy/common_utils/openai_error_payload.py

@@ -2,14 +2,17 @@
 ``type`` is a required string and ``param`` is nullable, neither of which the literal
 string ``"None"`` satisfies."""
 
+import sys
 from collections.abc import Mapping
 from types import MappingProxyType
 from typing import Final
 
-from fastapi import status
+from fastapi import HTTPException, status
 
 from litellm.constants import STRINGIFIED_NONE
-from litellm.proxy._types import ProxyException
+from litellm.proxy._types import ProxyErrorTypes, ProxyException, proxy_exception_status_code
+from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded
+from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler, exception_chain
 
 LITELLM_CALL_ID_HEADER: Final = "x-litellm-call-id"
 
@@ -21,6 +24,19 @@ _OPENAI_ERROR_TYPE_BY_STATUS: Final[Mapping[int, str]] = MappingProxyType(
     }
 )
 
+_AUTH_FAILURE_STATUS_CODES: Final = frozenset({status.HTTP_401_UNAUTHORIZED, status.HTTP_403_FORBIDDEN})
+
+_FAILOVER_SQLSTATES: Final = frozenset({"25006", "57P01", "57P02", "57P03"})
+
+_FAILOVER_REFUSALS: Final = (
+    "is not currently accepting connections",
+    "the database system is starting up",
+    "the database system is shutting down",
+    "the database system is in recovery mode",
+    "the database system is not yet accepting connections",
+    "terminating connection due to administrator command",
+)
+
 
 def attribute_of(value: object, name: str, default: object = None) -> object:
     return getattr(value, name, default)
@@ -61,6 +77,82 @@ def openai_error_param(exc: object) -> str | None:
     return carried if isinstance(carried, str) and carried != STRINGIFIED_NONE else None
 
 
+def proxy_exception_for(exc: Exception, default_status_code: int) -> ProxyException:
+    match exc:
+        case ProxyException():
+            return exc
+        case _ if _is_database_outage(exc):
+            return ProxyException(
+                message=PrismaDBExceptionHandler.database_unavailable_message(exc),
+                type=ProxyErrorTypes.no_db_connection.value,
+                param=None,
+                code=status.HTTP_503_SERVICE_UNAVAILABLE,
+            )
+        case HTTPException(status_code=status_code, detail=detail):
+            return _openai_shaped_proxy_exception(exc, str(detail), status_code)
+        case _:
+            return _openai_shaped_proxy_exception(
+                exc, str(exc) or type(exc).__name__, error_status_code(exc, default_status_code)
+            )
+
+
+def _is_database_outage(exc: Exception) -> bool:
+    return any(map(_is_database_outage_link, exception_chain(exc)))
+
+
+def _is_database_outage_link(exc: BaseException) -> bool:
+    match exc:
+        case DBLookupDeadlineExceeded():
+            return True
+        case Exception() if _prisma_is_loaded():
+            return _is_prisma_outage(exc)
+        case _:
+            return False
+
+
+def _prisma_is_loaded() -> bool:
+    return sys.modules.get("prisma") is not None
+
+
+def _is_prisma_outage(exc: Exception) -> bool:
+    if PrismaDBExceptionHandler.is_prisma_engine_internal_error(exc):
+        return True
+    if type(exc).__module__.partition(".")[0] != "prisma":
+        return False
+    return PrismaDBExceptionHandler.is_database_service_unavailable_error(exc) or _is_failover_refusal(exc)
+
+
+def _is_failover_refusal(exc: Exception) -> bool:
+    if PrismaDBExceptionHandler.is_read_only_transaction_error(exc):
+        return True
+    if PrismaDBExceptionHandler.postgres_sqlstate(exc) in _FAILOVER_SQLSTATES:
+        return True
+    message: Final = str(exc).lower()
+    return any(refusal in message for refusal in _FAILOVER_REFUSALS)
+
+
+def _openai_shaped_proxy_exception(exc: Exception, message: str, status_code: int) -> ProxyException:
+    answered_status: Final = int(proxy_exception_status_code(message, str(status_code)))
+    return ProxyException(
+        message=message,
+        type=_catch_all_error_type(exc, answered_status),
+        param=openai_error_param(exc),
+        code=answered_status,
+    )
+
+
+def _catch_all_error_type(exc: Exception, status_code: int) -> str:
+    if _rejects_the_caller(exc, status_code):
+        return ProxyErrorTypes.auth_error.value
+    return openai_error_type(exc, status_code)
+
+
+def _rejects_the_caller(exc: Exception, status_code: int) -> bool:
+    raised_as_http_answer: Final = isinstance(exc, HTTPException)
+    status_from_upstream: Final = error_status_code(exc, 0) == status_code
+    return status_code in _AUTH_FAILURE_STATUS_CODES and (raised_as_http_answer or not status_from_upstream)
+
+
 def litellm_call_id_headers(litellm_call_id: str | None) -> dict[str, str] | None:  # mutable-ok: ProxyException.headers
     if litellm_call_id is None:
         return None

litellm/proxy/credential_endpoints/endpoints.py

@@ -5,6 +5,7 @@ CRUD endpoints for storing reusable credentials.
 import time
 import unicodedata
 from collections.abc import Mapping
+from types import MappingProxyType
 from typing import (
     Annotated,
     Final,
@@ -23,7 +24,7 @@ from litellm.constants import (
     GITHUB_COPILOT_PER_USER_AUTH_TYPE,
 )
 from litellm.litellm_core_utils.credential_accessor import CredentialAccessor
-from litellm.litellm_core_utils.litellm_logging import get_masked_values
+from litellm.litellm_core_utils.litellm_logging import get_masked_values, has_sensitive_keyword
 from litellm.llms.anthropic.wif import (
     ExportedJwks,
     NotAnInternalIssuerCredential,
@@ -83,6 +84,11 @@ _CREDENTIAL_DICT_ADAPTER: Final = TypeAdapter(dict[str, object])
 _GITHUB_COPILOT_PROVIDER: Final = "github_copilot"
 _DISPLAY_NAME_MAX_LENGTH: Final = 255
 _DISPLAY_NAME_FORBIDDEN_CATEGORIES: Final = frozenset({"Cc", "Cf", "Cs", "Zl", "Zp"})
+_MASK_UNMASKED_LENGTH: Final = 4
+_MASK_ASTERISKS: Final = 4
+_SHORT_SECRET_MASK: Final = "*****"
+_MASKED_READ_BACK_DEPTH: Final = 20
+_NO_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
 
 
 def _normalized_github_copilot_values(
@@ -395,7 +401,14 @@ async def get_credentials(
     """
     try:
         masked_credentials: Final = [
-            _credential_view(credential, get_masked_values(credential.credential_values))
+            _credential_view(
+                credential,
+                get_masked_values(
+                    credential.credential_values,
+                    unmasked_length=_MASK_UNMASKED_LENGTH,
+                    number_of_asterisks=_MASK_ASTERISKS,
+                ),
+            )
             for credential in litellm.credential_list
         ]
         return {"success": True, "credentials": masked_credentials}
@@ -423,7 +436,11 @@ async def get_credential_by_name(
             if credential.credential_name == credential_name:
                 return _credential_view(
                     credential,
-                    get_masked_values(credential.credential_values, unmasked_length=4, number_of_asterisks=4),
+                    get_masked_values(
+                        credential.credential_values,
+                        unmasked_length=_MASK_UNMASKED_LENGTH,
+                        number_of_asterisks=_MASK_ASTERISKS,
+                    ),
                 )
         raise HTTPException(
             status_code=404,
@@ -521,8 +538,8 @@ async def get_credential_by_model(
             raise HTTPException(status_code=404, detail="Model not found")
         masked_credential_values: Final = get_masked_values(
             credential_values,
-            unmasked_length=4,
-            number_of_asterisks=4,
+            unmasked_length=_MASK_UNMASKED_LENGTH,
+            number_of_asterisks=_MASK_ASTERISKS,
         )
         credential: Final = CredentialItem(
             credential_name=f"{model.model_name}-credential-{model_id}",
@@ -908,6 +925,48 @@ def update_db_credential(
     return merged_credential
 
 
+def _is_masked_read_back(value: object) -> bool:
+    if not isinstance(value, str):
+        return False
+    if value == _SHORT_SECRET_MASK:
+        return True
+    shown: Final = _MASK_UNMASKED_LENGTH // 2
+    return len(value) == _MASK_UNMASKED_LENGTH + _MASK_ASTERISKS and value[shown:-shown] == "*" * _MASK_ASTERISKS
+
+
+def _as_fields(value: object) -> Mapping[str, object] | None:
+    try:
+        return _CREDENTIAL_DICT_ADAPTER.validate_python(value, strict=True)
+    except ValidationError:
+        return None
+
+
+def _kept_value(key: str, value: object, stored: Mapping[str, object], depth: int) -> tuple[str, object] | None:
+    if not has_sensitive_keyword(key):
+        return key, value
+    fields: Final = _as_fields(value)
+    if fields is not None and depth < _MASKED_READ_BACK_DEPTH:
+        return key, _without_masked_read_backs(fields, _as_fields(stored.get(key)) or _NO_FIELDS, depth + 1)
+    if not _is_masked_read_back(value):
+        return key, value
+    if key in stored:
+        return key, stored[key]
+    return None
+
+
+def _without_masked_read_backs(
+    requested: Mapping[str, object], stored: Mapping[str, object], depth: int = 0
+) -> Mapping[str, object]:
+    kept: Final = (_kept_value(key, value, stored, depth) for key, value in requested.items())
+    return _CREDENTIAL_DICT_ADAPTER.validate_python(
+        MappingProxyType(dict(entry for entry in kept if entry is not None))
+    )
+
+
+def _stored_objects(stored_values: Mapping[str, object]) -> Mapping[str, object]:
+    return MappingProxyType({key: value for key, value in stored_values.items() if _as_fields(value) is not None})
+
+
 @router.patch(
     "/credentials/{credential_name:path}",
     dependencies=[Depends(user_api_key_auth)],
@@ -936,12 +995,12 @@ async def update_credential(
             )
         requested_display_name: Final = _normalized_display_name(credential.display_name)
         _reject_overlapping_credential_values(credential)
-        incoming_values: Final = _CREDENTIAL_DICT_ADAPTER.validate_python(
+        requested_values: Final = _CREDENTIAL_DICT_ADAPTER.validate_python(
             _resolve_deployment_credentials(llm_router, credential.model_id)
             if credential.model_id
             else credential.credential_values or {}
         )
-        _reject_non_admin_wif_fields(_incoming_wif_fields(incoming_values, credential), user_api_key_dict)
+        _reject_non_admin_wif_fields(_incoming_wif_fields(requested_values, credential), user_api_key_dict)
         if prisma_client is None:
             raise HTTPException(
                 status_code=500,
@@ -952,19 +1011,20 @@ async def update_credential(
         if db_credential is None:
             raise _not_found_unless_config_defined(credential_name, "Credential not found in DB.")
         _reject_non_admin_wif_fields(_stored_wif_fields(db_credential), user_api_key_dict)
+        decrypted_stored_credential: Final = decrypted_credential(db_credential)
+        decrypted_values: Final = _CREDENTIAL_DICT_ADAPTER.validate_python(
+            decrypted_stored_credential.credential_values
+        )
+        incoming_values: Final = _without_masked_read_backs(requested_values, _stored_objects(decrypted_values))
         patch: Final = CredentialItem(
             credential_name=credential_name,
             display_name=(
                 requested_display_name if "display_name" in credential.model_fields_set else db_credential.display_name
             ),
             credential_info=_CREDENTIAL_DICT_ADAPTER.validate_python(credential.credential_info),
-            credential_values=incoming_values,
+            credential_values=_CREDENTIAL_DICT_ADAPTER.validate_python(incoming_values),
             credential_values_to_delete=credential.credential_values_to_delete,
         )
-        decrypted_stored_credential: Final = decrypted_credential(db_credential)
-        decrypted_values: Final = _CREDENTIAL_DICT_ADAPTER.validate_python(
-            decrypted_stored_credential.credential_values
-        )
         stored_info: Final = _CREDENTIAL_DICT_ADAPTER.validate_python(
             db_credential.credential_info  # pyright: ignore[reportUnknownMemberType]  # untyped credential field
         )

litellm/proxy/db/exception_handler.py

@@ -36,7 +36,7 @@ _LOCK_TIMEOUT_SQLSTATE: Final = "55P03"
 _LOCK_TIMEOUT_PHRASE: Final = "canceling statement due to lock timeout"
 
 
-def _exception_chain(e: BaseException) -> Iterator[BaseException]:
+def exception_chain(e: BaseException) -> Iterator[BaseException]:
     current = e  # rebind-ok: advances one link per iteration of the bounded walk
     for _ in range(_MAX_EXCEPTION_CHAIN_DEPTH):
         yield current
@@ -49,7 +49,7 @@ def _exception_chain(e: BaseException) -> Iterator[BaseException]:
 def _database_service_unavailable_errors(e: BaseException) -> tuple[Exception, ...]:
     return tuple(
         link
-        for link in _exception_chain(e)
+        for link in exception_chain(e)
         if isinstance(link, Exception) and PrismaDBExceptionHandler.is_database_service_unavailable_error(link)
     )
 

litellm/proxy/google_endpoints/endpoints.py

@@ -1,11 +1,11 @@
 from collections.abc import Mapping
 from itertools import chain
 from types import MappingProxyType
-from typing import Final
+from typing import Annotated, Final
 
 from fastapi import APIRouter, Depends, HTTPException, Request, Response
 from fastapi.responses import JSONResponse, ORJSONResponse
-from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError
+from pydantic import BaseModel, BeforeValidator, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError
 
 from litellm.litellm_core_utils.json_validation_rule import normalize_json_schema_types
 from litellm.proxy._types import *
@@ -27,14 +27,67 @@ from litellm.proxy.common_utils.token_count_settings import count_source_fields
 from litellm.types.llms.vertex_ai import TokenCountDetailsResponse
 
 
+def _google_repeated_field(value: JsonValue) -> JsonValue:
+    match value:
+        case None:
+            return []
+        case dict():
+            return [value]
+        case list():
+            return [item for item in value if item is not None]
+        case _:
+            return value
+
+
+def _google_content(value: JsonValue) -> JsonValue:
+    match value:
+        case {"parts": parts}:
+            return {**value, "parts": _google_repeated_field(parts)}
+        case _:
+            return value
+
+
+def _google_contents(value: JsonValue) -> JsonValue:
+    match _google_repeated_field(value):
+        case list() as contents:
+            return [_google_content(content) for content in contents]
+        case other:
+            return other
+
+
+_GOOGLE_PART_TEXT_AND_METADATA_FIELDS: Final = frozenset(
+    {"text", "thought", "thoughtSignature", "thought_signature", "partMetadata", "part_metadata"}
+)
+
+
+def _google_part_has_non_text_content(part: Mapping[str, JsonValue]) -> bool:
+    return any(field is not None for key, field in part.items() if key not in _GOOGLE_PART_TEXT_AND_METADATA_FIELDS)
+
+
+def _google_system_part(value: JsonValue) -> JsonValue:
+    match value:
+        case dict() if value.get("text") is None and _google_part_has_non_text_content(value):
+            return {key: field for key, field in value.items() if key != "text"}
+        case dict():
+            return {"text": None, **value}
+        case _:
+            return value
+
+
+_GOOGLE_REPEATED_FIELD: Final = BeforeValidator(_google_repeated_field)
+
+
 class _GoogleCountTokensTextPart(BaseModel):
     model_config = ConfigDict(frozen=True)
-    text: str
+    text: str | None
 
 
 class _GoogleCountTokensSystemInstruction(BaseModel):
     model_config = ConfigDict(frozen=True)
-    parts: tuple[_GoogleCountTokensTextPart, ...]
+    parts: Annotated[
+        tuple[Annotated[_GoogleCountTokensTextPart, BeforeValidator(_google_system_part)], ...],
+        _GOOGLE_REPEATED_FIELD,
+    ] = ()
 
 
 class _GoogleCountTokensFunctionDeclaration(BaseModel):
@@ -47,16 +100,16 @@ class _GoogleCountTokensFunctionDeclaration(BaseModel):
 
 class _GoogleCountTokensTool(BaseModel):
     model_config = ConfigDict(frozen=True, populate_by_name=True)
-    function_declarations: tuple[_GoogleCountTokensFunctionDeclaration, ...] = Field(
+    function_declarations: Annotated[tuple[_GoogleCountTokensFunctionDeclaration, ...], _GOOGLE_REPEATED_FIELD] = Field(
         default=(), alias="functionDeclarations"
     )
 
 
 class _GoogleCountTokensRequest(BaseModel):
     model_config = ConfigDict(frozen=True, extra="ignore", populate_by_name=True)
-    contents: tuple[Mapping[str, JsonValue], ...] = ()
+    contents: Annotated[tuple[Mapping[str, JsonValue], ...], BeforeValidator(_google_contents)] = ()
     system_instruction: _GoogleCountTokensSystemInstruction | None = Field(default=None, alias="systemInstruction")
-    tools: tuple[_GoogleCountTokensTool, ...] = ()
+    tools: Annotated[tuple[_GoogleCountTokensTool, ...], _GOOGLE_REPEATED_FIELD] = ()
 
 
 router: Final = APIRouter(
@@ -112,7 +165,11 @@ def _google_count_tokens_request(model_name: str, data: object) -> TokenCountReq
         GoogleGenAIAdapter().translate_generate_content_to_completion(model_name, contents).get("messages", [])
     )
     system: Final = (
-        [{"type": "text", "text": part.text} for part in validated_data.system_instruction.parts]
+        [
+            {"type": "text", "text": part.text}
+            for part in validated_data.system_instruction.parts
+            if part.text is not None
+        ]
         if validated_data.system_instruction is not None
         else None
     )

litellm/proxy/guardrails/_content_utils.py

@@ -62,6 +62,20 @@ def is_non_conversational_call_type(call_type: str) -> bool:
     return call_type in NON_CONVERSATIONAL_CALL_TYPES
 
 
+_PASSTHROUGH_EMBEDDINGS_ENDPOINT: Final = re.compile(r"/?(?:[\w-]+/)*embeddings/?")
+
+
+def is_non_conversational_request(call_type: str, data: Mapping[str, object]) -> bool:
+    if is_non_conversational_call_type(call_type):
+        return True
+    endpoint: Final = data.get("endpoint")
+    return (
+        call_type == "allm_passthrough_route"
+        and isinstance(endpoint, str)
+        and _PASSTHROUGH_EMBEDDINGS_ENDPOINT.fullmatch(endpoint) is not None
+    )
+
+
 TEXT_PART_TYPES: Final[frozenset[str]] = frozenset(
     {"text", "input_text", "output_text", "summary_text", "reasoning_text"}
 )

litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py

@@ -49,6 +49,7 @@ from litellm.llms.base_llm.guardrail_translation.utils import (
 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token, run_aws_signing
 from litellm.llms.bedrock.guardrail_attachments import find_request_attachments
 from litellm.llms.custom_httpx.http_handler import (
+    AsyncHTTPHandler,
     get_async_httpx_client,
     httpxSpecialProvider,
 )
@@ -286,9 +287,12 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
         streaming_sampling_rate: int | None = None,
         streaming_end_of_stream_only: bool | None = None,
         streaming_buffer_release_on_scan: bool | None = None,
+        async_handler: AsyncHTTPHandler | None = None,
         **kwargs,
     ):
-        self.async_handler = get_async_httpx_client(llm_provider=httpxSpecialProvider.GuardrailCallback)
+        self.async_handler = async_handler or get_async_httpx_client(
+            llm_provider=httpxSpecialProvider.GuardrailCallback
+        )
         self._set_streaming_params(
             BedrockGuardrailStreamingParams.from_extras(
                 MappingProxyType(

litellm/proxy/guardrails/guardrail_hooks/generic_guardrail_api/example_config.yaml

@@ -6,20 +6,19 @@ model_list:
       model: gpt-4
       api_key: os.environ/OPENAI_API_KEY
 
-litellm_settings:
-  guardrails:
-    - guardrail_name: "my-generic-guardrail"
-      litellm_params:
-        guardrail: generic_guardrail_api
-        mode: pre_call  # Options: pre_call, post_call, during_call, [pre_call, post_call]
-        api_key: os.environ/GENERIC_GUARDRAIL_API_KEY  # Optional if using Bearer auth
-        api_base: http://localhost:8080  # Required. Endpoint /beta/litellm_basic_guardrail_api is automatically appended
-        unreachable_fallback: fail_closed  # Options: fail_closed (default, raise), fail_open (proceed if endpoint unreachable or upstream returns 502/503/504)
-        default_on: false  # Set to true to apply to all requests by default
-        additional_provider_specific_params:
-          # Any additional parameters your guardrail API needs
-          api_version: "v1"
-          custom_param: "value"
+guardrails:
+  - guardrail_name: "my-generic-guardrail"
+    litellm_params:
+      guardrail: generic_guardrail_api
+      mode: pre_call  # Options: pre_call, post_call, during_call, [pre_call, post_call]
+      api_key: os.environ/GENERIC_GUARDRAIL_API_KEY  # Optional if using Bearer auth
+      api_base: http://localhost:8080  # Required. Endpoint /beta/litellm_basic_guardrail_api is automatically appended
+      unreachable_fallback: fail_closed  # Options: fail_closed (default, raise), fail_open (proceed if endpoint unreachable or upstream returns 502/503/504)
+      default_on: false  # Set to true to apply to all requests by default
+      additional_provider_specific_params:
+        # Any additional parameters your guardrail API needs
+        api_version: "v1"
+        custom_param: "value"
 
 # Usage examples:
 

litellm/proxy/guardrails/guardrail_hooks/panw_prisma_airs/panw_prisma_airs.py

@@ -30,6 +30,7 @@ from litellm.integrations.custom_guardrail import (
 from litellm.llms.base_llm.guardrail_translation.utils import (
     effective_scan_only_tool_results_for_guardrail,
     effective_skip_system_message_for_guardrail,
+    effective_skip_tool_message_for_guardrail,
     role_out_of_guardrail_scope,
 )
 from litellm.llms.custom_httpx.http_handler import (
@@ -37,12 +38,16 @@ from litellm.llms.custom_httpx.http_handler import (
     get_async_httpx_client,
     httpxSpecialProvider,
 )
-from litellm.llms.openai.responses.guardrail_translation.handler import scannable_instructions
+from litellm.llms.openai.responses.guardrail_translation.handler import (
+    scannable_instructions,
+    tool_output_texts,
+)
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.common_utils.callback_utils import (
     add_guardrail_scan_id,
     add_guardrail_to_applied_guardrails_header,
 )
+from litellm.responses.litellm_completion_transformation.transformation import TOOL_CALL_OUTPUT_ITEM_TYPES
 from litellm.types.guardrails import GuardrailEventHooks
 from litellm.types.llms.base import LiteLLMBaseModel
 from litellm.types.llms.openai import AllMessageValues
@@ -112,12 +117,16 @@ class _ResponsesInputItem(LiteLLMBaseModel):
     type: str | None = None
     role: str | None = None
     content: str | tuple[_ResponsesContentPart, ...] | None = None
+    output: object = None
+    call_id: object = None
 
-    def text_count(self, *, skip_system: bool) -> int:
-        if role_out_of_guardrail_scope(
-            (self.role or "").lower(), skip_system_message=skip_system, skip_tool_message=False
-        ):
+    def text_count(self, *, skip_system: bool, skip_tool: bool) -> int:
+        is_tool_output: Final = self.type in TOOL_CALL_OUTPUT_ITEM_TYPES
+        role: Final = "tool" if is_tool_output else (self.role or "").lower()
+        if role_out_of_guardrail_scope(role, skip_system_message=skip_system, skip_tool_message=skip_tool):
             return 0
+        if is_tool_output:
+            return len(tool_output_texts(self))
         if isinstance(self.content, str):
             return 1
         if self.content is None:
@@ -1692,13 +1701,14 @@ class PanwPrismaAirsHandler(CustomGuardrail):
         request_data: Mapping[str, object],
         *,
         skip_system: bool,
+        skip_tool: bool,
     ) -> frozenset[int] | None:
         """Return the ``texts`` indices flattened from Responses ``reasoning`` input items.
 
         The Responses translation handler gives those model-authored items the default
         ``user`` role, so the latest-turn selection must not mistake one for a human turn.
         Empty for requests without a Responses ``input`` item list; None when the raw items
-        (after the leading ``instructions`` text, both minus whatever ``skip_system`` drops)
+        (after the leading ``instructions`` text, both minus whatever ``skip_system`` and ``skip_tool`` drop)
         do not account for every entry of ``texts``.
         """
         try:
@@ -1708,7 +1718,7 @@ class PanwPrismaAirsHandler(CustomGuardrail):
         if not isinstance(raw_input, tuple):
             return frozenset()
         offset: Final = 0 if scannable_instructions(request_data, skip_system=skip_system) is None else 1
-        counts: Final = tuple(item.text_count(skip_system=skip_system) for item in raw_input)
+        counts: Final = tuple(item.text_count(skip_system=skip_system, skip_tool=skip_tool) for item in raw_input)
         if offset + sum(counts) != len(texts):
             return None
         starts: Final = itertools.accumulate(counts, initial=offset)
@@ -1738,7 +1748,10 @@ class PanwPrismaAirsHandler(CustomGuardrail):
         if sources is None:
             return None
         reasoning: Final = self._reasoning_item_text_indices(
-            texts, request_data, skip_system=effective_skip_system_message_for_guardrail(self)
+            texts,
+            request_data,
+            skip_system=effective_skip_system_message_for_guardrail(self),
+            skip_tool=effective_skip_tool_message_for_guardrail(self),
         )
         if reasoning is None:
             return None

litellm/proxy/health_endpoints/_health_endpoints.py

@@ -32,7 +32,6 @@ from litellm.proxy._types import (
     EnterpriseLicenseData,
     Litellm_EntityType,
     LitellmUserRoles,
-    ProxyErrorTypes,
     ProxyException,
     SpecialModelNames,
     UserAPIKeyAuth,
@@ -47,6 +46,7 @@ from litellm.proxy.auth.auth_utils import (
 from litellm.proxy.auth.model_checks import get_key_models
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.common_utils import http_parsing_utils
+from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
 from litellm.proxy.db.db_lookup_gate import db_lookup_stall_tracker
 from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
 from litellm.proxy.db.health_check_latest import (
@@ -683,21 +683,9 @@ async def health_services_endpoint(
     except Exception as e:
         verbose_proxy_logger.error("litellm.proxy.proxy_server.health_services_endpoint(): Exception occured - %s", e)
         verbose_proxy_logger.debug(traceback.format_exc())
-        if isinstance(e, HTTPException):
-            raise ProxyException(
-                message=getattr(e, "detail", f"Authentication Error({e})"),
-                type=ProxyErrorTypes.auth_error,
-                param=getattr(e, "param", "None"),
-                code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
-            )
-        elif isinstance(e, ProxyException):
-            raise e
-        raise ProxyException(
-            message="Authentication Error, " + str(e),
-            type=ProxyErrorTypes.auth_error,
-            param=getattr(e, "param", "None"),
-            code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-        )
+        if isinstance(e, ProxyException):
+            raise
+        raise proxy_exception_for(e, default_status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) from e
 
 
 def convert_health_check_to_dict(check) -> dict:

litellm/proxy/hooks/parallel_request_limiter_v3.py

@@ -76,6 +76,7 @@ from litellm.proxy.hooks.batch_enqueued_tokens import (
     canonical_provider_batch_id,
 )
 from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit
+from litellm.proxy.spend_tracking.input_tokens import tokenizer_models
 from litellm.responses.litellm_completion_transformation.compaction_items import billable_input_since_newest_compaction
 from litellm.router_utils.add_retry_fallback_headers import (
     ensure_response_additional_headers,
@@ -159,6 +160,12 @@ def _resolve_model_group_alias_via_proxy_router(model: str) -> str | None:
     return resolve_model_group_alias(llm_router.model_group_alias, model)
 
 
+def _tokenizer_models_via_proxy_router(model: str) -> tuple[str, ...]:
+    from litellm.proxy.proxy_server import llm_router
+
+    return tokenizer_models(model, llm_router)
+
+
 def _resolve_ptu_team_ceiling_via_proxy_router(team_id: str, model_group: str) -> PTUTeamCeiling | None:
     from litellm.proxy.proxy_server import llm_router
 
@@ -884,6 +891,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
         ] = _resolve_ptu_team_ceiling_via_proxy_router,
         fail_closed_resolver: Callable[[], bool] = _fail_closed_rate_limit_enforcement_from_general_settings,
         force_hash_tag_grouping_resolver: Callable[[], bool] = _force_redis_hash_tag_grouping_from_litellm_settings,
+        tokenizer_models_resolver: Callable[[str], tuple[str, ...]] = _tokenizer_models_via_proxy_router,
     ):
         self.internal_usage_cache = internal_usage_cache
         self._time_provider = time_provider or datetime.now
@@ -893,6 +901,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
         self._ptu_team_ceiling_resolver = ptu_team_ceiling_resolver
         self._fail_closed_resolver = fail_closed_resolver
         self._force_hash_tag_grouping_resolver = force_hash_tag_grouping_resolver
+        self._tokenizer_models_resolver = tokenizer_models_resolver
         if self.internal_usage_cache.dual_cache.redis_cache is not None:
             self.batch_rate_limiter_script = self.internal_usage_cache.dual_cache.redis_cache.async_register_script(
                 BATCH_RATE_LIMITER_SCRIPT
@@ -4009,15 +4018,18 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
         try:
             estimate: Final = max(
                 0,
-                int(
-                    token_counter(
-                        model=model or "",
-                        messages=countable_messages,
-                        text=selected_text,
-                        tools=countable_tools,
-                        tool_choice=countable_tool_choice,
-                        use_default_image_token_count=True,
+                *(
+                    int(
+                        token_counter(
+                            model=tokenizer_model,
+                            messages=countable_messages,
+                            text=selected_text,
+                            tools=countable_tools,
+                            tool_choice=countable_tool_choice,
+                            use_default_image_token_count=True,
+                        )
                     )
+                    for tokenizer_model in self._tokenizer_models_resolver(model or "")
                 ),
             )
             return estimate + audio_token_estimate

litellm/proxy/hooks/responses_id_security.py

@@ -31,7 +31,7 @@ if TYPE_CHECKING:
 
 
 _RESPONSES_API_PROVIDER_PREFIX: Final = "/openai"
-_RESPONSES_API_CREATE_ROUTES: Final = frozenset({"/v1/responses", "/responses"})
+_RESPONSES_API_ROUTES: Final = frozenset({"/v1/responses", "/responses"})
 _ADDRESSED_RESPONSE_ID_FIELD_BY_CALL_TYPE: Final[Mapping[str, str]] = MappingProxyType(
     {
         "aresponses": "previous_response_id",
@@ -104,7 +104,7 @@ def _rewrite_advertised_id(
     return event
 
 
-def _is_responses_api_create_route(request_route: str | None) -> bool:
+def _streams_response_ids(request_route: str | None) -> bool:
     if request_route is None:
         return False
     canonical: Final = (
@@ -112,7 +112,8 @@ def _is_responses_api_create_route(request_route: str | None) -> bool:
         if request_route.startswith(_RESPONSES_API_PROVIDER_PREFIX + "/")
         else request_route
     )
-    return canonical in _RESPONSES_API_CREATE_ROUTES
+    parent, _, response_id = canonical.rpartition("/")
+    return canonical in _RESPONSES_API_ROUTES or (parent in _RESPONSES_API_ROUTES and response_id != "")
 
 
 class ResponsesIDSecurity(CustomLogger):
@@ -339,7 +340,7 @@ class ResponsesIDSecurity(CustomLogger):
         async for chunk in response:
             if (
                 isinstance(chunk, BaseLiteLLMOpenAIResponseObject)
-                and _is_responses_api_create_route(user_api_key_dict.request_route)
+                and _streams_response_ids(user_api_key_dict.request_route)
                 and not general_settings.get("disable_responses_id_security", False)
             ):
                 chunk = self._encrypt_response_id(chunk, user_api_key_dict, request_encryption_cache)

litellm/proxy/management_endpoints/internal_user_endpoints.py

@@ -46,6 +46,7 @@ from litellm.proxy.auth.password_policy import (
 )
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast
+from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
 from litellm.proxy.common_utils.user_api_key_cache import (
     AUTH_OBJECTS_TARGET,
     object_permission_cache_key,
@@ -1729,21 +1730,9 @@ async def user_update(
     except Exception as e:
         verbose_proxy_logger.exception("litellm.proxy.proxy_server.user_update(): Exception occured - %s", e)
         verbose_proxy_logger.debug(traceback.format_exc())
-        if isinstance(e, HTTPException):
-            raise ProxyException(
-                message=getattr(e, "detail", f"Authentication Error({e})"),
-                type=ProxyErrorTypes.auth_error,
-                param=getattr(e, "param", "None"),
-                code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
-            )
-        elif isinstance(e, ProxyException):
-            raise e
-        raise ProxyException(
-            message="Authentication Error, " + str(e),
-            type=ProxyErrorTypes.auth_error,
-            param=getattr(e, "param", "None"),
-            code=status.HTTP_400_BAD_REQUEST,
-        )
+        if isinstance(e, ProxyException):
+            raise
+        raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
 
 
 async def bulk_update_processed_users(

litellm/proxy/management_endpoints/key_management_endpoints.py

@@ -75,6 +75,7 @@ from litellm.proxy.common_utils.config_sync_pubsub import (
     coordination_redis_cache,
     publish_config_change,
 )
+from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
 from litellm.proxy.common_utils.rbac_utils import check_org_admin_can_generate_keys
 from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time
 from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET, UserApiKeyCache
@@ -4087,21 +4088,9 @@ async def update_key_fn(
         # update based on remaining passed in values
     except Exception as e:
         verbose_proxy_logger.exception("litellm.proxy.proxy_server.update_key_fn(): Exception occured - %s", e)
-        if isinstance(e, HTTPException):
-            raise ProxyException(
-                message=getattr(e, "detail", f"Authentication Error({e})"),
-                type=ProxyErrorTypes.auth_error,
-                param=getattr(e, "param", "None"),
-                code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
-            )
-        elif isinstance(e, ProxyException):
-            raise e
-        raise ProxyException(
-            message="Authentication Error, " + str(e),
-            type=ProxyErrorTypes.auth_error,
-            param=getattr(e, "param", "None"),
-            code=status.HTTP_400_BAD_REQUEST,
-        )
+        if isinstance(e, ProxyException):
+            raise
+        raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
 
 
 @router.post(
@@ -7244,21 +7233,9 @@ async def list_keys(
 
     except Exception as e:
         verbose_proxy_logger.exception("Error in list_keys: %s", e)
-        if isinstance(e, HTTPException):
-            raise ProxyException(
-                message=getattr(e, "detail", f"error({e})"),
-                type=ProxyErrorTypes.internal_server_error,
-                param=getattr(e, "param", "None"),
-                code=getattr(e, "status_code", fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR),
-            )
-        elif isinstance(e, ProxyException):
-            raise e
-        raise ProxyException(
-            message="Authentication Error, " + str(e),
-            type=ProxyErrorTypes.internal_server_error,
-            param=getattr(e, "param", "None"),
-            code=fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR,
-        )
+        if isinstance(e, ProxyException):
+            raise
+        raise proxy_exception_for(e, default_status_code=fastapi.status.HTTP_500_INTERNAL_SERVER_ERROR) from e
 
 
 async def _apply_non_admin_alias_scope(
@@ -7402,21 +7379,9 @@ async def key_aliases(
 
     except Exception as e:
         verbose_proxy_logger.exception("Error in key_aliases: %s", e)
-        if isinstance(e, HTTPException):
-            raise ProxyException(
-                message=getattr(e, "detail", f"error({e})"),
-                type=ProxyErrorTypes.internal_server_error,
-                param=getattr(e, "param", "None"),
-                code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
-            )
-        elif isinstance(e, ProxyException):
-            raise e
-        raise ProxyException(
-            message="Authentication Error, " + str(e),
-            type=ProxyErrorTypes.internal_server_error,
-            param=getattr(e, "param", "None"),
-            code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-        )
+        if isinstance(e, ProxyException):
+            raise
+        raise proxy_exception_for(e, default_status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) from e
 
 
 def _validate_sort_params(

litellm/proxy/management_endpoints/model_management_endpoints.py

@@ -77,6 +77,7 @@ from litellm.proxy.common_utils.encrypt_decrypt_utils import (
     decrypt_value_helper,
     encrypt_value_helper,
 )
+from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
 from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
 from litellm.proxy.db.routing_prisma_wrapper import WriterPinnedClient
 from litellm.proxy.management.teams.authz import TEAM_ADMIN_ONLY, is_team_admin
@@ -2845,21 +2846,9 @@ async def delete_model(
 
     except Exception as e:
         verbose_proxy_logger.exception("Failed to delete model. Due to error - %s", e)
-        if isinstance(e, HTTPException):
-            raise ProxyException(
-                message=getattr(e, "detail", f"Authentication Error({e})"),
-                type=ProxyErrorTypes.auth_error,
-                param=getattr(e, "param", "None"),
-                code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
-            )
-        elif isinstance(e, ProxyException):
-            raise e
-        raise ProxyException(
-            message="Authentication Error, " + str(e),
-            type=ProxyErrorTypes.auth_error,
-            param=getattr(e, "param", "None"),
-            code=status.HTTP_400_BAD_REQUEST,
-        )
+        if isinstance(e, ProxyException):
+            raise
+        raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
 
 
 async def delete_team_model_alias(
@@ -2899,6 +2888,13 @@ async def delete_team_model_alias(
     return removed_model_aliases
 
 
+def _model_not_added_to_db() -> HTTPException:
+    return HTTPException(
+        status_code=500,
+        detail={"error": "Failed to add model to db. Check your server logs for more details."},
+    )
+
+
 @router.post(
     "/model/new",
     description="Allows adding new models to the model list in the config.yaml",
@@ -3080,6 +3076,8 @@ async def add_new_model(
                 if isinstance(e, HTTPException):
                     raise
                 verbose_proxy_logger.exception("Exception in add_new_model: %s", e)
+                if model_response is None:
+                    raise _model_not_added_to_db() from e
 
         else:
             raise HTTPException(
@@ -3088,10 +3086,7 @@ async def add_new_model(
             )
 
         if model_response is None:
-            raise HTTPException(
-                status_code=500,
-                detail={"error": "Failed to add model to db. Check your server logs for more details."},
-            )
+            raise _model_not_added_to_db()
 
         ## CREATE AUDIT LOG ##
         asyncio.create_task(
@@ -3121,21 +3116,9 @@ async def add_new_model(
 
     except Exception as e:
         verbose_proxy_logger.exception("litellm.proxy.proxy_server.add_new_model(): Exception occured - %s", e)
-        if isinstance(e, HTTPException):
-            raise ProxyException(
-                message=getattr(e, "detail", f"Authentication Error({e})"),
-                type=ProxyErrorTypes.auth_error,
-                param=getattr(e, "param", "None"),
-                code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
-            )
-        elif isinstance(e, ProxyException):
-            raise e
-        raise ProxyException(
-            message="Authentication Error, " + str(e),
-            type=ProxyErrorTypes.auth_error,
-            param=getattr(e, "param", "None"),
-            code=status.HTTP_400_BAD_REQUEST,
-        )
+        if isinstance(e, ProxyException):
+            raise
+        raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
 
 
 #### MODEL MANAGEMENT ####
@@ -3364,21 +3347,9 @@ async def update_model(
             return None if model_response is None else _model_write_response(model_response, member_write)
     except Exception as e:
         verbose_proxy_logger.exception("litellm.proxy.proxy_server.update_model(): Exception occured - %s", e)
-        if isinstance(e, HTTPException):
-            raise ProxyException(
-                message=getattr(e, "detail", f"Authentication Error({e})"),
-                type=ProxyErrorTypes.auth_error,
-                param=getattr(e, "param", "None"),
-                code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
-            )
-        elif isinstance(e, ProxyException):
-            raise e
-        raise ProxyException(
-            message="Authentication Error, " + str(e),
-            type=ProxyErrorTypes.auth_error,
-            param=getattr(e, "param", "None"),
-            code=status.HTTP_400_BAD_REQUEST,
-        )
+        if isinstance(e, ProxyException):
+            raise
+        raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
 
 
 @router.post(

litellm/proxy/management_endpoints/organization_endpoints.py

@@ -41,6 +41,7 @@ from litellm.proxy.auth.auth_checks import (
 )
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast
+from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
 from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time
 from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
 from litellm.proxy.management_endpoints.budget_management_endpoints import (
@@ -1396,21 +1397,9 @@ async def organization_member_add(
         )
     except Exception as e:
         verbose_proxy_logger.exception("Error adding member to organization: %s", e)
-        if isinstance(e, HTTPException):
-            raise ProxyException(
-                message=getattr(e, "detail", f"Authentication Error({e})"),
-                type=ProxyErrorTypes.auth_error,
-                param=getattr(e, "param", "None"),
-                code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
-            )
-        elif isinstance(e, ProxyException):
-            raise e
-        raise ProxyException(
-            message="Authentication Error, " + str(e),
-            type=ProxyErrorTypes.auth_error,
-            param=getattr(e, "param", "None"),
-            code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-        )
+        if isinstance(e, ProxyException):
+            raise
+        raise proxy_exception_for(e, default_status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) from e
 
 
 async def find_member_if_email(user_email: str, prisma_client: PrismaClient) -> LiteLLM_UserTable:

litellm/proxy/management_endpoints/team_endpoints.py

@@ -132,6 +132,7 @@ from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import (
 )
 from litellm.proxy.common_utils.callback_utils import encrypt_callback_vars
 from litellm.proxy.common_utils.json_merge_patch import apply_json_merge_patch
+from litellm.proxy.common_utils.openai_error_payload import proxy_exception_for
 from litellm.proxy.common_utils.user_api_key_cache import (
     UserApiKeyCache,
     team_membership_auth_cache_key,
@@ -5526,21 +5527,9 @@ async def team_info(
             e,
             traceback.format_exc(),
         )
-        if isinstance(e, HTTPException):
-            raise ProxyException(
-                message=getattr(e, "detail", f"Authentication Error({e})"),
-                type=ProxyErrorTypes.auth_error,
-                param=getattr(e, "param", "None"),
-                code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
-            )
-        elif isinstance(e, ProxyException):
-            raise e
-        raise ProxyException(
-            message="Authentication Error, " + str(e),
-            type=ProxyErrorTypes.auth_error,
-            param=getattr(e, "param", "None"),
-            code=status.HTTP_400_BAD_REQUEST,
-        )
+        if isinstance(e, ProxyException):
+            raise
+        raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
 
 
 @router.get(

litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py

@@ -50,6 +50,7 @@ from litellm.constants import (
 from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
 from litellm.integrations.custom_guardrail import guardrail_request_data_with_streaming, is_guardrail_intervention
 from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
+from litellm.litellm_core_utils.fallback_generalizations import match_routing_generalization
 from litellm.llms.anthropic.common_utils import AnthropicModelInfo, merge_anthropic_beta_headers
 from litellm.llms.azure.passthrough.transformation import (
     foreign_azure_deployment,
@@ -698,6 +699,20 @@ async def typesafe_proxy_route(
     return await endpoint_func(request, fastapi_response, user_api_key_dict)
 
 
+_COST_MAP_ENTRIES: Final = TypeAdapter(Mapping[str, object])
+
+
+def typesafe_allowlist_model_names() -> tuple[str, ...]:
+    if not passthrough_endpoint_router.has_credentials(custom_llm_provider="typesafe"):
+        return ()
+    cost_map: Final = _COST_MAP_ENTRIES.validate_python(litellm.model_cost)  # pyright: ignore[reportUnknownMemberType]  # legacy registry is validated at this boundary
+    priced_names: Final = (name.removeprefix("typesafe/") for name in cost_map if name.startswith("typesafe/"))
+    jev_models_typesafe_serves: Final = sorted(
+        name for name in priced_names if match_routing_generalization(name) == "typesafe"
+    )
+    return ("typesafe/*", *jev_models_typesafe_serves)
+
+
 @router.post(
     "/laya/v1/systemone",
     tags=["Laya Pass-through", "pass-through"],

litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py

@@ -1,4 +1,5 @@
 import json
+import os
 from collections.abc import Callable
 from typing import TYPE_CHECKING, Final
 
@@ -88,6 +89,14 @@ class PassthroughEndpointRouter:
         )
         return _resolve_oidc_reference(get_secret_str(_env_variable_name))
 
+    def has_credentials(self, custom_llm_provider: str) -> bool:
+        if self._get_deployment_api_key(custom_llm_provider=custom_llm_provider, region_name=None) is not None:
+            return True
+        env_variable_name: Final = self._get_default_env_variable_name_passthrough_endpoint(
+            custom_llm_provider=custom_llm_provider,
+        )
+        return bool(os.environ.get(env_variable_name))
+
     def _get_deployment_api_key(
         self,
         custom_llm_provider: str,

litellm/proxy/proxy_server.py

@@ -100,7 +100,11 @@ from litellm.litellm_core_utils.litellm_logging import (
 )
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
-from litellm.litellm_core_utils.token_counter import UnsupportedContentTypeError, offload_token_count
+from litellm.litellm_core_utils.token_counter import (
+    UnsupportedContentTypeError,
+    deployment_openai_model,
+    offload_token_count,
+)
 from litellm.llms.bedrock.count_tokens.kimi_k3 import (
     HUGGING_FACE_SOURCE,
     KimiK3Encodings,
@@ -524,6 +528,7 @@ from litellm.proxy.common_utils.openai_error_payload import (
     LITELLM_CALL_ID_HEADER,
     headers_with_litellm_call_id,
     litellm_call_id_headers,
+    proxy_exception_for,
     with_litellm_call_id,
 )
 from litellm.proxy.common_utils.periodic_reload_schedule import (
@@ -12390,6 +12395,12 @@ async def model_list(
 
     hidden_names: Final = blocked_names | unhealthy_names
 
+    from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import typesafe_allowlist_model_names
+
+    asks_for_allowlist_names: Final = (
+        include_model_access_groups is True and return_wildcard_routes is True and only_model_access_groups is not True
+    )
+
     # If scope=expand and user has admin privileges, return all proxy models
     if should_expand_scope:
         # Get all proxy models as if user is a proxy admin
@@ -12410,7 +12421,10 @@ async def model_list(
         all_models = get_complete_model_list(
             key_models=[],
             team_models=[],
-            proxy_model_list=proxy_model_list,
+            proxy_model_list=[
+                *proxy_model_list,
+                *(typesafe_allowlist_model_names() if asks_for_allowlist_names else ()),
+            ],
             user_model=None,
             infer_model_from_keys=False,
             return_wildcard_routes=include_wildcard_routes,
@@ -12471,7 +12485,7 @@ async def model_list(
 
     # Otherwise, use the normal behavior (current implementation)
     # Get available models for the user
-    all_models = await get_available_models_for_user(
+    available_models: Final = await get_available_models_for_user(
         user_api_key_dict=user_api_key_dict,
         llm_router=llm_router,
         general_settings=general_settings,
@@ -12484,6 +12498,11 @@ async def model_list(
         return_wildcard_routes=include_wildcard_routes,
         user_api_key_cache=user_api_key_cache,
     )
+    offers_typesafe_allowlist_names: Final = (
+        asks_for_allowlist_names and team_id is None and user_api_key_has_admin_view(user_api_key_dict)
+    )
+    typesafe_names_to_offer: Final = typesafe_allowlist_model_names() if offers_typesafe_allowlist_names else ()
+    all_models = [*available_models, *(name for name in typesafe_names_to_offer if name not in available_models)]
 
     undiscoverable_names: Final = undiscoverable_model_names(
         all_models, llm_router, user_api_key_dict, team_id or user_api_key_dict.team_id
@@ -15527,7 +15546,7 @@ async def _count_request_tokens(
             count_source="local_estimate",
         )
     estimate: Final = await _local_token_estimate(
-        model=model_to_use,
+        model=deployment_openai_model(deployment_view) or model_to_use,
         prompt=prompt if counted_messages is None else None,
         messages=counted_messages,
         tools=counted_tools,
@@ -18522,25 +18541,13 @@ async def async_queue_request(
         await proxy_logging_obj.post_call_failure_hook(
             user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
         )
-        if isinstance(e, HTTPException):
-            raise ProxyException(
-                message=getattr(e, "detail", f"Authentication Error({e})"),
-                type=ProxyErrorTypes.auth_error,
-                param=getattr(e, "param", "None"),
-                code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
-            )
-        elif isinstance(e, ProxyException):
-            raise e
-        elif isinstance(e, litellm.BadRequestError):
+        if isinstance(e, litellm.BadRequestError):
             raise ProxyException(
                 message=e.message, type="invalid_request_error", param=getattr(e, "param", None), code=e.status_code
-            )
-        raise ProxyException(
-            message="Authentication Error, " + str(e),
-            type=ProxyErrorTypes.auth_error,
-            param=getattr(e, "param", "None"),
-            code=status.HTTP_400_BAD_REQUEST,
-        )
+            ) from e
+        if isinstance(e, ProxyException):
+            raise
+        raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
 
 
 @app.get("/fallback/login", tags=["experimental"], include_in_schema=False)
@@ -19819,21 +19826,9 @@ async def update_config(
     except Exception as e:
         verbose_proxy_logger.error("litellm.proxy.proxy_server.update_config(): Exception occured - %s", e)
         verbose_proxy_logger.debug(traceback.format_exc())
-        if isinstance(e, HTTPException):
-            raise ProxyException(
-                message=getattr(e, "detail", f"Authentication Error({e})"),
-                type=ProxyErrorTypes.auth_error,
-                param=getattr(e, "param", "None"),
-                code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
-            )
-        elif isinstance(e, ProxyException):
-            raise e
-        raise ProxyException(
-            message="Authentication Error, " + str(e),
-            type=ProxyErrorTypes.auth_error,
-            param=getattr(e, "param", "None"),
-            code=status.HTTP_400_BAD_REQUEST,
-        )
+        if isinstance(e, ProxyException):
+            raise
+        raise proxy_exception_for(e, default_status_code=status.HTTP_400_BAD_REQUEST) from e
 
 
 ### CONFIG GENERAL SETTINGS
@@ -21016,21 +21011,9 @@ async def get_config(
         }
     except Exception as e:
         verbose_proxy_logger.exception("litellm.proxy.proxy_server.get_config(): Exception occured - %s", e)
-        if isinstance(e, HTTPException):
-            raise ProxyException(
-                message=getattr(e, "detail", f"Authentication Error({e})"),
-                type=ProxyErrorTypes.auth_error,
-                param=getattr(e, "param", "None"),
-                code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
-            )
-        elif isinstance(e, ProxyException):
-            raise e
-        raise ProxyException(
-            message="Authentication Error, " + str(e),
-            type=ProxyErrorTypes.auth_error,
-            param=getattr(e, "param", "None"),
-            code=status.HTTP_400_BAD_REQUEST,
-        )
+        if isinstance(e, ProxyException):
+            raise
+        raise proxy_exception_for(e, default_status_code=status.HTTP_500_INTERNAL_SERVER_ERROR) from e
 
 
 @router.get(

litellm/proxy/public_endpoints/agent_create_fields.json

@@ -127,8 +127,8 @@
       {
         "key": "agent_id",
         "label": "Agent ID",
-        "placeholder": "asst_abc123",
-        "tooltip": "The agent/assistant ID from your Azure AI Foundry project (e.g., asst_abc123)",
+        "placeholder": "my-agent",
+        "tooltip": "The agent name from your Azure AI Foundry project, optionally with a version (e.g., my-agent or my-agent:2), or a classic assistant ID (e.g., asst_abc123)",
         "required": true,
         "field_type": "text",
         "default_value": null,

litellm/proxy/public_endpoints/provider_create_fields.json

@@ -3519,6 +3519,34 @@
     ],
     "default_model_placeholder": "inferenceapis/deepseek-ai/DeepSeek-V4-Flash"
   },
+  {
+    "provider": "StreamLake",
+    "provider_display_name": "StreamLake",
+    "litellm_provider": "streamlake",
+    "credential_fields": [
+      {
+        "key": "api_base",
+        "label": "API Base",
+        "placeholder": "https://vanchin.streamlake.ai/api/gateway/v1/endpoints",
+        "tooltip": null,
+        "required": false,
+        "field_type": "text",
+        "options": null,
+        "default_value": null
+      },
+      {
+        "key": "api_key",
+        "label": "API Key",
+        "placeholder": null,
+        "tooltip": null,
+        "required": true,
+        "field_type": "password",
+        "options": null,
+        "default_value": null
+      }
+    ],
+    "default_model_placeholder": "streamlake/<model-or-endpoint>"
+  },
   {
     "provider": "Snowflake",
     "provider_display_name": "Snowflake",
@@ -4200,5 +4228,33 @@
       }
     ],
     "default_model_placeholder": "cursor/claude-4-sonnet"
+  },
+  {
+    "provider": "BIGMODEL",
+    "provider_display_name": "BigModel (Zhipu AI)",
+    "litellm_provider": "bigmodel",
+    "credential_fields": [
+      {
+        "key": "api_key",
+        "label": "BigModel (Zhipu AI) API Key",
+        "placeholder": null,
+        "tooltip": null,
+        "required": true,
+        "field_type": "password",
+        "options": null,
+        "default_value": null
+      },
+      {
+        "key": "api_base",
+        "label": "API Base",
+        "placeholder": "https://open.bigmodel.cn",
+        "tooltip": null,
+        "required": false,
+        "field_type": "text",
+        "options": null,
+        "default_value": "https://open.bigmodel.cn"
+      }
+    ],
+    "default_model_placeholder": "bigmodel/glm-5.3"
   }
 ]

litellm/proxy/response_api_endpoints/endpoints.py

@@ -39,6 +39,7 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping, get_metadata
 from litellm.litellm_core_utils.internal_call_metadata import is_background_response, requests_background
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
+from litellm.litellm_core_utils.token_counter import usage_tokenizer_model
 from litellm.litellm_core_utils.tool_fields import TOOL_FIELDS, first_validation_error
 from litellm.llms.openai.responses.guardrail_translation.handler import (
     OpenAIResponsesHandler,
@@ -85,6 +86,7 @@ from litellm.proxy.route_llm_request import raise_if_required_body_param_missing
 from litellm.proxy.utils import count_request_input_tokens, handle_exception_on_proxy
 from litellm.responses.litellm_completion_transformation.compaction_items import messages_since_newest_compaction
 from litellm.responses.litellm_completion_transformation.count_request import MODEL_FILE_ID_MAPPING_FIELD
+from litellm.responses.retrieve_stream_query import RETRIEVE_INCLUDE_QUERY_KEYS, RETRIEVE_STREAM_QUERY_KEYS
 from litellm.responses.streaming_iterator import GuardedResponsesWsFrame, ResponsesWsFrameGuard, spawn_logging_task
 from litellm.router import Router
 from litellm.types.llms.base import LiteLLMBaseModel
@@ -96,6 +98,7 @@ from litellm.types.llms.openai import (
 from litellm.types.responses.main import DeleteResponseResult
 from litellm.types.token_count import TokenCountSourceFields
 from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, TokenCountResponse
+from litellm.utils import get_base_model_from_metadata
 
 if TYPE_CHECKING:
     from litellm.proxy.utils import ProxyLogging
@@ -104,11 +107,43 @@ router: Final = APIRouter()
 _RESPONSES_WS_CONFIG_VALUE_ADAPTER: Final[TypeAdapter[object | None]] = TypeAdapter(object | None)
 _DECODED_JSON: Final = TypeAdapter(object)
 _PROXY_SERVER_REQUEST_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
+_RETRIEVE_QUERY_PARAMS: Final = tuple(sorted(RETRIEVE_INCLUDE_QUERY_KEYS | RETRIEVE_STREAM_QUERY_KEYS))
+_RETRIEVE_QUERY_OPENAPI: Final = {
+    "parameters": [
+        {
+            "name": "stream",
+            "in": "query",
+            "required": False,
+            "schema": {"type": "boolean"},
+            "description": "Replay the response's events as server-sent events (OpenAI, Azure, and Azure AI Foundry)",
+        },
+        {
+            "name": "starting_after",
+            "in": "query",
+            "required": False,
+            "schema": {"type": "integer", "minimum": 0},
+            "description": "With stream=true, replay only the events after this sequence number",
+        },
+        {
+            "name": "include",
+            "in": "query",
+            "required": False,
+            "schema": {"type": "array", "items": {"type": "string"}},
+            "description": "Additional output data to include, forwarded to the provider",
+        },
+        {
+            "name": "include_obfuscation",
+            "in": "query",
+            "required": False,
+            "schema": {"type": "boolean"},
+            "description": "With stream=true, whether delta events carry obfuscation padding",
+        },
+    ]
+}
 
 _ResponseDocSchemas: TypeAlias = dict[int | str, dict[str, object]]  # fastapi's responses kwarg
 
-RESPONSES_API_RESPONSE_SCHEMAS: Final[_ResponseDocSchemas] = {200: {"model": ResponsesAPIResponse}}
-RESPONSES_API_CREATE_RESPONSE_SCHEMAS: Final[_ResponseDocSchemas] = {
+RESPONSES_API_RESPONSE_SCHEMAS: Final[_ResponseDocSchemas] = {
     200: {
         "model": ResponsesAPIResponse,
         "content": {
@@ -252,19 +287,19 @@ async def _resolve_cursor_model_variant_before_auth(request: Request) -> None:
     "/v1/responses",
     dependencies=[Depends(user_api_key_auth)],
     tags=["responses"],
-    responses=RESPONSES_API_CREATE_RESPONSE_SCHEMAS,
+    responses=RESPONSES_API_RESPONSE_SCHEMAS,
 )
 @router.post(
     "/responses",
     dependencies=[Depends(user_api_key_auth)],
     tags=["responses"],
-    responses=RESPONSES_API_CREATE_RESPONSE_SCHEMAS,
+    responses=RESPONSES_API_RESPONSE_SCHEMAS,
 )
 @router.post(
     "/openai/v1/responses",
     dependencies=[Depends(user_api_key_auth)],
     tags=["responses"],
-    responses=RESPONSES_API_CREATE_RESPONSE_SCHEMAS,
+    responses=RESPONSES_API_RESPONSE_SCHEMAS,
 )
 async def responses_api(
     request: Request,
@@ -537,7 +572,10 @@ async def cursor_model_list(
 
 def _dispatched_prompt_token_counter(logging_obj: LiteLLMLoggingObj) -> Callable[[], int]:
     return lambda: count_request_input_tokens(
-        model=str(logging_obj.model_call_details.get("model") or ""),
+        model=usage_tokenizer_model(
+            str(logging_obj.model_call_details.get("model") or ""),
+            get_base_model_from_metadata(logging_obj.model_call_details),
+        ),
         request_input=messages_since_newest_compaction(logging_obj.model_call_details.get("messages")),
         system_input=None,
     )
@@ -744,18 +782,21 @@ async def cursor_chat_completions(
     dependencies=[Depends(user_api_key_auth)],
     tags=["responses"],
     responses=RESPONSES_API_RESPONSE_SCHEMAS,
+    openapi_extra=_RETRIEVE_QUERY_OPENAPI,
 )
 @router.get(
     "/responses/{response_id}",
     dependencies=[Depends(user_api_key_auth)],
     tags=["responses"],
     responses=RESPONSES_API_RESPONSE_SCHEMAS,
+    openapi_extra=_RETRIEVE_QUERY_OPENAPI,
 )
 @router.get(
     "/openai/v1/responses/{response_id}",
     dependencies=[Depends(user_api_key_auth)],
     tags=["responses"],
     responses=RESPONSES_API_RESPONSE_SCHEMAS,
+    openapi_extra=_RETRIEVE_QUERY_OPENAPI,
 )
 async def get_response(
     response_id: str,
@@ -780,6 +821,10 @@ async def get_response(
     # Get provider response
     curl -X GET http://localhost:4000/v1/responses/resp_abc123 \
     -H "Authorization: Bearer $LITELLM_MASTER_KEY"
+
+    # Resume a background response's event stream after sequence number 7
+    curl -N "http://localhost:4000/v1/responses/resp_abc123?stream=true&starting_after=7" \
+    -H "Authorization: Bearer $LITELLM_MASTER_KEY"
     ```
     """
     from litellm.proxy.proxy_server import (
@@ -827,6 +872,14 @@ async def get_response(
     data: Final = await read_request_body(request=request)
     read_settles_billing: Final = not carries_client_side_overrides(data)
     data["response_id"] = response_id
+    retrieve_query: Final = {
+        key: values[0] if len(values) == 1 else values
+        for key in _RETRIEVE_QUERY_PARAMS
+        if (values := request.query_params.getlist(key))
+    }
+    if retrieve_query:
+        body_extra_query: Final[object] = data.get("extra_query")
+        data["extra_query"] = {**(body_extra_query if isinstance(body_extra_query, dict) else {}), **retrieve_query}
     processor: Final = ProxyBaseLLMRequestProcessing(data=data)
     try:
         response: Final[object] = await processor.base_process_llm_request(
@@ -838,7 +891,12 @@ async def get_response(
             llm_router=llm_router,
             general_settings=general_settings,
             proxy_config=proxy_config,
-            select_data_generator=select_data_generator,
+            select_data_generator=partial(
+                _streamed_retrieval_data_generator,
+                partial(select_data_generator, responses_stream_errors=True),
+                response_id,
+                read_settles_billing,
+            ),
             model=None,
             user_model=user_model,
             user_temperature=user_temperature,
@@ -912,12 +970,24 @@ async def _bill_finished_background_response(
     fastapi_response: Response,
     read_settles_billing: bool,
 ) -> None:
+    _report_charge(
+        await _settle_finished_background_read(response, response_id, user_api_key_dict, read_settles_billing),
+        fastapi_response,
+    )
+
+
+async def _settle_finished_background_read(
+    response: object,
+    response_id: str,
+    user_api_key_dict: UserAPIKeyAuth,
+    read_settles_billing: bool,
+) -> "_BilledCharge | None":
     from litellm.proxy.proxy_server import proxy_logging_obj
 
     if not isinstance(response, ResponsesAPIResponse) or not is_background_response(response):
-        return
+        return None
     if response.status not in TERMINAL_RESPONSE_STATUSES:
-        return
+        return None
     managed_files_hook: Final = _managed_files_hook(proxy_logging_obj)
     track_untracked: Final = (
         partial(
@@ -928,12 +998,11 @@ async def _bill_finished_background_response(
         if managed_files_hook is not None
         else None
     )
-    charge: Final = await _settle_background_billing(
+    return await _settle_background_billing(
         (lambda cost_check: cost_check.bill_if_finished(response_id, response, track_untracked))
         if read_settles_billing
         else (lambda cost_check: cost_check.bill_from_deployment(response_id, track_untracked))
     )
-    _report_charge(charge, fastapi_response)
 
 
 class _BilledCharge(Protocol):
@@ -1083,27 +1152,53 @@ def _streamed_background_data_generator(
         user_api_key_dict=user_api_key_dict,
         request_tags=_background_response_spend_tags(request_data),
     )
-    return _bill_streamed_background_response(response, chunks, track_untracked)
+    return _bill_streamed_background_response(
+        response, chunks, partial(_bill_finished_stream, track_untracked=track_untracked)
+    )
+
+
+def _streamed_retrieval_data_generator(
+    select_data_generator: Callable[..., AsyncIterator[str | bytes]],
+    response_id: str,
+    read_settles_billing: bool,
+    response: object,
+    user_api_key_dict: UserAPIKeyAuth,
+    request_data: Mapping[str, object],
+    request: Request | None = None,
+) -> AsyncIterator[str | bytes]:
+    chunks: Final = select_data_generator(
+        response=response, user_api_key_dict=user_api_key_dict, request_data=request_data, request=request
+    )
+    return _bill_streamed_background_response(
+        response,
+        chunks,
+        partial(
+            _settle_finished_background_read,
+            response_id=response_id,
+            user_api_key_dict=user_api_key_dict,
+            read_settles_billing=read_settles_billing,
+        ),
+    )
 
 
 async def _bill_streamed_background_response(
     stream: object,
     chunks: AsyncIterator[str | bytes],
-    track_untracked: Callable[[str, ResponsesAPIResponse], Awaitable[bool]],
+    bill: Callable[[ResponsesAPIResponse], Awaitable[object]],
 ) -> AsyncIterator[str | bytes]:
     async for chunk in chunks:
         completed: object = ProxyBaseLLMRequestProcessing.extract_completed_responses_response(stream)
         if not isinstance(completed, ResponsesAPIResponse):
             yield chunk
             continue
-        await _bill_finished_stream(completed, track_untracked)
+        await bill(completed)
         yield chunk
         async for remaining_chunk in chunks:
             yield remaining_chunk
         return
     late_completed: Final = ProxyBaseLLMRequestProcessing.extract_completed_responses_response(stream)
     if isinstance(late_completed, ResponsesAPIResponse):
-        await _bill_finished_stream(late_completed, track_untracked)
+        await bill(late_completed)
 
 
 async def _bill_finished_stream(